{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91249,"databundleVersionId":11294684,"sourceType":"competition"},{"sourceId":11315004,"sourceType":"datasetVersion","datasetId":6959173}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Reference\n- {Discussion} [More Motor Annotations](https://www.kaggle.com/competitions/byu-locating-bacterial-flagellar-motors-2025/discussion/569921)\n- {Notebook} [@brendanartley extra data [competition format]](https://www.kaggle.com/code/fautei/brendanartley-extra-data-competition-format/notebook)\n- {Notebook} [Flagellar Motors Dataset Code](https://www.kaggle.com/code/brendanartley/flagellar-motors-dataset-code)\n- {Discussion} [CryoET Dataset with Pixel Anomalies Corrected](https://www.kaggle.com/competitions/byu-locating-bacterial-flagellar-motors-2025/discussion/575028)\n- {Notebook} [📥 BYU: Download & Correct Pixel Annomaly](https://www.kaggle.com/code/tatamikenn/byu-download-correct-pixel-annomaly)","metadata":{}},{"cell_type":"code","source":"!pip install zarr cryoet_data_portal -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T12:44:54.245181Z","iopub.execute_input":"2025-05-17T12:44:54.245432Z","iopub.status.idle":"2025-05-17T12:45:04.828156Z","shell.execute_reply.started":"2025-05-17T12:44:54.245408Z","shell.execute_reply":"2025-05-17T12:45:04.827028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\nfrom pathlib import Path\nfrom PIL import Image\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport zarr\nfrom cryoet_data_portal import Client, Dataset, Run","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T13:11:31.283306Z","iopub.execute_input":"2025-05-17T13:11:31.283862Z","iopub.status.idle":"2025-05-17T13:11:31.288840Z","shell.execute_reply.started":"2025-05-17T13:11:31.283823Z","shell.execute_reply":"2025-05-17T13:11:31.287770Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRUST = 12","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T13:11:37.364357Z","iopub.execute_input":"2025-05-17T13:11:37.364773Z","iopub.status.idle":"2025-05-17T13:11:37.369180Z","shell.execute_reply.started":"2025-05-17T13:11:37.364744Z","shell.execute_reply":"2025-05-17T13:11:37.368032Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Labels","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/cryoet-flagellar-motors-dataset/labels.csv\")\nprint(df.shape)\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T13:11:39.419409Z","iopub.execute_input":"2025-05-17T13:11:39.419751Z","iopub.status.idle":"2025-05-17T13:11:39.444281Z","shell.execute_reply.started":"2025-05-17T13:11:39.419721Z","shell.execute_reply":"2025-05-17T13:11:39.443367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T13:11:40.822967Z","iopub.execute_input":"2025-05-17T13:11:40.823299Z","iopub.status.idle":"2025-05-17T13:11:40.848095Z","shell.execute_reply.started":"2025-05-17T13:11:40.823271Z","shell.execute_reply":"2025-05-17T13:11:40.847261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Unique Tomograms: {df['tomo_id'].nunique()}\")\nprint(f\"Unique Datasets:  {df['dataset_id'].nunique()}\")\nprint(f\"{(df.groupby('tomo_id').size()==1).sum():4d} tomos with one motor\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T13:11:41.417142Z","iopub.execute_input":"2025-05-17T13:11:41.417492Z","iopub.status.idle":"2025-05-17T13:11:41.429719Z","shell.execute_reply.started":"2025-05-17T13:11:41.417464Z","shell.execute_reply":"2025-05-17T13:11:41.428636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tomo_counts = df['tomo_id'].value_counts()\ntomo_counts","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tomo_ids = tomo_counts[tomo_counts==1].index\nlen(tomo_ids)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tomo_ids_1 = tomo_ids[:280]\ntomo_ids_2 = tomo_ids[280:560]\ntomo_ids_3 = tomo_ids[560:840]\ntomo_ids_4 = tomo_ids[840:]\nprint(len(tomo_ids_1))\nprint(len(tomo_ids_2))\nprint(len(tomo_ids_3))\nprint(len(tomo_ids_4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T13:13:01.289982Z","iopub.execute_input":"2025-05-17T13:13:01.290296Z","iopub.status.idle":"2025-05-17T13:13:01.297299Z","shell.execute_reply.started":"2025-05-17T13:13:01.290272Z","shell.execute_reply":"2025-05-17T13:13:01.296170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TOMO_IDS = tomo_ids_1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T13:03:18.888145Z","iopub.execute_input":"2025-05-17T13:03:18.888538Z","iopub.status.idle":"2025-05-17T13:03:18.892776Z","shell.execute_reply.started":"2025-05-17T13:03:18.888509Z","shell.execute_reply":"2025-05-17T13:03:18.891658Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Directories","metadata":{}},{"cell_type":"code","source":"path_tmp = Path('/kaggle/working/tmp')\npath_jpgs = Path('/kaggle/working/jpgs')\nD, H, W = 128, 512, 512","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T13:03:29.064522Z","iopub.execute_input":"2025-05-17T13:03:29.064870Z","iopub.status.idle":"2025-05-17T13:03:29.069658Z","shell.execute_reply.started":"2025-05-17T13:03:29.064839Z","shell.execute_reply":"2025-05-17T13:03:29.068585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tmp dir\nif path_tmp.is_dir():\n    shutil.rmtree(path_tmp)\npath_tmp.mkdir()\n\n# Out dir\npath_jpgs.mkdir()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T13:03:31.689534Z","iopub.execute_input":"2025-05-17T13:03:31.689846Z","iopub.status.idle":"2025-05-17T13:03:31.872804Z","shell.execute_reply.started":"2025-05-17T13:03:31.689823Z","shell.execute_reply":"2025-05-17T13:03:31.871501Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# CziiCollector","metadata":{}},{"cell_type":"code","source":"def normalize_to_uint8(x):\n    lower, upper= np.min(x), np.max(x)\n    x = (x - lower) / (upper - lower)\n    return (x*255).astype(np.uint8)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"client = Client()\n\nrow_id_list = []\ntomo_id_list = []\ncz_list = []\ncy_list = []\ncx_list = []\nzmax_list = []\nymax_list = []\nxmax_list = []\nvoxel_spacing_list = []\nnum_motor_list = []\n\n# ========= Process Run ==========\nrow_id = 0\nfor idx, tomo_id in enumerate(TOMO_IDS):\n    df_motors = df[df['tomo_id'] == tomo_id]\n    row = df_motors.iloc[0]\n    \n    run = Run.find(client, query_filters=[Run.name == tomo_id])\n    if len(run) == 0:\n        print(\"MISSING: \", tomo_id)\n        continue\n    else:\n        run= run[0]\n\n    # Download tomo\n    try:\n        tomo= run.tomograms[0]\n        tomo.download_omezarr(dest_path=path_tmp)\n\n        # Load tomo\n        path_zarr = path_tmp / f\"{tomo_id}.zarr\"\n        arr = zarr.open(path_zarr, mode='r')\n        arr = arr[0]        \n        shape = arr.shape\n\n        cz = round(row['z'] * (shape[0] / D))\n        cy = round(row['y'] * (shape[1] / H))\n        cx = round(row['x'] * (shape[2] / W))\n        zmax = shape[0]\n        zmin = max(0, cz - TRUST)\n        zmax = min(zmax - 1, cz + TRUST)\n        \n        np_slices = arr[zmin:zmax+1]\n        np_slices_norm = normalize_to_uint8(np_slices)\n\n        path_tomo = path_jpgs / tomo_id  # /kaggle/working/jpgs/[tomo_id]\n        path_tomo.mkdir(parents=True, exist_ok=True)\n        for i in range(np_slices_norm.shape[0]):            \n            np_img = np_slices_norm[i]\n            slice_filename = f\"slice_{zmin + i:04d}.jpg\"\n            path_dest = path_tomo / slice_filename\n            Image.fromarray(np_img).save(path_dest)\n\n        # prepare row\n        row_id_list.append(row_id)\n        tomo_id_list.append(run.name)\n        cz_list.append(cz)\n        cy_list.append(cy)\n        cx_list.append(cx)\n        zmax_list.append(shape[0])\n        ymax_list.append(shape[1])\n        xmax_list.append(shape[2])\n        voxel_spacing_list.append(tomo.voxel_spacing)\n        num_motor_list.append(len(df_motors))\n        row_id += 1\n    \n    except Exception as e:\n        print(e)\n        print(\"FAILED:\", tomo_id)\n\n    # Clear tmp\n    shutil.rmtree(path_tmp)\n    path_tmp.mkdir()\n\n    print(f\"[{idx+1:04d} / {len(TOMO_IDS)}] Done. {tomo_id}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T01:09:06.655961Z","iopub.execute_input":"2025-05-17T01:09:06.65653Z","iopub.status.idle":"2025-05-17T01:09:57.600169Z","shell.execute_reply.started":"2025-05-17T01:09:06.656483Z","shell.execute_reply":"2025-05-17T01:09:57.598334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# labels.csv\ndf_new_label = pd.DataFrame({\n    \"row_id\": row_id_list,\n    \"tomo_id\": tomo_id_list,\n    \"Motor axis 0\": cz_list,\n    \"Motor axis 1\": cy_list,\n    \"Motor axis 2\": cx_list,\n    \"Array shape (axis 0)\": zmax_list,\n    \"Array shape (axis 1)\": ymax_list,\n    \"Array shape (axis 2)\": xmax_list,\n    \"Voxel spacing\": voxel_spacing_list,\n    \"Number of motors\":num_motor_list,\n})\ndf_new_label.to_csv(\"labels.csv\", index=False)\nprint(df_new_label.shape)\ndf_new_label.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-17T01:10:04.651128Z","iopub.execute_input":"2025-05-17T01:10:04.651508Z","iopub.status.idle":"2025-05-17T01:10:04.668632Z","shell.execute_reply.started":"2025-05-17T01:10:04.651467Z","shell.execute_reply":"2025-05-17T01:10:04.667561Z"}},"outputs":[],"execution_count":null}]}