{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91249,"databundleVersionId":11294684,"sourceType":"competition"},{"sourceId":11153971,"sourceType":"datasetVersion","datasetId":6959173}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Thanks @brendanartley for sharing his extra data data.\n\nThis notebook preload tomograms from CZII portal and rescale labels provided by @brendanartley in order to match competition dataset format to help people like me that have custom data processing pipelines. It's not tested a lot(~20 tomograms was loaded and processed this way, and seem to work), please provide feedback if you find errors.\n\noriginal code - https://www.kaggle.com/code/brendanartley/flagellar-motors-dataset-code \noriginal dataset - https://www.kaggle.com/datasets/brendanartley/cryoet-flagellar-motors-dataset\n","metadata":{}},{"cell_type":"code","source":"!pip install zarr cryoet_data_portal -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-25T18:11:01.217778Z","iopub.execute_input":"2025-03-25T18:11:01.218037Z","iopub.status.idle":"2025-03-25T18:11:11.708763Z","shell.execute_reply.started":"2025-03-25T18:11:01.218012Z","shell.execute_reply":"2025-03-25T18:11:11.707687Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom typing import Tuple\nimport os\nimport glob\nimport shutil\nimport scipy\n\nimport zarr\nfrom cryoet_data_portal import Client, Dataset\nimport numpy as np\nfrom tqdm import tqdm\nimport cv2\n\nclient = Client()\n\n# Datasets by Author\n# ds_all = Dataset.find(client, [Dataset.authors.name == \"Yi-Wei Chang\"])\n# ds_all = Dataset.find(client, [Dataset.authors.name == \"Ariane Briegel\"])\nds_all = Dataset.find(client, [Dataset.authors.name == \"Morgan Beeby\"])\nprint(\"=\"*25)\nprint(\"N_DATASETS:\", len(ds_all))\nprint(\"=\"*25)\n\nexample = pd.read_csv(\"/kaggle/input/byu-locating-bacterial-flagellar-motors-2025/train_labels.csv\")\nprocessed_data = pd.read_csv(\"/kaggle/input/cryoet-flagellar-motors-dataset/labels.csv\")\n\n\nrow_id = 0\nnew_data = example.iloc[0:0].copy()\n\nD,H,W = 128,512,512 #original shape\n\ntmp_dir = \"new_data/tmp\"\n# ========= Process Single Dataset ==========\nfor ds in ds_all:\n    s= \"TOTAL: {:<10}     TITLE: {}\".format(\n        len(ds.runs),\n        ds.title,\n    )\n    print(s)\n\n    # Process runs\n    for run in ds.runs:\n        os.makedirs(tmp_dir, exist_ok=True)\n        try:\n            out_dir = f\"new_data/train/{run.name}\"\n            if not os.path.exists(out_dir):\n                os.makedirs(out_dir)\n\n            tomo= run.tomograms[0]\n\n            zarr_path = f\"new_data/tmp/{run.name}.zarr\"\n            #if not os.path.exists(zarr_path):\n            tomo.download_omezarr(dest_path=tmp_dir)\n\n            arr = zarr.open(zarr_path, mode='r')\n            arr = arr[0]\n\n            for i, img in tqdm(enumerate(arr)):\n                cv2.imwrite(f\"{out_dir}/slice_{str(i).zfill(4)}.jpg\", (img*255).astype(np.uint8))\n\n\n            shape = arr.shape\n            data = processed_data[processed_data.tomo_id == run.name]\n\n            for i,row in data.iterrows():\n                new_data.loc[len(new_data)] = {\n                    \"row_id\": row_id,\n                    \"tomo_id\": run.name,\n                    \"Motor axis 0\": row.z * (shape[0]/D),\n                    \"Motor axis 1\": row.y * (shape[1]/H),\n                    \"Motor axis 2\": row.x * (shape[2]/W),\n                    \"Array shape (axis 0)\": shape[0],\n                    \"Array shape (axis 1)\": shape[1],\n                    \"Array shape (axis 2)\": shape[2],\n                    \"Voxel spacing\": tomo.voxel_spacing,\n                    \"Number of motors\":len(data)\n                }\n                row_id +=1\n\n            \n        except Exception as e:\n            print(e)\n        shutil.rmtree(tmp_dir)\n    #     break # <-------------- COMMENT OUT HERE FOR FULL COLLECTION\n    # break  # <-------------- COMMENT OUT HERE FOR FULL COLLECTION\n        \nnew_data.to_csv(\"new_data/new_labels.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-25T18:19:04.628266Z","iopub.execute_input":"2025-03-25T18:19:04.628643Z","iopub.status.idle":"2025-03-25T18:20:30.757307Z","shell.execute_reply.started":"2025-03-25T18:19:04.628605Z","shell.execute_reply":"2025-03-25T18:20:30.755727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}