{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"}],"dockerImageVersionId":30839,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install git+https://github.com/copick/copick-utils.git matplotlib tqdm copick \n!pip install -q \"monai-weekly[mlflow]\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:17:40.290815Z","iopub.execute_input":"2025-01-20T06:17:40.291276Z","iopub.status.idle":"2025-01-20T06:18:13.572947Z","shell.execute_reply.started":"2025-01-20T06:17:40.291237Z","shell.execute_reply":"2025-01-20T06:18:13.571653Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install zarr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:18:13.574615Z","iopub.execute_input":"2025-01-20T06:18:13.574935Z","iopub.status.idle":"2025-01-20T06:18:18.122865Z","shell.execute_reply.started":"2025-01-20T06:18:13.574906Z","shell.execute_reply":"2025-01-20T06:18:18.121574Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\nconfig_blob = \"\"\"{\n    \"name\": \"czii_cryoet_mlchallenge_2024\",\n    \"description\": \"2024 CZII CryoET ML Challenge training data.\",\n    \"version\": \"1.0.0\",\n\n    \"pickable_objects\": [\n        {\n            \"name\": \"apo-ferritin\",\n            \"is_particle\": true,\n            \"pdb_id\": \"4V1W\",\n            \"label\": 1,\n            \"color\": [  0, 117, 220, 128],\n            \"radius\": 60,\n            \"map_threshold\": 0.0418\n        },\n        {\n            \"name\": \"beta-amylase\",\n            \"is_particle\": true,\n            \"pdb_id\": \"1FA2\",\n            \"label\": 2,\n            \"color\": [153,  63,   0, 128],\n            \"radius\": 65,\n            \"map_threshold\": 0.035\n        },\n        {\n            \"name\": \"beta-galactosidase\",\n            \"is_particle\": true,\n            \"pdb_id\": \"6X1Q\",\n            \"label\": 3,\n            \"color\": [ 76,   0,  92, 128],\n            \"radius\": 90,\n            \"map_threshold\": 0.0578\n        },\n        {\n            \"name\": \"ribosome\",\n            \"is_particle\": true,\n            \"pdb_id\": \"6EK0\",\n            \"label\": 4,\n            \"color\": [  0,  92,  49, 128],\n            \"radius\": 150,\n            \"map_threshold\": 0.0374\n        },\n        {\n            \"name\": \"thyroglobulin\",\n            \"is_particle\": true,\n            \"pdb_id\": \"6SCJ\",\n            \"label\": 5,\n            \"color\": [ 43, 206,  72, 128],\n            \"radius\": 130,\n            \"map_threshold\": 0.0278\n        },\n        {\n            \"name\": \"virus-like-particle\",\n            \"is_particle\": true,\n            \"pdb_id\": \"6N4V\",            \n            \"label\": 6,\n            \"color\": [255, 204, 153, 128],\n            \"radius\": 135,\n            \"map_threshold\": 0.201\n        }\n    ],\n\n    \"overlay_root\": \"/kaggle/working/overlay\",\n\n    \"overlay_fs_args\": {\n        \"auto_mkdir\": true\n    },\n\n    \"static_root\": \"/kaggle/input/czii-cryo-et-object-identification/train/static\"\n}\"\"\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:19:11.477907Z","iopub.execute_input":"2025-01-20T06:19:11.478345Z","iopub.status.idle":"2025-01-20T06:19:11.483949Z","shell.execute_reply.started":"2025-01-20T06:19:11.478308Z","shell.execute_reply":"2025-01-20T06:19:11.482574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"copick_config_path = \"/kaggle/working/copick.config\"\nouput_overlay = \"/kaggle/working/overlay\"\n\nwith open(copick_config_path,\"w\") as f:\n    f.write(config_blob)\n\nsource_dir =  '/kaggle/input/czii-cryo-et-object-identification/train/overlay'\ndestination_dir = '/kaggle/working/overlay'\n\nfor root, dirs,files in os.walk(source_dir):\n    relative_path = os.path.relpath(root,source_dir)\n    target_dir= os.path.join(destination_dir,relative_path)\n    os.makedirs(target_dir,exist_ok = True)\n    #print(relative_path)\n    #print(root)\n    #print(dirs)\n    #print(files)\n    #print(\"-----\")\n\n    # copy and rename each file\n    for file in files:\n        if file.startswith(\"curation_0_\"):\n            new_filename = file\n        else:\n            new_filename = f\"curation_0_{file}\"\n\n        #Define full paths for the source and destination files\n        source_file = os.path.join(root,file)\n        destination_file = os.path.join(target_dir,new_filename)\n\n        # copy the file with the new name\n        shutil.copy2(source_file,destination_file)\n        print(f\"Copied {source_file} to {destination_file}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:19:11.756402Z","iopub.execute_input":"2025-01-20T06:19:11.756759Z","iopub.status.idle":"2025-01-20T06:19:12.023811Z","shell.execute_reply.started":"2025-01-20T06:19:11.756729Z","shell.execute_reply":"2025-01-20T06:19:12.022566Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom pathlib import Path\n\nimport torch\nimport torchinfo\nimport zarr,copick\nfrom tqdm import tqdm\nfrom monai.data import DataLoader, Dataset,CacheDataset,decollate_batch\nfrom monai.transforms import(\n    Compose,\n    EnsureChannelFirstd,\n    Orientationd,\n    AsDiscrete,\n    RandFlipd,\n    RandRotate90d, \n    NormalizeIntensityd,\n    RandCropByLabelClassesd,\n)\nfrom monai.networks.nets import UNet\nfrom monai.losses import DiceLoss, FocalLoss, TverskyLoss\nfrom monai.metrics import DiceMetric, ConfusionMatrixMetric\nimport mlflow\nimport mlflow.pytorch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:19:16.565066Z","iopub.execute_input":"2025-01-20T06:19:16.565431Z","iopub.status.idle":"2025-01-20T06:19:53.127615Z","shell.execute_reply.started":"2025-01-20T06:19:16.565401Z","shell.execute_reply":"2025-01-20T06:19:53.126752Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"root = copick.from_file(copick_config_path)\ncopick_user_name = \"copickUtils\"\ncopick_segmentation_name = \"paintedPicks\"\nvoxel_size =10\ntomo_type=\"denoised\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:34:02.500344Z","iopub.execute_input":"2025-01-20T06:34:02.500728Z","iopub.status.idle":"2025-01-20T06:34:02.506891Z","shell.execute_reply.started":"2025-01-20T06:34:02.500702Z","shell.execute_reply":"2025-01-20T06:34:02.505754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#print(root.version)             # \"1.0.0\"\n#print(root.static_root)         # \"/kaggle/input/czii-cryo-et-object-identification/train/static\"\nprint(root.pickable_objects)    # List of pickable objects\nprint(root.runs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:34:02.806726Z","iopub.execute_input":"2025-01-20T06:34:02.807306Z","iopub.status.idle":"2025-01-20T06:34:02.824073Z","shell.execute_reply.started":"2025-01-20T06:34:02.807259Z","shell.execute_reply":"2025-01-20T06:34:02.823087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from copick_utils.segmentation import segmentation_from_picks\nimport copick_utils.writers.write as write\nfrom collections import defaultdict\n\n#Just do this once\ngenerate_masks = True\n\nif generate_masks:\n    target_objects = defaultdict(dict)\n    for object in root.pickable_objects:\n        if object.is_particle:\n            target_objects[object.name]['label'] = object.label\n            target_objects[object.name]['radius'] = object.radius\n\n    for run in tqdm(root.runs):\n        tomo = run.get_voxel_spacing(10)\n        tomo = tomo.get_tomogram(tomo_type).numpy()\n        target = np.zeros(tomo.shape,dtype=np.uint8)\n        for pickable_object in root.pickable_objects:\n            pick = run.get_picks(object_name = pickable_object.name, user_id = \"curation\")\n            if len(pick):\n                target = segmentation_from_picks.from_picks(pick[0],\n                                                            target,\n                                                            target_objects[pickable_object.name]['radius']*0.8,\n                                                            target_objects[pickable_object.name]['label']\n                                                           )\n                write.segmentation(run, target, copick_user_name, name=copick_segmentation_name)\n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:34:04.590839Z","iopub.execute_input":"2025-01-20T06:34:04.591368Z","iopub.status.idle":"2025-01-20T06:34:25.513512Z","shell.execute_reply.started":"2025-01-20T06:34:04.591199Z","shell.execute_reply":"2025-01-20T06:34:25.512541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dicts = []\nfor run in tqdm(root.runs):\n    tomogram = run.get_voxel_spacing(voxel_size).get_tomogram(tomo_type).numpy()\n    segmentation = run.get_segmentations(name=copick_segmentation_name, user_id=copick_user_name, voxel_size=voxel_size, is_multilabel=True)[0].numpy()\n    data_dicts.append({\"name\": run.name, \"image\": tomogram, \"label\": segmentation})\n    \nprint(np.unique(data_dicts[0]['label']))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:34:25.514634Z","iopub.execute_input":"2025-01-20T06:34:25.514979Z","iopub.status.idle":"2025-01-20T06:34:42.576456Z","shell.execute_reply.started":"2025-01-20T06:34:25.514948Z","shell.execute_reply":"2025-01-20T06:34:42.575500Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dicts[0]['label'].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:34:42.578776Z","iopub.execute_input":"2025-01-20T06:34:42.579155Z","iopub.status.idle":"2025-01-20T06:34:42.588176Z","shell.execute_reply.started":"2025-01-20T06:34:42.579120Z","shell.execute_reply":"2025-01-20T06:34:42.587077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dicts[0]['image'].shape # (184, 630, 630)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:34:42.589337Z","iopub.execute_input":"2025-01-20T06:34:42.589769Z","iopub.status.idle":"2025-01-20T06:34:42.612933Z","shell.execute_reply.started":"2025-01-20T06:34:42.589735Z","shell.execute_reply":"2025-01-20T06:34:42.611674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(7):\n    with open(f\"train_image_{data_dicts[i]['name']}.npy\", 'wb') as f:\n        np.save(f, data_dicts[i]['image'])\n        \n    with open(f\"train_label_{data_dicts[i]['name']}.npy\", 'wb') as f:\n        np.save(f, data_dicts[i]['label'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:34:42.614149Z","iopub.execute_input":"2025-01-20T06:34:42.614582Z","iopub.status.idle":"2025-01-20T06:34:50.895345Z","shell.execute_reply.started":"2025-01-20T06:34:42.614546Z","shell.execute_reply":"2025-01-20T06:34:50.889665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls -lh","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-20T06:34:50.898817Z","iopub.execute_input":"2025-01-20T06:34:50.899386Z","iopub.status.idle":"2025-01-20T06:34:51.169792Z","shell.execute_reply.started":"2025-01-20T06:34:50.899322Z","shell.execute_reply":"2025-01-20T06:34:51.165645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}