{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip -q install copick","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:36:58.743562Z","iopub.execute_input":"2024-11-29T17:36:58.744019Z","iopub.status.idle":"2024-11-29T17:37:09.563992Z","shell.execute_reply.started":"2024-11-29T17:36:58.743984Z","shell.execute_reply":"2024-11-29T17:37:09.56236Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -q monai-weekly","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:37:09.566663Z","iopub.execute_input":"2024-11-29T17:37:09.567092Z","iopub.status.idle":"2024-11-29T17:37:21.285313Z","shell.execute_reply.started":"2024-11-29T17:37:09.567052Z","shell.execute_reply":"2024-11-29T17:37:21.282255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -q git+https://github.com/copick/copick-utils.git matplotlib tqdm copick","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:37:21.289105Z","iopub.execute_input":"2024-11-29T17:37:21.289787Z","iopub.status.idle":"2024-11-29T17:37:44.513021Z","shell.execute_reply.started":"2024-11-29T17:37:21.289709Z","shell.execute_reply":"2024-11-29T17:37:44.510874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import copick\nimport os\nimport shutil\nimport pandas as pd\nimport numpy as np\nimport gc\nimport pickle\n\nimport torch\nimport torch.nn.functional as F\n\nfrom tqdm import tqdm\n\nfrom monai.losses import DiceLoss, FocalLoss, TverskyLoss\nfrom monai.metrics import DiceMetric, ConfusionMatrixMetric\n\nfrom copick_utils.segmentation import segmentation_from_picks\nimport copick_utils.writers.write as write\nfrom collections import defaultdict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:37:44.516033Z","iopub.execute_input":"2024-11-29T17:37:44.516685Z","iopub.status.idle":"2024-11-29T17:37:44.526605Z","shell.execute_reply.started":"2024-11-29T17:37:44.516618Z","shell.execute_reply":"2024-11-29T17:37:44.525468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"config_blob = \"\"\"{\n    \"name\": \"czii_cryoet_mlchallenge_2024\",\n    \"description\": \"2024 CZII CryoET ML Challenge training data.\",\n    \"version\": \"1.0.0\",\n\n    \"pickable_objects\": [\n        {\n            \"name\": \"apo-ferritin\",\n            \"is_particle\": true,\n            \"pdb_id\": \"4V1W\",\n            \"label\": 1,\n            \"color\": [  0, 117, 220, 128],\n            \"radius\": 60,\n            \"map_threshold\": 0.0418\n        },\n        {\n            \"name\": \"beta-amylase\",\n            \"is_particle\": true,\n            \"pdb_id\": \"1FA2\",\n            \"label\": 2,\n            \"color\": [153,  63,   0, 128],\n            \"radius\": 65,\n            \"map_threshold\": 0.035\n        },\n        {\n            \"name\": \"beta-galactosidase\",\n            \"is_particle\": true,\n            \"pdb_id\": \"6X1Q\",\n            \"label\": 3,\n            \"color\": [ 76,   0,  92, 128],\n            \"radius\": 90,\n            \"map_threshold\": 0.0578\n        },\n        {\n            \"name\": \"ribosome\",\n            \"is_particle\": true,\n            \"pdb_id\": \"6EK0\",\n            \"label\": 4,\n            \"color\": [  0,  92,  49, 128],\n            \"radius\": 150,\n            \"map_threshold\": 0.0374\n        },\n        {\n            \"name\": \"thyroglobulin\",\n            \"is_particle\": true,\n            \"pdb_id\": \"6SCJ\",\n            \"label\": 5,\n            \"color\": [ 43, 206,  72, 128],\n            \"radius\": 130,\n            \"map_threshold\": 0.0278\n        },\n        {\n            \"name\": \"virus-like-particle\",\n            \"is_particle\": true,\n            \"label\": 6,\n            \"color\": [255, 204, 153, 128],\n            \"radius\": 135,\n            \"map_threshold\": 0.201\n        },\n        {\n            \"name\": \"membrane\",\n            \"is_particle\": false,\n            \"label\": 8,\n            \"color\": [100, 100, 100, 128]\n        },\n        {\n            \"name\": \"background\",\n            \"is_particle\": false,\n            \"label\": 9,\n            \"color\": [10, 150, 200, 128]\n        }\n    ],\n\n    \"overlay_root\": \"/kaggle/working/overlay\",\n\n    \"overlay_fs_args\": {\n        \"auto_mkdir\": true\n    },\n\n    \"static_root\": \"/kaggle/input/czii-cryo-et-object-identification/train/static\"\n}\"\"\"\n\ncopick_config_path = \"/kaggle/working/copick.config\"\noutput_overlay = \"/kaggle/working/overlay\"\n\nwith open(copick_config_path, \"w\") as f:\n    f.write(config_blob)\n    \n# Update the overlay\n# Define source and destination directories\nsource_dir = '/kaggle/input/czii-cryo-et-object-identification/train/overlay'\ndestination_dir = '/kaggle/working/overlay'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:37:44.529958Z","iopub.execute_input":"2024-11-29T17:37:44.530292Z","iopub.status.idle":"2024-11-29T17:37:44.550001Z","shell.execute_reply.started":"2024-11-29T17:37:44.53026Z","shell.execute_reply":"2024-11-29T17:37:44.548776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for root, dirs, files in os.walk(source_dir):\n    relative_path = os.path.relpath(root, source_dir)\n    target_dir = os.path.join(destination_dir, relative_path)\n    # print()\n    # print(relative_path)\n    # print(target_dir)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:37:44.551125Z","iopub.execute_input":"2024-11-29T17:37:44.551467Z","iopub.status.idle":"2024-11-29T17:37:44.587657Z","shell.execute_reply.started":"2024-11-29T17:37:44.551436Z","shell.execute_reply":"2024-11-29T17:37:44.58639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Walk through the source directory\nfor root, dirs, files in os.walk(source_dir):\n    # Create corresponding subdirectories in the destination\n    relative_path = os.path.relpath(root, source_dir)\n    target_dir = os.path.join(destination_dir, relative_path)\n    os.makedirs(target_dir, exist_ok=True)\n    \n    # Copy and rename each file\n    for file in files:\n        if file.startswith(\"curation_0_\"):\n            new_filename = file\n        else:\n            new_filename = f\"curation_0_{file}\"\n            \n        \n        # Define full paths for the source and destination files\n        source_file = os.path.join(root, file)\n        destination_file = os.path.join(target_dir, new_filename)\n        \n        # Copy the file with the new name\n        shutil.copy2(source_file, destination_file)\n        # print(f\"Copied {source_file} to {destination_file}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:37:44.589292Z","iopub.execute_input":"2024-11-29T17:37:44.58969Z","iopub.status.idle":"2024-11-29T17:37:44.774907Z","shell.execute_reply.started":"2024-11-29T17:37:44.589655Z","shell.execute_reply":"2024-11-29T17:37:44.773659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"root = copick.from_file(copick_config_path)\n\ncopick_user_name = \"copickUtils\"\ncopick_segmentation_name = \"paintedPicks\"\nvoxel_size = 10#.012144 # https://www.kaggle.com/competitions/czii-cryo-et-object-identification/discussion/544895#3040071\ntomo_type = \"denoised\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:38:28.459774Z","iopub.execute_input":"2024-11-29T17:38:28.460909Z","iopub.status.idle":"2024-11-29T17:38:28.470657Z","shell.execute_reply.started":"2024-11-29T17:38:28.460827Z","shell.execute_reply":"2024-11-29T17:38:28.469394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"generate_masks = True\n\nif generate_masks:\n    target_objects = defaultdict(dict)\n    for object in root.pickable_objects:\n        if object.is_particle:\n            target_objects[object.name]['label'] = object.label\n            target_objects[object.name]['radius'] = object.radius\n\n\n    for run in tqdm(root.runs):\n        tomo = run.get_voxel_spacing(10)\n        tomo = tomo.get_tomogram(tomo_type).numpy()\n        target = np.zeros(tomo.shape, dtype=np.uint8)\n        for pickable_object in root.pickable_objects:\n            pick = run.get_picks(object_name=pickable_object.name, user_id=\"curation\")\n            if len(pick):  \n                target = segmentation_from_picks.from_picks(pick[0], \n                                                            target, \n                                                            target_objects[pickable_object.name]['radius'] * 0.8,\n                                                            target_objects[pickable_object.name]['label']\n                                                            )\n        write.segmentation(run, target, copick_user_name, name=copick_segmentation_name)\n\n        print(copick_segmentation_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:38:30.179118Z","iopub.execute_input":"2024-11-29T17:38:30.179542Z","iopub.status.idle":"2024-11-29T17:38:38.712913Z","shell.execute_reply.started":"2024-11-29T17:38:30.179508Z","shell.execute_reply":"2024-11-29T17:38:38.711735Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dicts = []\nfor run in tqdm(root.runs):\n    tomogram = run.get_voxel_spacing(voxel_size).get_tomogram(tomo_type).numpy()\n    segmentation = run.get_segmentations(name=copick_segmentation_name, user_id=copick_user_name, voxel_size=voxel_size, is_multilabel=True)[0].numpy()\n    data_dicts.append({\"image\": tomogram, \"label\": segmentation})\n    \nprint(np.unique(data_dicts[0]['label']))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:38:40.629219Z","iopub.execute_input":"2024-11-29T17:38:40.629712Z","iopub.status.idle":"2024-11-29T17:38:49.760595Z","shell.execute_reply.started":"2024-11-29T17:38:40.629673Z","shell.execute_reply":"2024-11-29T17:38:49.759448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dicts[0][\"image\"].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:38:55.814158Z","iopub.execute_input":"2024-11-29T17:38:55.814555Z","iopub.status.idle":"2024-11-29T17:38:55.823043Z","shell.execute_reply.started":"2024-11-29T17:38:55.81452Z","shell.execute_reply":"2024-11-29T17:38:55.82185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dicts[0][\"label\"].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:38:55.825302Z","iopub.execute_input":"2024-11-29T17:38:55.825848Z","iopub.status.idle":"2024-11-29T17:38:55.842161Z","shell.execute_reply.started":"2024-11-29T17:38:55.825813Z","shell.execute_reply":"2024-11-29T17:38:55.840985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"type(data_dicts[0][\"label\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:38:55.843592Z","iopub.execute_input":"2024-11-29T17:38:55.843987Z","iopub.status.idle":"2024-11-29T17:38:55.855033Z","shell.execute_reply.started":"2024-11-29T17:38:55.843955Z","shell.execute_reply":"2024-11-29T17:38:55.853994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"save_path = \"/kaggle/working\" \nsave_path","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:38:55.856478Z","iopub.execute_input":"2024-11-29T17:38:55.856915Z","iopub.status.idle":"2024-11-29T17:38:55.870869Z","shell.execute_reply.started":"2024-11-29T17:38:55.856843Z","shell.execute_reply":"2024-11-29T17:38:55.869642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!mkdir /kaggle/working/images\n!mkdir /kaggle/working/labels","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:38:55.873419Z","iopub.execute_input":"2024-11-29T17:38:55.873832Z","iopub.status.idle":"2024-11-29T17:38:58.330953Z","shell.execute_reply.started":"2024-11-29T17:38:55.873798Z","shell.execute_reply":"2024-11-29T17:38:58.329092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in tqdm(range(7)):\n    image_name = save_path + f\"/images/image_{i}.npz\"        \n    image = data_dicts[i]['image']\n    np.savez_compressed(image_name, image)\n\n    label_name = save_path + f\"/labels/label_{i}.npz\"\n    label = data_dicts[i]['label']\n    np.savez_compressed(label_name, label)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:38:58.333204Z","iopub.execute_input":"2024-11-29T17:38:58.333742Z","iopub.status.idle":"2024-11-29T17:41:09.538653Z","shell.execute_reply.started":"2024-11-29T17:38:58.333686Z","shell.execute_reply":"2024-11-29T17:41:09.537183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del data_dicts\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:41:09.540444Z","iopub.execute_input":"2024-11-29T17:41:09.540822Z","iopub.status.idle":"2024-11-29T17:41:09.557623Z","shell.execute_reply.started":"2024-11-29T17:41:09.540788Z","shell.execute_reply":"2024-11-29T17:41:09.556284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"images = []\nlabels = []\nfor i in tqdm(range(7)):\n    image_name = save_path + f\"/images/image_{i}.npz\"\n    image_npz = np.load(image_name)\n    images.append(image_npz[image_npz.files[0]])\n\n    label_name = save_path + f\"/labels/label_{i}.npz\"\n    label_npz =np.load(label_name)\n    labels.append(label_npz[label_npz.files[0]])\n\n    del image_npz, label_npz\n    gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:41:10.447408Z","iopub.execute_input":"2024-11-29T17:41:10.447764Z","iopub.status.idle":"2024-11-29T17:41:30.055189Z","shell.execute_reply.started":"2024-11-29T17:41:10.44773Z","shell.execute_reply":"2024-11-29T17:41:30.053927Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# HERE IS MY FUNCTION","metadata":{}},{"cell_type":"code","source":"def get_item(exp_num, dim_num, slice_num):\n    # get pair of image slice and corresponding tarhet slice\n    # exp_num - number of experiment, total 7\n    # dim_num - number of dimension, total 3\n    # slice_num - number of slice\n\n    if (exp_num < 0) or (exp_num > 6):\n        raise ValueError(\"Number of experiment is invalid\")\n\n    if (dim_num < 1) or (dim_num > 3):\n        raise ValueError(\"Number of dimension is invalid\")\n\n    if dim_num == 1:\n        if slice_num >= 184:\n            raise ValueError(\"Number of slice is invalid\")\n       \n    if dim_num > 1:\n        if slice_num >= 630:\n            raise ValueError(\"Number of slice is invalid\")\n    \n    if dim_num == 1:\n        slice = images[exp_num][slice_num,...]\n        target = labels[exp_num][slice_num,...]\n    if dim_num == 2:\n        slice = images[exp_num][:,slice_num,:]\n        target = labels[exp_num][:,slice_num,:]\n    if dim_num == 3:\n        slice = images[exp_num][:,:,slice_num]\n        target = labels[exp_num][:,:,slice_num]\n    \n    return slice, target","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:41:30.059393Z","iopub.execute_input":"2024-11-29T17:41:30.059926Z","iopub.status.idle":"2024-11-29T17:41:30.068557Z","shell.execute_reply.started":"2024-11-29T17:41:30.059862Z","shell.execute_reply":"2024-11-29T17:41:30.067372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exp_num = 1\ndim_num = 1\nslice_num = 1\n\ns, t = get_item(exp_num, dim_num, slice_num)\n\ns.shape, t.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:41:30.070037Z","iopub.execute_input":"2024-11-29T17:41:30.070486Z","iopub.status.idle":"2024-11-29T17:41:30.091188Z","shell.execute_reply.started":"2024-11-29T17:41:30.070437Z","shell.execute_reply":"2024-11-29T17:41:30.089854Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exp_num = 2\ndim_num = 2\nslice_num = 2\n\ns, t = get_item(exp_num, dim_num, slice_num)\n\ns.shape, t.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:41:30.092523Z","iopub.execute_input":"2024-11-29T17:41:30.092981Z","iopub.status.idle":"2024-11-29T17:41:30.107024Z","shell.execute_reply.started":"2024-11-29T17:41:30.09294Z","shell.execute_reply":"2024-11-29T17:41:30.10587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exp_num = 3\ndim_num = 3\nslice_num = 3\n\ns, t = get_item(exp_num, dim_num, slice_num)\n\ns.shape, t.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:41:30.108335Z","iopub.execute_input":"2024-11-29T17:41:30.108686Z","iopub.status.idle":"2024-11-29T17:41:30.123735Z","shell.execute_reply.started":"2024-11-29T17:41:30.108654Z","shell.execute_reply":"2024-11-29T17:41:30.122438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exp_num = 4\ndim_num = 4\nslice_num = 4\n\ns, t = get_item(exp_num, dim_num, slice_num)\n\ns.shape, t.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:41:30.125529Z","iopub.execute_input":"2024-11-29T17:41:30.125847Z","iopub.status.idle":"2024-11-29T17:41:30.233821Z","shell.execute_reply.started":"2024-11-29T17:41:30.125816Z","shell.execute_reply":"2024-11-29T17:41:30.232226Z"}},"outputs":[],"execution_count":null}]}