{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":104.820086,"end_time":"2024-12-05T09:07:06.020274","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-12-05T09:05:21.200188","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install zarr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T03:03:59.479757Z","iopub.execute_input":"2024-12-22T03:03:59.480081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import json\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport zarr\nfrom tqdm import tqdm\nimport glob, os\nimport cv2","metadata":{"papermill":{"duration":1.485698,"end_time":"2024-12-05T09:05:41.501599","exception":false,"start_time":"2024-12-05T09:05:40.015901","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T03:05:01.165301Z","iopub.execute_input":"2024-12-22T03:05:01.165657Z","iopub.status.idle":"2024-12-22T03:05:01.181160Z","shell.execute_reply.started":"2024-12-22T03:05:01.165626Z","shell.execute_reply":"2024-12-22T03:05:01.179872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"runs = sorted(glob.glob('/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/*'))\nruns = [os.path.basename(x) for x in runs]\ni2r_dict = {i:r for i, r in zip(range(len(runs)), runs)}\nr2t_dict = {r:i for i, r in zip(range(len(runs)), runs)}\ni2r_dict","metadata":{"papermill":{"duration":0.029085,"end_time":"2024-12-05T09:05:41.536029","exception":false,"start_time":"2024-12-05T09:05:41.506944","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T03:05:03.042291Z","iopub.execute_input":"2024-12-22T03:05:03.042691Z","iopub.status.idle":"2024-12-22T03:05:03.057589Z","shell.execute_reply.started":"2024-12-22T03:05:03.042656Z","shell.execute_reply":"2024-12-22T03:05:03.056211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def convert_to_8bit(x):\n    lower, upper = np.percentile(x, (0.5, 99.5))\n    x = np.clip(x, lower, upper)\n    x = (x - x.min()) / (x.max() - x.min() + 1e-12) * 255\n    return x.round().astype(\"uint8\")","metadata":{"papermill":{"duration":0.015646,"end_time":"2024-12-05T09:05:41.635797","exception":false,"start_time":"2024-12-05T09:05:41.620151","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T03:05:05.964539Z","iopub.execute_input":"2024-12-22T03:05:05.964944Z","iopub.status.idle":"2024-12-22T03:05:05.970592Z","shell.execute_reply.started":"2024-12-22T03:05:05.964906Z","shell.execute_reply":"2024-12-22T03:05:05.969438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"p2i_dict = {\n        'apo-ferritin': 0,\n        'beta-amylase': 1,\n        'beta-galactosidase': 2,\n        'ribosome': 3,\n        'thyroglobulin': 4,\n        'virus-like-particle': 5\n    }\n\ni2p = {v:k for k, v in p2i_dict.items()}\n\nparticle_radius = {\n        'apo-ferritin': 60,\n        'beta-amylase': 65,\n        'beta-galactosidase': 90,\n        'ribosome': 150,\n        'thyroglobulin': 130,\n        'virus-like-particle': 135,\n    }","metadata":{"papermill":{"duration":0.014787,"end_time":"2024-12-05T09:05:41.674147","exception":false,"start_time":"2024-12-05T09:05:41.65936","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T03:05:08.281762Z","iopub.execute_input":"2024-12-22T03:05:08.282138Z","iopub.status.idle":"2024-12-22T03:05:08.288532Z","shell.execute_reply.started":"2024-12-22T03:05:08.282105Z","shell.execute_reply":"2024-12-22T03:05:08.287252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"particle_names = ['apo-ferritin', 'beta-amylase', 'beta-galactosidase', 'ribosome', 'thyroglobulin', 'virus-like-particle']","metadata":{"papermill":{"duration":0.01351,"end_time":"2024-12-05T09:05:41.692783","exception":false,"start_time":"2024-12-05T09:05:41.679273","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T03:05:10.722823Z","iopub.execute_input":"2024-12-22T03:05:10.723499Z","iopub.status.idle":"2024-12-22T03:05:10.728344Z","shell.execute_reply.started":"2024-12-22T03:05:10.723456Z","shell.execute_reply":"2024-12-22T03:05:10.727203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_annotate_yolo(run_name, is_train_path=True,fold=-1):\n    # to split validation\n    is_train_path = 'train' if is_train_path else 'val'\n    vol = zarr.open(f'/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns/{r}/VoxelSpacing10.000/denoised.zarr', mode='r')\n    # use largest images\n    vol = vol[0]\n    # normalize [0, 255]\n    vol2 = convert_to_8bit(vol)\n    \n    n_imgs = vol2.shape[0]\n    # process each slices\n    for j in range(n_imgs):\n        newvol = vol2[j]\n        newvolf = np.stack([newvol]*3, axis=-1)\n        newvolf = cv2.resize(newvolf, (640,640))\n        cv2.imwrite(f'{fold}/images/{is_train_path}/{run_name}_{j*10}.png', newvolf)\n        with open(f'{fold}/labels/{is_train_path}/{run_name}_{j*10}.txt', 'w'):\n            pass # make empty file\n    for p, particle in enumerate(tqdm(particle_names)):\n        if particle==\"beta-amylase\":\n            continue\n        json_each_paticle = f\"/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/{run_name}/Picks/{particle}.json\"\n        df = pd.read_json(json_each_paticle) \n        # pick each coordinate of particles\n        for axis in \"x\", \"y\", \"z\":\n            df[axis] = df.points.apply(lambda x: x[\"location\"][axis])\n        radius = particle_radius[particle]\n        for i, row in df.iterrows():\n            start_z = np.round(row['z'] - radius).astype(np.int32)\n            start_z = max(0, start_z//10) # 10 means pixelspacing\n            end_z = np.round(row['z'] + radius).astype(np.int32)\n            end_z = min(n_imgs, end_z//10) # 10 means pixelspacing\n            \n            for j in range(start_z+1, end_z+1-1, 1):\n                # white the results of annotation\n                with open(f'{fold}/labels/{is_train_path}/{run_name}_{j*10}.txt', 'a') as f:\n                    f.write(f'{p2i_dict[particle]} {row[\"x\"]/10/vol2.shape[1]} {row[\"y\"]/10/vol2.shape[2]} {radius/10/vol2.shape[1]*2} {radius/10/vol2.shape[2]*2} \\n')\n    ","metadata":{"papermill":{"duration":0.019182,"end_time":"2024-12-05T09:05:41.717008","exception":false,"start_time":"2024-12-05T09:05:41.697826","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T03:05:45.124577Z","iopub.execute_input":"2024-12-22T03:05:45.124956Z","iopub.status.idle":"2024-12-22T03:05:45.136575Z","shell.execute_reply.started":"2024-12-22T03:05:45.124924Z","shell.execute_reply":"2024-12-22T03:05:45.135164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\nfor j in range(len(runs)):\n    os.makedirs(f\"{j}/images/train\", exist_ok=True)\n    os.makedirs(f\"{j}/images/val\", exist_ok=True)\n    os.makedirs(f\"{j}/labels/val\", exist_ok=True)\n    os.makedirs(f\"{j}/labels/train\", exist_ok=True)\n    for i, r in enumerate(runs):\n        make_annotate_yolo(r, is_train_path=False if i==j else True,fold=j)\n    os.makedirs('datasets/czii_det2d', exist_ok=True)\n    shutil.move(f'{j}/images/train', f'datasets/czii_det2d/{j}/images/train')\n    shutil.move(f'{j}/images/val', f'datasets/czii_det2d/{j}/images/val')\n    shutil.move(f'{j}/labels/train', f'datasets/czii_det2d/{j}/labels/train')\n    shutil.move(f'{j}/labels/val', f'datasets/czii_det2d/{j}/labels/val')\n    break","metadata":{"papermill":{"duration":81.881721,"end_time":"2024-12-05T09:07:03.62435","exception":false,"start_time":"2024-12-05T09:05:41.742629","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T03:05:50.883188Z","iopub.execute_input":"2024-12-22T03:05:50.884029Z"}},"outputs":[],"execution_count":null}]}