{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"CREATE DATASET","metadata":{}},{"cell_type":"code","source":"!pip install zarr","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-13T02:52:26.629261Z","iopub.execute_input":"2025-01-13T02:52:26.629580Z","iopub.status.idle":"2025-01-13T02:52:34.516082Z","shell.execute_reply.started":"2025-01-13T02:52:26.629555Z","shell.execute_reply":"2025-01-13T02:52:34.515219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import json\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport zarr\nfrom tqdm import tqdm\nimport glob, os\nimport cv2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T02:52:34.517189Z","iopub.execute_input":"2025-01-13T02:52:34.517418Z","iopub.status.idle":"2025-01-13T02:52:35.266829Z","shell.execute_reply.started":"2025-01-13T02:52:34.517401Z","shell.execute_reply":"2025-01-13T02:52:35.266119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"runs = sorted(glob.glob('/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/*'))\nruns = [os.path.basename(x) for x in runs]\ni2r_dict = {i:r for i, r in zip(range(len(runs)), runs)}\nr2t_dict = {r:i for i, r in zip(range(len(runs)), runs)}\ni2r_dict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T02:52:35.268425Z","iopub.execute_input":"2025-01-13T02:52:35.268955Z","iopub.status.idle":"2025-01-13T02:52:35.281146Z","shell.execute_reply.started":"2025-01-13T02:52:35.268932Z","shell.execute_reply":"2025-01-13T02:52:35.280514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def convert_to_8bit(x):\n    lower, upper = np.percentile(x, (0.5, 99.5))\n    x = np.clip(x, lower, upper)\n    x = (x - x.min()) / (x.max() - x.min() + 1e-12) * 255\n    return x.round().astype(\"uint8\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T02:52:35.282391Z","iopub.execute_input":"2025-01-13T02:52:35.282682Z","iopub.status.idle":"2025-01-13T02:52:35.287839Z","shell.execute_reply.started":"2025-01-13T02:52:35.282653Z","shell.execute_reply":"2025-01-13T02:52:35.286944Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#子（パーティクル）の名前と対応するインデックスを定義する辞書です。\n#例: 'apo-ferritin' は 0、'ribosome' は 3 に対応。\np2i_dict = {\n        'apo-ferritin': 0,\n        'beta-amylase': 1,\n        'beta-galactosidase': 2,\n        'ribosome': 3,\n        'thyroglobulin': 4,\n        'virus-like-particle': 5\n    }\n#p2i_dict を逆転させた辞書で、インデックスから粒子名を取得できます。\n#例: 3 から 'ribosome' を取得。\ni2p = {v:k for k, v in p2i_dict.items()}\n\n#各粒子の半径（ナノメートル単位など）を定義する辞書です。\n#例: 'apo-ferritin' の半径は 60、'ribosome' の半径は 150。\nparticle_radius = {\n        'apo-ferritin': 60,\n        'beta-amylase': 65,\n        'beta-galactosidase': 90,\n        'ribosome': 150,\n        'thyroglobulin': 130,\n        'virus-like-particle': 135,\n    }\n#粒子の名前をリスト形式で保持しています。 解析や表示、ループ処理などで役立ちます。\nparticle_names = ['apo-ferritin', 'beta-amylase', 'beta-galactosidase', 'ribosome', 'thyroglobulin', 'virus-like-particle']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T02:52:35.288686Z","iopub.execute_input":"2025-01-13T02:52:35.288946Z","iopub.status.idle":"2025-01-13T02:52:35.301463Z","shell.execute_reply.started":"2025-01-13T02:52:35.288926Z","shell.execute_reply":"2025-01-13T02:52:35.300738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_annotate_yolo(run_name, is_train_path=True):\n    is_train_path = 'train' if is_train_path else 'val'\n    \n    # Zarrファイルリストを定義\n    zarr_files = [\n        'denoised.zarr',\n        'ctfdeconvolved.zarr',\n        'isonetcorrected.zarr',\n        'wbp.zarr'\n    ]\n    \n    for zarr_file in zarr_files:\n        vol = zarr.open(f'/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns/{run_name}/VoxelSpacing10.000/{zarr_file}', mode='r')\n        vol = vol[0]\n        vol2 = convert_to_8bit(vol)\n        n_imgs = vol2.shape[0]\n        \n        for j in range(n_imgs):\n            newvol = vol2[j]\n            newvolf = np.stack([newvol] * 3, axis=-1)\n            newvolf = cv2.resize(newvolf, (640, 640))\n            \n            # ファイル名にzarrの種類を加える\n            zarr_type = zarr_file.split('.')[0]\n            cv2.imwrite(f'images/{is_train_path}/{run_name}_{zarr_type}_{j * 10}.png', newvolf)\n            \n            # ラベルファイル作成\n            with open(f'labels/{is_train_path}/{run_name}_{zarr_type}_{j * 10}.txt', 'w'):\n                pass\n\n        # 粒子ごとのアノテーション（既存処理）\n        for p, particle in enumerate(tqdm(particle_names)):\n            if particle == \"beta-amylase\":\n                continue\n            json_each_particle = f\"/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/{run_name}/Picks/{particle}.json\"\n            df = pd.read_json(json_each_particle)\n            for axis in \"x\", \"y\", \"z\":\n                df[axis] = df.points.apply(lambda x: x[\"location\"][axis])\n            \n            radius = particle_radius[particle]\n            for i, row in df.iterrows():\n                start_z = np.round(row['z'] - radius).astype(np.int32)\n                start_z = max(0, start_z // 10)\n                end_z = np.round(row['z'] + radius).astype(np.int32)\n                end_z = min(n_imgs, end_z // 10)\n                \n                for j in range(start_z + 1, end_z + 1 - 1, 1):\n                    with open(f'labels/{is_train_path}/{run_name}_{zarr_type}_{j * 10}.txt', 'a') as f:\n                        f.write(f'{p2i_dict[particle]} {row[\"x\"] / 10 / vol2.shape[1]} {row[\"y\"] / 10 / vol2.shape[2]} {radius / 10 / vol2.shape[1] * 2} {radius / 10 / vol2.shape[2] * 2} \\n')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T02:52:35.302178Z","iopub.execute_input":"2025-01-13T02:52:35.302476Z","iopub.status.idle":"2025-01-13T02:52:35.317080Z","shell.execute_reply.started":"2025-01-13T02:52:35.302456Z","shell.execute_reply":"2025-01-13T02:52:35.316124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.makedirs(\"images/train\", exist_ok=True)\nos.makedirs(\"images/val\", exist_ok=True)\nos.makedirs(\"labels/val\", exist_ok=True)\nos.makedirs(\"labels/train\", exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T02:52:35.317943Z","iopub.execute_input":"2025-01-13T02:52:35.318245Z","iopub.status.idle":"2025-01-13T02:52:35.336146Z","shell.execute_reply.started":"2025-01-13T02:52:35.318216Z","shell.execute_reply":"2025-01-13T02:52:35.335223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# use TS_5_4 as validation\nfor i, r in enumerate(runs):\n    make_annotate_yolo(r, is_train_path=False if i==0 else True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T02:52:35.338412Z","iopub.execute_input":"2025-01-13T02:52:35.338732Z","iopub.status.idle":"2025-01-13T02:57:39.882694Z","shell.execute_reply.started":"2025-01-13T02:52:35.338687Z","shell.execute_reply":"2025-01-13T02:57:39.881803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\nos.makedirs('datasets/czii_det2d', exist_ok=True)\nshutil.move('images/train', 'datasets/czii_det2d/images/train')\nshutil.move('images/val', 'datasets/czii_det2d/images')\nshutil.move('labels/train', 'datasets/czii_det2d/labels/train')\nshutil.move('labels/val', 'datasets/czii_det2d/labels')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T02:57:39.883766Z","iopub.execute_input":"2025-01-13T02:57:39.884068Z","iopub.status.idle":"2025-01-13T02:57:46.713354Z","shell.execute_reply.started":"2025-01-13T02:57:39.884039Z","shell.execute_reply":"2025-01-13T02:57:46.712277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile czii_conf.yaml\n\npath: /kaggle/input/czii-datasets-yota02/datasets/czii_det2d\ntrain: images/train # train images (relative to 'path') \nval: images/val # val images (relative to 'path') \n\n# Classes\nnames:\n  0: apo-ferritin\n  1: beta-amylase\n  2: beta-galactosidase\n  3: ribosome\n  4: thyroglobulin\n  5: virus-like-particle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T02:57:46.714159Z","iopub.execute_input":"2025-01-13T02:57:46.714619Z","iopub.status.idle":"2025-01-13T02:57:46.720374Z","shell.execute_reply.started":"2025-01-13T02:57:46.714596Z","shell.execute_reply":"2025-01-13T02:57:46.718806Z"}},"outputs":[],"execution_count":null}]}