{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"},{"sourceId":9867543,"sourceType":"datasetVersion","datasetId":6040935}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"!cp -r '/kaggle/input/hengck-czii-cryo-et-01/wheel_file' '/kaggle/working/'\n!pip install /kaggle/working/wheel_file/asciitree-0.3.3/asciitree-0.3.3\n!pip install --no-index --find-links=/kaggle/working/wheel_file zarr\n!pip install --no-index --find-links=/kaggle/working/wheel_file connected-components-3d","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:41:24.966663Z","iopub.execute_input":"2024-11-30T04:41:24.967163Z","iopub.status.idle":"2024-11-30T04:42:34.572161Z","shell.execute_reply.started":"2024-11-30T04:41:24.967113Z","shell.execute_reply":"2024-11-30T04:42:34.570513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nimport os\nimport sys\nimport pickle\nimport random as rnd\nimport numpy as np\nfrom numpy import random as np_rnd\nimport pandas as pd\nfrom tqdm import tqdm\nimport zarr\nimport glob\nimport json\n\nimport torch\nfrom torchvision import transforms\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:34.574945Z","iopub.execute_input":"2024-11-30T04:42:34.575344Z","iopub.status.idle":"2024-11-30T04:42:39.820256Z","shell.execute_reply.started":"2024-11-30T04:42:34.575306Z","shell.execute_reply":"2024-11-30T04:42:39.818801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    debug = True\n    max_coord_values = {\"x\": 6400, \"y\": 6400, \"z\": 1840}\n    # 0: high, 1: medium, 2: low\n    resolution_multiplier = {0: 10, 1: 20, 2: 40}\n    resolution = 0\n    channel_sampling_interval = 20\n    img_size = (224, 224)\n    label2id = {\n        \"no-object\": 0,\n        \"apo-ferritin\": 1,\n        \"beta-galactosidase\": 2,\n        \"ribosome\": 3,\n        \"thyroglobulin\": 4,\n        \"virus-like-particle\": 5,\n    }\n    id2label = {\n        0: \"no-object\",\n        1: \"apo-ferritin\",\n        2: \"beta-galactosidase\",\n        3: \"ribosome\",\n        4: \"thyroglobulin\",\n        5: \"virus-like-particle\",\n    }\n    particle_radius = {\n        'apo-ferritin': 60,\n        'beta-amylase': 65,\n        'beta-galactosidase': 90,\n        'ribosome': 150,\n        'thyroglobulin': 130,\n        'virus-like-particle': 135,\n    }\n    weights = {\n        'apo-ferritin': 1,\n        'beta-amylase': 0,\n        'beta-galactosidase': 2,\n        'ribosome': 1,\n        'thyroglobulin': 2,\n        'virus-like-particle': 1,\n    }\n    n_folds = 7","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:39.821771Z","iopub.execute_input":"2024-11-30T04:42:39.822613Z","iopub.status.idle":"2024-11-30T04:42:39.829411Z","shell.execute_reply.started":"2024-11-30T04:42:39.822537Z","shell.execute_reply":"2024-11-30T04:42:39.828070Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed=42):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    # python random\n    rnd.seed(seed)\n    # numpy random\n    np_rnd.seed(seed)\n    # RAPIDS random\n    try:\n        cupy.random.seed(seed)\n    except:\n        pass\n    # tf random\n    try:\n        tf_rnd.set_seed(seed)\n    except:\n        pass\n    # pytorch random\n    try:\n        torch.backends.cudnn.benchmark = False\n        torch.backends.cudnn.deterministic = True\n        torch.manual_seed(seed)\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n    except:\n        pass\n\ndef pickleIO(obj, src, op=\"r\"):\n    if op==\"w\":\n        with open(src, op + \"b\") as f:\n            pickle.dump(obj, f)\n    elif op==\"r\":\n        with open(src, op + \"b\") as f:\n            tmp = pickle.load(f)\n        return tmp\n    else:\n        print(\"unknown operation\")\n        return obj\n    \ndef createFolder(directory):\n    try:\n        if not os.path.exists(directory):\n            os.makedirs(directory)\n    except OSError:\n        print('Error: Creating directory. ' + directory)\n\ndef findIdx(data_x, col_names):\n    return [int(i) for i, j in enumerate(data_x) if j in col_names]\n\ndef diff(first, second):\n    second = set(second)\n    return [item for item in first if item not in second]\n\ndef minmax_scaler(data, feature_range=(0, 255)):\n    min_val, max_val = feature_range\n    scaled_data = (data - np.min(data)) / (np.max(data) - np.min(data))\n    scaled_data = scaled_data * (max_val - min_val) + min_val\n    return scaled_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:39.832597Z","iopub.execute_input":"2024-11-30T04:42:39.833077Z","iopub.status.idle":"2024-11-30T04:42:39.856812Z","shell.execute_reply.started":"2024-11-30T04:42:39.833025Z","shell.execute_reply":"2024-11-30T04:42:39.855569Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Loading rawdata","metadata":{}},{"cell_type":"markdown","source":"### for annotation data","metadata":{}},{"cell_type":"code","source":"df_annot = []\nfor run in glob.glob(\"/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/*\"):\n    for particle in glob.glob(f\"{run}/Picks/*.json\"):\n        with open(particle, \"r\") as f:\n            annot = json.load(f)\n        for point in annot[\"points\"]:\n            df_annot.append({\n                \"run\": annot[\"run_name\"],\n                \"img_path\": f\"/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns/{annot['run_name']}/VoxelSpacing10.000/denoised.zarr\",\n                \"obj\": annot[\"pickable_object_name\"],\n                **point[\"location\"]\n            })\ndf_annot = pd.DataFrame(df_annot).set_index(\"run\")\ndf_annot[[\"x\", \"y\", \"z\"]] = df_annot[[\"x\", \"y\", \"z\"]].astype(\"float32\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:39.858171Z","iopub.execute_input":"2024-11-30T04:42:39.858543Z","iopub.status.idle":"2024-11-30T04:42:40.096655Z","shell.execute_reply.started":"2024-11-30T04:42:39.858501Z","shell.execute_reply":"2024-11-30T04:42:40.095368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_annot.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:40.098038Z","iopub.execute_input":"2024-11-30T04:42:40.098335Z","iopub.status.idle":"2024-11-30T04:42:40.123233Z","shell.execute_reply.started":"2024-11-30T04:42:40.098306Z","shell.execute_reply":"2024-11-30T04:42:40.122170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_annot[\"x\"].describe(), df_annot[\"y\"].describe(), df_annot[\"z\"].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:40.124906Z","iopub.execute_input":"2024-11-30T04:42:40.125358Z","iopub.status.idle":"2024-11-30T04:42:40.149652Z","shell.execute_reply.started":"2024-11-30T04:42:40.125310Z","shell.execute_reply":"2024-11-30T04:42:40.148548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_annot","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:40.150939Z","iopub.execute_input":"2024-11-30T04:42:40.151218Z","iopub.status.idle":"2024-11-30T04:42:40.170595Z","shell.execute_reply.started":"2024-11-30T04:42:40.151191Z","shell.execute_reply":"2024-11-30T04:42:40.169422Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### for image data","metadata":{}},{"cell_type":"code","source":"df_img = {k: np.array(zarr.open(v)[CFG.resolution], dtype=\"float32\") for k, v in df_annot[\"img_path\"].drop_duplicates().items()}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:40.171978Z","iopub.execute_input":"2024-11-30T04:42:40.172410Z","iopub.status.idle":"2024-11-30T04:42:58.773229Z","shell.execute_reply.started":"2024-11-30T04:42:40.172361Z","shell.execute_reply":"2024-11-30T04:42:58.771970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for run, img in df_img.items():\n    print(f\"RUN: {run}, SHAPE: {img.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:58.776385Z","iopub.execute_input":"2024-11-30T04:42:58.776795Z","iopub.status.idle":"2024-11-30T04:42:58.782736Z","shell.execute_reply.started":"2024-11-30T04:42:58.776755Z","shell.execute_reply":"2024-11-30T04:42:58.781523Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# channel sampling, resizing, rescaling\nresizer = transforms.Resize(CFG.img_size)\ndf_img = {run: minmax_scaler(resizer(torch.tensor(img[::CFG.channel_sampling_interval])).detach().cpu().numpy(), (0, 1)) for run, img in df_img.items()}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:58.784129Z","iopub.execute_input":"2024-11-30T04:42:58.784433Z","iopub.status.idle":"2024-11-30T04:42:59.182115Z","shell.execute_reply.started":"2024-11-30T04:42:58.784402Z","shell.execute_reply":"2024-11-30T04:42:59.180717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for run, img in df_img.items():\n    print(f\"RUN: {run}, SHAPE: {img.shape}, RANGE: {img.max(), img.min()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:59.184186Z","iopub.execute_input":"2024-11-30T04:42:59.184717Z","iopub.status.idle":"2024-11-30T04:42:59.208640Z","shell.execute_reply.started":"2024-11-30T04:42:59.184662Z","shell.execute_reply":"2024-11-30T04:42:59.207593Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# mapping image size & update coordinates\ndf_annot[[\"c\", \"w\", \"h\"]] = pd.DataFrame([dict(zip([\"c\", \"w\", \"h\"], df_img[run].shape)) for run in df_annot.index], dtype=\"int32\", index=df_annot.index)\ndf_annot[\"x\"] = (df_annot[\"x\"] / CFG.max_coord_values[\"x\"]) * df_annot[\"w\"]\ndf_annot[\"y\"] = (df_annot[\"y\"] / CFG.max_coord_values[\"y\"]) * df_annot[\"h\"]\ndf_annot[\"z\"] = (df_annot[\"z\"] / CFG.max_coord_values[\"z\"]) * df_annot[\"c\"]\ndf_annot[[\"x\", \"y\", \"z\"]] = df_annot[[\"x\", \"y\", \"z\"]].astype(\"float32\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:44:22.875212Z","iopub.execute_input":"2024-11-30T04:44:22.875886Z","iopub.status.idle":"2024-11-30T04:44:22.895768Z","shell.execute_reply.started":"2024-11-30T04:44:22.875835Z","shell.execute_reply":"2024-11-30T04:44:22.894397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_annot.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:44:23.862825Z","iopub.execute_input":"2024-11-30T04:44:23.863347Z","iopub.status.idle":"2024-11-30T04:44:23.879188Z","shell.execute_reply.started":"2024-11-30T04:44:23.863305Z","shell.execute_reply":"2024-11-30T04:44:23.877922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_annot[\"x\"].describe(), df_annot[\"y\"].describe(), df_annot[\"z\"].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:59.248998Z","iopub.execute_input":"2024-11-30T04:42:59.249293Z","iopub.status.idle":"2024-11-30T04:42:59.273343Z","shell.execute_reply.started":"2024-11-30T04:42:59.249265Z","shell.execute_reply":"2024-11-30T04:42:59.272015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_annot","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:59.274645Z","iopub.execute_input":"2024-11-30T04:42:59.275099Z","iopub.status.idle":"2024-11-30T04:42:59.292786Z","shell.execute_reply.started":"2024-11-30T04:42:59.275051Z","shell.execute_reply":"2024-11-30T04:42:59.291582Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Validation Split","metadata":{}},{"cell_type":"code","source":"df_annot[\"fold\"] = -1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:59.294248Z","iopub.execute_input":"2024-11-30T04:42:59.294732Z","iopub.status.idle":"2024-11-30T04:42:59.306810Z","shell.execute_reply.started":"2024-11-30T04:42:59.294684Z","shell.execute_reply":"2024-11-30T04:42:59.305632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for fold, run in enumerate(df_annot.index.unique()):\n    df_annot.loc[run, \"fold\"] = fold","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:59.308322Z","iopub.execute_input":"2024-11-30T04:42:59.308821Z","iopub.status.idle":"2024-11-30T04:42:59.324208Z","shell.execute_reply.started":"2024-11-30T04:42:59.308769Z","shell.execute_reply":"2024-11-30T04:42:59.323066Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_annot","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:59.325668Z","iopub.execute_input":"2024-11-30T04:42:59.326133Z","iopub.status.idle":"2024-11-30T04:42:59.350826Z","shell.execute_reply.started":"2024-11-30T04:42:59.326085Z","shell.execute_reply":"2024-11-30T04:42:59.349542Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Save data","metadata":{}},{"cell_type":"code","source":"pickleIO(df_annot, \"df_annot.pkl\", \"w\")\nnp.savez(\"df_img\", **df_img)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T04:42:59.352224Z","iopub.execute_input":"2024-11-30T04:42:59.352616Z","iopub.status.idle":"2024-11-30T04:42:59.505708Z","shell.execute_reply.started":"2024-11-30T04:42:59.352538Z","shell.execute_reply":"2024-11-30T04:42:59.504457Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}