{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"},{"sourceId":9867543,"sourceType":"datasetVersion","datasetId":6040935}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport json\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patch\nimport tensorflow as tf\nimport gc\nfrom tqdm import tqdm\n# !pip install ipywidgets==8.1.5","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:13:54.577681Z","iopub.execute_input":"2024-12-26T15:13:54.578341Z","iopub.status.idle":"2024-12-26T15:14:47.183137Z","shell.execute_reply.started":"2024-12-26T15:13:54.578309Z","shell.execute_reply":"2024-12-26T15:14:47.181962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try:\n    import zarr\nexcept: \n    !cp -r '/kaggle/input/hengck-czii-cryo-et-01/wheel_file' '/kaggle/working/'\n    !pip install /kaggle/working/wheel_file/asciitree-0.3.3/asciitree-0.3.3\n    !pip install --no-index --find-links=/kaggle/working/wheel_file zarr\n    !pip install --no-index --find-links=/kaggle/working/wheel_file connected-components-3d\n    import zarr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:14:47.185522Z","iopub.execute_input":"2024-12-26T15:14:47.186690Z","iopub.status.idle":"2024-12-26T15:15:47.610529Z","shell.execute_reply.started":"2024-12-26T15:14:47.186646Z","shell.execute_reply":"2024-12-26T15:15:47.609811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tracemalloc\ntracemalloc.start()\n\ndef print_memory():\n    snapshot = tracemalloc.take_snapshot()\n    total_memory = sum(stat.size for stat in snapshot.statistics('lineno')) \n    total_memory_mb = total_memory / 1024**2  \n    print(f\"{total_memory_mb:.2f} MB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:15:47.612040Z","iopub.execute_input":"2024-12-26T15:15:47.612885Z","iopub.status.idle":"2024-12-26T15:15:47.620545Z","shell.execute_reply.started":"2024-12-26T15:15:47.612854Z","shell.execute_reply":"2024-12-26T15:15:47.619234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def label_gen():\n    ulti_df = pd.DataFrame(columns = ['pickable_object_name', 'run_name', 'instance_id', 'location.x', 'location.y', 'location.z'])\n    for file in os.listdir('/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns'):\n        for sub_file in os.listdir(f'/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/{file}/Picks'):\n            with open (f'/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/{file}/Picks/{sub_file}', 'r') as json_file:\n                temp_json = json.load(json_file)\n            temp_60 = pd.json_normalize(temp_json['points'])\n            temp_60[['pickable_object_name', 'run_name']] = pd.DataFrame(temp_json)[['pickable_object_name', 'run_name']]\n            ulti_df = pd.concat([ulti_df, temp_60], ignore_index = True)\n    return ulti_df[['pickable_object_name', 'run_name', 'location.x', 'location.y', 'location.z']].rename(columns = {'location.x': 'x', 'location.y': 'y', 'location.z': 'z'})\nlabel_df = label_gen()\n\nOBJ_DICT = {'beta-galactosidase': 0, 'apo-ferritin': 1, 'virus-like-particle': 2, 'ribosome': 3, 'thyroglobulin': 4}\n# h, e, e, e, h\nlabel_df=label_df[label_df.pickable_object_name.isin(OBJ_DICT.keys())].reset_index(drop=True)\nPARTICLE_RADIUS = {'ribosome': 150.0, 'virus-like-particle': 135.0, 'apo-ferritin': 60.0, 'beta-galactosidase': 90.0, 'thyroglobulin': 130.0}\nPARTICLE_COLOR = {'ribosome': 'yellow', 'virus-like-particle': 'blue', 'apo-ferritin': 'green', 'beta-galactosidase': 'red', 'thyroglobulin': 'pink'}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:15:47.625059Z","iopub.execute_input":"2024-12-26T15:15:47.625507Z","iopub.status.idle":"2024-12-26T15:15:48.117018Z","shell.execute_reply.started":"2024-12-26T15:15:47.625462Z","shell.execute_reply":"2024-12-26T15:15:48.115742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.crosstab(label_df.run_name, label_df.pickable_object_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:15:48.118747Z","iopub.execute_input":"2024-12-26T15:15:48.119155Z","iopub.status.idle":"2024-12-26T15:15:48.174460Z","shell.execute_reply.started":"2024-12-26T15:15:48.119114Z","shell.execute_reply":"2024-12-26T15:15:48.172803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def image_plotter(image, layer, qual, labels, scale):\n    im = zarr.open(f'/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns/{image}/VoxelSpacing10.000/denoised.zarr')[qual][layer]\n    plt.figure(figsize = (8, 8))\n    plt.imshow(im, cmap = 'gray', vmin = -0.00005, vmax = 0.00005)\n    ax = plt.gca()\n    temp_labels = labels[(labels.z / scale).astype(int) == layer]\n    for idx, row in temp_labels.iterrows():\n        cir = plt.Circle((row.x / scale, row.y / scale), radius = PARTICLE_RADIUS[row.pickable_object_name] / scale, \n                         color = PARTICLE_COLOR[row.pickable_object_name], fill = False)\n        ax.add_patch(cir)\n    plt.show()\nimage_plotter('TS_5_4', 20, 2, label_df[(label_df.run_name == 'TS_5_4')].copy(), 20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:15:48.176831Z","iopub.execute_input":"2024-12-26T15:15:48.177457Z","iopub.status.idle":"2024-12-26T15:15:49.176601Z","shell.execute_reply.started":"2024-12-26T15:15:48.177406Z","shell.execute_reply":"2024-12-26T15:15:49.174949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRIMPATH='/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns'\nNEXTPATH='VoxelSpacing10.000/denoised.zarr'\nIM_FILES=os.listdir(TRIMPATH)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:15:50.197165Z","iopub.execute_input":"2024-12-26T15:15:50.197686Z","iopub.status.idle":"2024-12-26T15:15:50.208448Z","shell.execute_reply.started":"2024-12-26T15:15:50.197619Z","shell.execute_reply":"2024-12-26T15:15:50.207164Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2d_unet","metadata":{}},{"cell_type":"code","source":"def eightbit(x):\n    lower, upper = np.percentile(x, (0.5, 99.5))\n    x = np.clip(x, lower, upper)\n    x = (x - x.min()) / (x.max() - x.min() + 1e-12)\n    return x.astype('float16')\n    \ndef preprocess(im_files,startpath,nextpath,labels=None):\n    return_data=None\n    if labels is not None:\n        labels.loc[:,['x','y','z']]=labels[['x','y','z']] // 10\n        labels.loc[:,'pickable_object_name']=labels.loc[:,'pickable_object_name'].map(OBJ_DICT)\n        for i in ['x', 'y', 'z']:\n            labels[i] = labels[i].astype(int)\n    datasets=[]\n    for file_ in tqdm(im_files):\n        image=zarr.open(os.path.join(os.path.join(startpath,file_),nextpath))[0]\n        image=np.expand_dims(eightbit(image), -1)\n        if labels is not None:\n            curr_df=labels[labels.run_name==file_]\n            temp_tensor = np.zeros((184, 630, 630, 1), dtype='uint8')\n            temp_tensor[curr_df.z,curr_df.y,curr_df.x,0]=curr_df.pickable_object_name+1\n            datasets.append((image, temp_tensor))\n            del temp_tensor, curr_df\n        else:\n            datasets.append((image,))\n        del image\n        gc.collect()\n    gc.collect()\n    if labels is not None:\n        del labels\n        return tf.data.Dataset.from_tensor_slices((np.concatenate([i[0] for i in datasets]),np.concatenate([i[1] for i in datasets])))\n    return tf.data.Dataset.from_tensor_slices((np.concatenate([i[0] for i in datasets]),))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:31:40.885707Z","iopub.execute_input":"2024-12-26T15:31:40.886334Z","iopub.status.idle":"2024-12-26T15:31:40.923705Z","shell.execute_reply.started":"2024-12-26T15:31:40.886264Z","shell.execute_reply":"2024-12-26T15:31:40.921971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def simple_2dunet():\n    input_layer = tf.keras.layers.Input((630, 630, 1))\n    x = tf.keras.layers.Conv2D(16, 3, padding = 'same',activation='relu')(input_layer)\n    x = tf.keras.layers.MaxPooling2D(padding = 'same')(x)\n    x = tf.keras.layers.Conv2D(32, 3, padding = 'same',activation='relu')(x)\n    x = tf.keras.layers.Conv2D(32, 3, padding = 'same',activation='relu')(x)\n    x = tf.keras.layers.UpSampling2D((2, 2))(x)\n    x = tf.keras.layers.Conv2D(1, 3, padding = 'same',activation='sigmoid')(x)\n    return tf.keras.Model(input_layer, x)\ntemp9 = simple_2dunet()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:23:21.350926Z","iopub.execute_input":"2024-12-26T15:23:21.351578Z","iopub.status.idle":"2024-12-26T15:23:21.427383Z","shell.execute_reply.started":"2024-12-26T15:23:21.351512Z","shell.execute_reply":"2024-12-26T15:23:21.425756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temp10=preprocess(IM_FILES,TRIMPATH,NEXTPATH, label_df.copy())\ntemp50=temp10.batch(5)\ndel temp10\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:24:34.817619Z","iopub.execute_input":"2024-12-26T15:24:34.818219Z","iopub.status.idle":"2024-12-26T15:25:03.238576Z","shell.execute_reply.started":"2024-12-26T15:24:34.818161Z","shell.execute_reply":"2024-12-26T15:25:03.236797Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"save=tf.keras.callbacks.ModelCheckpoint('model1.keras')\ntemp9.compile(optimizer='Adam',loss=tf.keras.losses.CategoricalCrossentropy())\ntemp9.fit(temp50,epochs=10,callbacks=[save],batch_size=5)\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:28:23.948164Z","iopub.execute_input":"2024-12-26T15:28:23.948839Z","iopub.status.idle":"2024-12-26T15:29:58.734957Z","shell.execute_reply.started":"2024-12-26T15:28:23.948773Z","shell.execute_reply":"2024-12-26T15:29:58.733254Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TEIMPATH='/kaggle/input/czii-cryo-et-object-identification/test/static/ExperimentRuns'\nNEXTPATH='VoxelSpacing10.000/denoised.zarr'\nTEST_FILES=os.listdir(TEIMPATH)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:30:10.719024Z","iopub.execute_input":"2024-12-26T15:30:10.719675Z","iopub.status.idle":"2024-12-26T15:30:10.730848Z","shell.execute_reply.started":"2024-12-26T15:30:10.719594Z","shell.execute_reply":"2024-12-26T15:30:10.729194Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temp_pred=preprocess(TEST_FILES,TEIMPATH,NEXTPATH)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:31:44.993459Z","iopub.execute_input":"2024-12-26T15:31:44.994117Z","iopub.status.idle":"2024-12-26T15:31:56.484216Z","shell.execute_reply.started":"2024-12-26T15:31:44.994052Z","shell.execute_reply":"2024-12-26T15:31:56.482837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred90=temp_pred.batch(5).cache()\ndel temp_pred\ngc.collect()\npreds__=temp9.predict(pred90)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T15:31:59.424925Z","iopub.execute_input":"2024-12-26T15:31:59.425993Z","iopub.status.idle":"2024-12-26T15:32:03.307564Z","shell.execute_reply.started":"2024-12-26T15:31:59.425928Z","shell.execute_reply":"2024-12-26T15:32:03.306117Z"}},"outputs":[],"execution_count":null}]}