{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install zarr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:19.082498Z","iopub.execute_input":"2024-12-11T07:57:19.083158Z","iopub.status.idle":"2024-12-11T07:57:31.675790Z","shell.execute_reply.started":"2024-12-11T07:57:19.083118Z","shell.execute_reply":"2024-12-11T07:57:31.674767Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport json\nimport matplotlib.pyplot as plt\nimport zarr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:31.677427Z","iopub.execute_input":"2024-12-11T07:57:31.677708Z","iopub.status.idle":"2024-12-11T07:57:32.082299Z","shell.execute_reply.started":"2024-12-11T07:57:31.677680Z","shell.execute_reply":"2024-12-11T07:57:32.081607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install zarr &> null\n!pip install git+https://github.com/rostepifanov/voxelmentations &> null","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:32.083300Z","iopub.execute_input":"2024-12-11T07:57:32.083906Z","iopub.status.idle":"2024-12-11T07:57:51.167665Z","shell.execute_reply.started":"2024-12-11T07:57:32.083855Z","shell.execute_reply":"2024-12-11T07:57:51.166206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from collections import OrderedDict\n# from torch.utils.data import Dataset\n\n# class cryoetIMDataset(Dataset):\n#     \"\"\"Dataset for loading CT images and points\n#     \"\"\"\n#     def __init__(self, datapath, names, *, augs=None):\n#         \"\"\"\n#             :args:\n#                 datapath: pathlib.Path\n#                     directiry path with data\n#                 names: list of str\n#                     names of data to load\n#                 augs: dict like\n#                     option to augs\n#         \"\"\"\n#         self.augs = augs\n\n#         self.imgs = OrderedDict()\n#         self.masks = dict()\n#         self.points = dict()\n#         self.types = dict()\n\n#         self.keys = list()\n#         self.shapes = list()\n\n#         self.mtypes = ['apo-ferritin', 'beta-galactosidase', 'ribosome', 'thyroglobulin', 'virus-like-particle']\n\n#         particle_radius = {\n#             'apo-ferritin': 60,\n#             'beta-amylase': 65,\n#             'beta-galactosidase': 90,\n#             'ribosome': 150,\n#             'thyroglobulin': 130,\n#             'virus-like-particle': 135,\n#         }\n        \n#         for name in names:\n#             group = zarr.open(datapath / 'static/ExperimentRuns' / name / 'VoxelSpacing10.000/denoised.zarr', mode='r')\n#             voxel = group[0].astype(np.float32)\n#             voxel = np.transpose(voxel, (1, 2, 0)) # set orientation as xyz\n            \n#             scale = group.attrs['multiscales'][0]['datasets'][0]['coordinateTransformations'][0]['scale'] # read correct scale not just 10\n            \n#             mask = np.zeros_like(voxel, dtype=np.uint8)\n\n#             points = []\n#             types = []\n            \n#             x, y, z = np.indices(mask.shape)\n\n#             for tdx, type_ in enumerate(self.mtypes):\n#                 with open(datapath / 'overlay/ExperimentRuns' / name / 'Picks' / (type_ + '.json')) as f:\n#                     data = json.load(f)\n\n#                     for entry in data['points']:\n#                         location = entry['location']\n#                         point = np.array([location['y'], location['x'], location['z']]) / scale\n\n#                         ### some code to draw point on mask\n#                         xc, yc, zc = point\n\n#                         distance_sq = (x - xc)**2 + (y - yc)**2 + (z - zc)**2\n\n#                         radius = 15 * particle_radius[type_] / 150\n#                         mask[distance_sq <= radius**2] = tdx + 1\n#                         ###\n\n#                         points.append(np.array([*point, 1.], dtype=np.float32))\n#                         types.append(tdx+1)\n\n#             points = np.array(points, dtype=np.float32)\n#             types = np.array(types, dtype=np.uint8)\n\n#             self.imgs[name] = voxel\n#             self.masks[name] = mask\n#             self.points[name] = points\n#             self.types[name] = types\n\n#             self.keys.append(name)\n#             self.shapes.append(voxel.shape)\n\n#     def __len__(self):\n#         return len(self.imgs)\n\n#     def __getitem__(self, idx):\n#         key, selector = idx\n\n#         voxel = self.imgs[key][selector]\n#         voxel = voxel[:, :, :, None]\n\n#         mask = self.masks[key][selector]\n\n#         if self.augs:\n#             transformed = augs(voxel=voxel, mask=mask)\n\n#             voxel = transformed['voxel']\n#             mask = transformed['mask']\n\n#         voxel = np.moveaxis(voxel, -1, 0)\n\n#         return voxel, mask, key, selector","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:51.171070Z","iopub.execute_input":"2024-12-11T07:57:51.171511Z","iopub.status.idle":"2024-12-11T07:57:51.179132Z","shell.execute_reply.started":"2024-12-11T07:57:51.171466Z","shell.execute_reply":"2024-12-11T07:57:51.178380Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import voxelmentations as V\n\n# from pathlib import Path\n\n# augs = V.Sequential([\n#     V.AxialPlaneFlip(p=0.5),\n#     V.AxialPlaneTranpose(p=0.5),\n# ])\n\n# gpath = Path('/kaggle/input/czii-cryo-et-object-identification')\n\n# dataset = cryoetIMDataset(gpath / 'train', ['TS_5_4'], augs=augs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:51.180168Z","iopub.execute_input":"2024-12-11T07:57:51.180575Z","iopub.status.idle":"2024-12-11T07:57:51.199602Z","shell.execute_reply.started":"2024-12-11T07:57:51.180548Z","shell.execute_reply":"2024-12-11T07:57:51.198621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n\n# fig, ax = plt.subplots(1, 2, figsize=(10, 20))\n\n# selector = (slice(0, None), slice(0, None), slice(0, 24))\n# data = dataset[('TS_5_4', selector)]\n\n# ax[0].set_title('Selected patch of image\\nwith the shape of 600x600x24')\n# ax[0].imshow(data[0][0, :, :, 12])\n# ax[0].axis('off')\n\n# ax[1].set_title('Selected patch of mask\\nwith the shape of 600x600x24')\n# ax[1].imshow(data[1][:, :, 12])\n# ax[1].axis('off')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:51.200915Z","iopub.execute_input":"2024-12-11T07:57:51.201306Z","iopub.status.idle":"2024-12-11T07:57:51.208975Z","shell.execute_reply.started":"2024-12-11T07:57:51.201263Z","shell.execute_reply":"2024-12-11T07:57:51.208216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import json, os, torch, cv2, random, numpy as np, albumentations as A, nibabel as nib\nfrom matplotlib import pyplot as plt; from glob import glob\nfrom torch.utils.data import random_split, Dataset, DataLoader\nfrom albumentations.pytorch import ToTensorV2; from PIL import Image\nfrom torchvision import transforms as tfs\n# from natsort import natsorted  # Import natsort\nfrom tqdm import tqdm ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:51.210028Z","iopub.execute_input":"2024-12-11T07:57:51.210582Z","iopub.status.idle":"2024-12-11T07:57:56.834043Z","shell.execute_reply.started":"2024-12-11T07:57:51.210555Z","shell.execute_reply":"2024-12-11T07:57:56.833129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"z_ts_6_4 = zarr.open('/kaggle/input/czii-cryo-et-object-identification/test/static/ExperimentRuns/TS_6_4/VoxelSpacing10.000/denoised.zarr', mode='r')\nz_ts_6_4_iso = zarr.open('/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns/TS_6_4/VoxelSpacing10.000/isonetcorrected.zarr', mode='r')\nz_ts_6_4_dcon = zarr.open('/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns/TS_6_4/VoxelSpacing10.000/ctfdeconvolved.zarr', mode='r')\nz_ts_6_4_wbp = zarr.open('/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns/TS_6_4/VoxelSpacing10.000/wbp.zarr', mode='r')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:56.835255Z","iopub.execute_input":"2024-12-11T07:57:56.835778Z","iopub.status.idle":"2024-12-11T07:57:56.937575Z","shell.execute_reply.started":"2024-12-11T07:57:56.835748Z","shell.execute_reply":"2024-12-11T07:57:56.936651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the first zarr.\nz_ts_6_4 = zarr.open('/kaggle/input/czii-cryo-et-object-identification/test/static/ExperimentRuns/TS_6_4/VoxelSpacing10.000/denoised.zarr', mode='r')\nz_ts_6_4_iso = zarr.open('/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns/TS_6_4/VoxelSpacing10.000/isonetcorrected.zarr', mode='r')\nz_ts_6_4_dcon = zarr.open('/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns/TS_6_4/VoxelSpacing10.000/ctfdeconvolved.zarr', mode='r')\nz_ts_6_4_wbp = zarr.open('/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns/TS_6_4/VoxelSpacing10.000/wbp.zarr', mode='r')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:56.938812Z","iopub.execute_input":"2024-12-11T07:57:56.939218Z","iopub.status.idle":"2024-12-11T07:57:56.947526Z","shell.execute_reply.started":"2024-12-11T07:57:56.939177Z","shell.execute_reply":"2024-12-11T07:57:56.946638Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(z_ts_6_4)\nprint(z_ts_6_4[0].shape)\nprint(z_ts_6_4[1].shape)\nprint(z_ts_6_4[2].shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:56.951101Z","iopub.execute_input":"2024-12-11T07:57:56.951360Z","iopub.status.idle":"2024-12-11T07:57:56.981060Z","shell.execute_reply.started":"2024-12-11T07:57:56.951336Z","shell.execute_reply":"2024-12-11T07:57:56.980299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(6.3,6.3))\n# _ = plt.imshow(z_ts_6_4[0][0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:56.982199Z","iopub.execute_input":"2024-12-11T07:57:56.982531Z","iopub.status.idle":"2024-12-11T07:57:56.986206Z","shell.execute_reply.started":"2024-12-11T07:57:56.982494Z","shell.execute_reply":"2024-12-11T07:57:56.985288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(10,74))\n# for i in range(184):\n#     ax = plt.subplot(37, 5, i + 1)\n#     plt.xticks([])\n#     plt.yticks([])\n#     plt.imshow(z_ts_6_4[0][i])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:56.987224Z","iopub.execute_input":"2024-12-11T07:57:56.987458Z","iopub.status.idle":"2024-12-11T07:57:56.995643Z","shell.execute_reply.started":"2024-12-11T07:57:56.987434Z","shell.execute_reply":"2024-12-11T07:57:56.994833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(10,38))\n# for i in range(92):\n#     ax = plt.subplot(19, 5, i + 1)\n#     plt.xticks([])\n#     plt.yticks([])\n#     plt.imshow(z_ts_6_4[1][i])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:56.996736Z","iopub.execute_input":"2024-12-11T07:57:56.997616Z","iopub.status.idle":"2024-12-11T07:57:57.006084Z","shell.execute_reply.started":"2024-12-11T07:57:56.997587Z","shell.execute_reply":"2024-12-11T07:57:57.005222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(10,20))\n# for i in range(46):\n#     ax = plt.subplot(10, 5, i + 1)\n#     plt.xticks([])\n#     plt.yticks([])\n#     plt.imshow(z_ts_6_4[2][i])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.006981Z","iopub.execute_input":"2024-12-11T07:57:57.007223Z","iopub.status.idle":"2024-12-11T07:57:57.015688Z","shell.execute_reply.started":"2024-12-11T07:57:57.007200Z","shell.execute_reply":"2024-12-11T07:57:57.014763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(10,10))\n# ax = plt.subplot(2, 2, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Denoised')\n# plt.imshow(z_ts_6_4[0][62], cmap='gray')\n# ax = plt.subplot(2, 2, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('IsoNet Corrected')\n# plt.imshow(z_ts_6_4_iso[0][62], cmap='gray')\n# ax = plt.subplot(2, 2, 3)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('CTF Deconvolved')\n# plt.imshow(z_ts_6_4_dcon[0][62], cmap='gray')\n# ax = plt.subplot(2, 2, 4)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Weighted Back Projection')\n# _ = plt.imshow(z_ts_6_4_wbp[0][62], cmap='gray')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.016878Z","iopub.execute_input":"2024-12-11T07:57:57.017182Z","iopub.status.idle":"2024-12-11T07:57:57.024800Z","shell.execute_reply.started":"2024-12-11T07:57:57.017127Z","shell.execute_reply":"2024-12-11T07:57:57.024019Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(10,10))\n# ax = plt.subplot(2, 2, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Denoised')\n# plt.imshow(z_ts_6_4[0][10], cmap='gray')\n# ax = plt.subplot(2, 2, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('IsoNet Corrected')\n# plt.imshow(z_ts_6_4_iso[0][10], cmap='gray')\n# ax = plt.subplot(2, 2, 3)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('CTF Deconvolved')\n# plt.imshow(z_ts_6_4_dcon[0][10], cmap='gray')\n# ax = plt.subplot(2, 2, 4)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Weighted Back Projection')\n# _ = plt.imshow(z_ts_6_4_wbp[0][10], cmap='gray')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.025734Z","iopub.execute_input":"2024-12-11T07:57:57.026086Z","iopub.status.idle":"2024-12-11T07:57:57.034036Z","shell.execute_reply.started":"2024-12-11T07:57:57.026049Z","shell.execute_reply":"2024-12-11T07:57:57.033284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ribosomes_x = []\n# ribosomes_y = []\n# f = open('/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/TS_6_4/Picks/ribosome.json')\n# for p in json.loads(f.read())['points']:\n#     z = float(p['location']['z'])\n#     if z >= 600 and z < 650:\n#         ribosomes_x.append(float(p['location']['x'])/10)\n#         ribosomes_y.append(float(p['location']['y'])/10)\n#         print(p['location'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.034908Z","iopub.execute_input":"2024-12-11T07:57:57.035163Z","iopub.status.idle":"2024-12-11T07:57:57.045531Z","shell.execute_reply.started":"2024-12-11T07:57:57.035127Z","shell.execute_reply":"2024-12-11T07:57:57.044653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# from mpl_toolkits.mplot3d import Axes3D\n\n# # Dữ liệu tọa độ (có thể thay bằng tọa độ của bạn)\n# coordinates = [\n#     {'x': 5274.903, 'y': 5288.121, 'z': 619.798},\n#     {'x': 5312.439, 'y': 4964.156, 'z': 602.574},\n#     {'x': 5130.837, 'y': 5121.036, 'z': 631.702},\n#     {'x': 5106.838, 'y': 4835.263, 'z': 619.225},\n#     {'x': 1273.648, 'y': 4386.703, 'z': 631.656},\n#     {'x': 440.984, 'y': 3866.177, 'z': 643.857},\n#     {'x': 1015.725, 'y': 640.198, 'z': 622.387}\n# ]\n\n# # Lấy các tọa độ x, y, z\n# x_vals = [coord['x'] for coord in coordinates]\n# y_vals = [coord['y'] for coord in coordinates]\n# z_vals = [coord['z'] for coord in coordinates]\n\n# # Tạo hình vẽ 3D\n# fig = plt.figure(figsize=(7, 8))\n# ax = fig.add_subplot(111, projection='3d')\n\n# # Vẽ các điểm 3D\n# ax.scatter(x_vals, y_vals, z_vals, c='r', marker='o')\n\n# # Đặt tên trục\n# ax.set_xlabel('X Label')\n# ax.set_ylabel('Y Label')\n# ax.set_zlabel('Z Label')\n\n# # Hiển thị đồ thị\n# plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.046587Z","iopub.execute_input":"2024-12-11T07:57:57.046900Z","iopub.status.idle":"2024-12-11T07:57:57.054323Z","shell.execute_reply.started":"2024-12-11T07:57:57.046866Z","shell.execute_reply":"2024-12-11T07:57:57.053548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(10, 5))\n\n# # Vẽ hình ảnh đầu tiên\n# ax = plt.subplot(1, 2, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][62], cmap='gray', vmin=-0.00005, vmax=0.00005)\n\n# # Thêm tọa độ ribosome vào hình ảnh đầu tiên\n# for x, y in zip(ribosomes_x, ribosomes_y):\n#     plt.scatter(x, y, edgecolor='red', facecolor='none')  # Vẽ điểm ribosome\n#     plt.text(x, y, f'({x:.2f}, {y:.2f})', color='red', fontsize=8, ha='right', va='bottom')  # Hiển thị tọa độ\n\n# # Vẽ hình ảnh thứ hai\n# ax = plt.subplot(1, 2, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][62], cmap='gray', vmin=-0.00005, vmax=0.00005)\n\n# # Thêm tọa độ ribosome vào hình ảnh thứ hai\n# for x, y in zip(ribosomes_x, ribosomes_y):\n#     plt.scatter(x, y, edgecolor='red', facecolor='none')  # Vẽ điểm ribosome\n#     plt.text(x, y, f'({x:.2f}, {y:.2f})', color='red', fontsize=8, ha='right', va='bottom')  # Hiển thị tọa độ\n\n# plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.055217Z","iopub.execute_input":"2024-12-11T07:57:57.055476Z","iopub.status.idle":"2024-12-11T07:57:57.068699Z","shell.execute_reply.started":"2024-12-11T07:57:57.055451Z","shell.execute_reply":"2024-12-11T07:57:57.067952Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(10, 5))\n\n# # Vẽ hình ảnh đầu tiên\n# ax = plt.subplot(1, 2, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][5], cmap='gray', vmin=-0.00005, vmax=0.00005)\n\n# # Thêm tọa độ ribosome vào hình ảnh đầu tiên\n# for x, y in zip(ribosomes_x, ribosomes_y):\n#     plt.scatter(x, y, edgecolor='red', facecolor='none')  # Vẽ điểm ribosome\n#     plt.text(x, y, f'({x:.2f}, {y:.2f})', color='red', fontsize=8, ha='right', va='bottom')  # Hiển thị tọa độ\n\n# # Vẽ hình ảnh thứ hai\n# ax = plt.subplot(1, 2, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][5], cmap='gray', vmin=-0.00005, vmax=0.00005)\n\n# # Thêm tọa độ ribosome vào hình ảnh thứ hai\n# for x, y in zip(ribosomes_x, ribosomes_y):\n#     plt.scatter(x, y, edgecolor='red', facecolor='none')  # Vẽ điểm ribosome\n#     plt.text(x, y, f'({x:.2f}, {y:.2f})', color='red', fontsize=8, ha='right', va='bottom')  # Hiển thị tọa độ\n\n# plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.069658Z","iopub.execute_input":"2024-12-11T07:57:57.069999Z","iopub.status.idle":"2024-12-11T07:57:57.081524Z","shell.execute_reply.started":"2024-12-11T07:57:57.069960Z","shell.execute_reply":"2024-12-11T07:57:57.080722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# virus_x = []\n# virus_y = []\n# f = open('/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/TS_6_4/Picks/virus-like-particle.json')\n# for p in json.loads(f.read())['points']:\n#     z = float(p['location']['z'])\n#     if z >= 670 and z < 700:\n#         virus_x.append(float(p['location']['x'])/10)\n#         virus_y.append(float(p['location']['y'])/10)\n#         print(p['location'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.082656Z","iopub.execute_input":"2024-12-11T07:57:57.082998Z","iopub.status.idle":"2024-12-11T07:57:57.089791Z","shell.execute_reply.started":"2024-12-11T07:57:57.082963Z","shell.execute_reply":"2024-12-11T07:57:57.089147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(10,5))\n# ax = plt.subplot(1, 2, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][68], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# ax = plt.subplot(1, 2, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][68], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# _ = plt.scatter(virus_x, virus_y, edgecolor='red', facecolor='none')\n# for x, y in zip(virus_x, virus_y):\n#     plt.scatter(x, y, edgecolor='red', facecolor='none')  # Vẽ điểm ribosome\n#     plt.text(x, y, f'({x:.2f}, {y:.2f})', color='red', fontsize=8, ha='right', va='bottom')  # Hiển thị tọa độ\n\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.090820Z","iopub.execute_input":"2024-12-11T07:57:57.091129Z","iopub.status.idle":"2024-12-11T07:57:57.101746Z","shell.execute_reply.started":"2024-12-11T07:57:57.091104Z","shell.execute_reply":"2024-12-11T07:57:57.101128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# apo_ferritin_x = []\n# apo_ferritin_y = []\n# f = open('/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/TS_6_4/Picks/apo-ferritin.json')\n# for p in json.loads(f.read())['points']:\n#     z = float(p['location']['z'])\n#     if z >= 400 and z < 450:\n#         apo_ferritin_x.append(float(p['location']['x'])/10)\n#         apo_ferritin_y.append(float(p['location']['y'])/10)\n#         print(p['location'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.102736Z","iopub.execute_input":"2024-12-11T07:57:57.103115Z","iopub.status.idle":"2024-12-11T07:57:57.112156Z","shell.execute_reply.started":"2024-12-11T07:57:57.103078Z","shell.execute_reply":"2024-12-11T07:57:57.111397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(10,5))\n# ax = plt.subplot(1, 2, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][42], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# ax = plt.subplot(1, 2, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][42], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# _ = plt.scatter(apo_ferritin_x, apo_ferritin_y, edgecolor='red', facecolor='none')\n# for x, y in zip(apo_ferritin_x, apo_ferritin_y):\n#     plt.scatter(x, y, edgecolor='red', facecolor='none')  # Vẽ điểm ribosome\n#     plt.text(x, y, f'({x:.2f}, {y:.2f})', color='red', fontsize=8, ha='right', va='bottom')  # Hiển thị tọa độ\n\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.113109Z","iopub.execute_input":"2024-12-11T07:57:57.113359Z","iopub.status.idle":"2024-12-11T07:57:57.121460Z","shell.execute_reply.started":"2024-12-11T07:57:57.113335Z","shell.execute_reply":"2024-12-11T07:57:57.120643Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# beta_galactosidase_x = []\n# beta_galactosidase_y = []\n# f = open('/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/TS_6_4/Picks/beta-galactosidase.json')\n# for p in json.loads(f.read())['points']:\n#     z = float(p['location']['z'])\n#     if z >= 450 and z < 500:\n#         beta_galactosidase_x.append(float(p['location']['x'])/10)\n#         beta_galactosidase_y.append(float(p['location']['y'])/10)\n#         print(p['location'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.122345Z","iopub.execute_input":"2024-12-11T07:57:57.122608Z","iopub.status.idle":"2024-12-11T07:57:57.132521Z","shell.execute_reply.started":"2024-12-11T07:57:57.122569Z","shell.execute_reply":"2024-12-11T07:57:57.131710Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(10,5))\n# ax = plt.subplot(1, 2, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][47], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# ax = plt.subplot(1, 2, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][47], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# _ = plt.scatter(beta_galactosidase_x, beta_galactosidase_y, edgecolor='red', facecolor='none')\n# for x, y in zip(beta_galactosidase_x, beta_galactosidase_y):\n#     plt.scatter(x, y, edgecolor='red', facecolor='none')  # Vẽ điểm ribosome\n#     plt.text(x, y, f'({x:.2f}, {y:.2f})', color='red', fontsize=8, ha='right', va='bottom')  # Hiển thị tọa độ\n\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.133458Z","iopub.execute_input":"2024-12-11T07:57:57.133698Z","iopub.status.idle":"2024-12-11T07:57:57.141080Z","shell.execute_reply.started":"2024-12-11T07:57:57.133670Z","shell.execute_reply":"2024-12-11T07:57:57.140238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# thyroglobulin_x = []\n# thyroglobulin_y = []\n# f = open('/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/TS_6_4/Picks/thyroglobulin.json')\n# for p in json.loads(f.read())['points']:\n#     z = float(p['location']['z'])\n#     if z >= 550 and z < 600:\n#         thyroglobulin_x.append(float(p['location']['x'])/10)\n#         thyroglobulin_y.append(float(p['location']['y'])/10)\n#         print(p['location'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.141999Z","iopub.execute_input":"2024-12-11T07:57:57.142218Z","iopub.status.idle":"2024-12-11T07:57:57.149562Z","shell.execute_reply.started":"2024-12-11T07:57:57.142196Z","shell.execute_reply":"2024-12-11T07:57:57.148944Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fig = plt.figure(figsize=(10,5))\n# ax = plt.subplot(1, 2, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][57], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# ax = plt.subplot(1, 2, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.imshow(z_ts_6_4[0][57], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# _ = plt.scatter(thyroglobulin_x, thyroglobulin_y, edgecolor='red', facecolor='none')\n# for x, y in zip(thyroglobulin_x, thyroglobulin_y):\n#     plt.scatter(x, y, edgecolor='red', facecolor='none')  # Vẽ điểm ribosome\n#     plt.text(x, y, f'({x:.2f}, {y:.2f})', color='red', fontsize=8, ha='right', va='bottom')  # Hiển thị tọa độ\n\n# plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.155321Z","iopub.execute_input":"2024-12-11T07:57:57.155649Z","iopub.status.idle":"2024-12-11T07:57:57.161671Z","shell.execute_reply.started":"2024-12-11T07:57:57.155625Z","shell.execute_reply":"2024-12-11T07:57:57.161124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # {'x': 5106.838, 'y': 4835.263, 'z': 619.225}\n# fig = plt.figure(figsize=(10,2.5))\n# ax = plt.subplot(1, 4, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Original')\n# plt.imshow(z_ts_6_4[0][61], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# _ = plt.scatter([5106.838/10], [4835.263/10], edgecolor='red', facecolor='none')\n# ax = plt.subplot(1, 4, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Straight On')\n# plt.imshow(z_ts_6_4[0][61, 461:505, 488:532], cmap='gray')\n# ax = plt.subplot(1, 4, 3)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Side View')\n# plt.imshow(np.transpose(z_ts_6_4[0], axes=(2,1,0))[510, 461:505, 39:83], cmap='gray')\n# ax = plt.subplot(1, 4, 4)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Top View')\n# _ = plt.imshow(np.transpose(z_ts_6_4[0], axes=(1,0,2))[483, 39:83, 488:532], cmap='gray')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.162769Z","iopub.execute_input":"2024-12-11T07:57:57.163107Z","iopub.status.idle":"2024-12-11T07:57:57.171640Z","shell.execute_reply.started":"2024-12-11T07:57:57.163070Z","shell.execute_reply":"2024-12-11T07:57:57.170899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # {'x': 5580.108, 'y': 1240.86, 'z': 692.222}\n# fig = plt.figure(figsize=(10,2.5))\n# ax = plt.subplot(1, 4, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Original')\n# plt.imshow(z_ts_6_4[0][69], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# _ = plt.scatter([5580.108/10], [1240.86/10], edgecolor='red', facecolor='none')\n# ax = plt.subplot(1, 4, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Straight On')\n# plt.imshow(z_ts_6_4[0][69, 102:146, 536:580], cmap='gray')\n# ax = plt.subplot(1, 4, 3)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Side View')\n# plt.imshow(np.transpose(z_ts_6_4[0], axes=(2,1,0))[558, 102:146, 47:91], cmap='gray')\n# ax = plt.subplot(1, 4, 4)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Top View')\n# _ = plt.imshow(np.transpose(z_ts_6_4[0], axes=(1,0,2))[124, 47:91, 536:580], cmap='gray')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.172538Z","iopub.execute_input":"2024-12-11T07:57:57.172796Z","iopub.status.idle":"2024-12-11T07:57:57.180555Z","shell.execute_reply.started":"2024-12-11T07:57:57.172771Z","shell.execute_reply":"2024-12-11T07:57:57.179770Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # {'x': 1019.831, 'y': 1859.831, 'z': 400.424}\n# fig = plt.figure(figsize=(10,2.5))\n# ax = plt.subplot(1, 4, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Original')\n# plt.imshow(z_ts_6_4[0][40], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# _ = plt.scatter([1019.831/10], [1859.831/10], edgecolor='red', facecolor='none')\n# ax = plt.subplot(1, 4, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Straight On')\n# plt.imshow(z_ts_6_4[0][40, 163:207, 79:123], cmap='gray')\n# ax = plt.subplot(1, 4, 3)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Side View')\n# plt.imshow(np.transpose(z_ts_6_4[0], axes=(2,1,0))[101, 163:207, 18:62], cmap='gray')\n# ax = plt.subplot(1, 4, 4)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Top View')\n# _ = plt.imshow(np.transpose(z_ts_6_4[0], axes=(1,0,2))[185, 18:62, 79:123], cmap='gray')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.181594Z","iopub.execute_input":"2024-12-11T07:57:57.181939Z","iopub.status.idle":"2024-12-11T07:57:57.192163Z","shell.execute_reply.started":"2024-12-11T07:57:57.181905Z","shell.execute_reply":"2024-12-11T07:57:57.191408Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # {'x': 804.615, 'y': 1977.846, 'z': 489.385}\n# fig = plt.figure(figsize=(10,2.5))\n# ax = plt.subplot(1, 4, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Original')\n# plt.imshow(z_ts_6_4[0][48], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# _ = plt.scatter([804.615/10], [1977.846/10], edgecolor='red', facecolor='none')\n# ax = plt.subplot(1, 4, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Straight On')\n# plt.imshow(z_ts_6_4[0][48, 175:219, 58:102], cmap='gray')\n# ax = plt.subplot(1, 4, 3)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Side View')\n# plt.imshow(np.transpose(z_ts_6_4[0], axes=(2,1,0))[80, 175:219, 26:70], cmap='gray')\n# ax = plt.subplot(1, 4, 4)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Top View')\n# _ = plt.imshow(np.transpose(z_ts_6_4[0], axes=(1,0,2))[197, 26:70, 58:102], cmap='gray')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.193018Z","iopub.execute_input":"2024-12-11T07:57:57.193324Z","iopub.status.idle":"2024-12-11T07:57:57.199868Z","shell.execute_reply.started":"2024-12-11T07:57:57.193289Z","shell.execute_reply":"2024-12-11T07:57:57.199210Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # {'x': 1242.919, 'y': 1464.644, 'z': 581.353}\n# fig = plt.figure(figsize=(10,2.5))\n# ax = plt.subplot(1, 4, 1)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Original')\n# plt.imshow(z_ts_6_4[0][58], cmap='gray', vmin=-0.00005, vmax=0.00005)\n# _ = plt.scatter([1242.919/10], [1464.644/10], edgecolor='red', facecolor='none')\n# ax = plt.subplot(1, 4, 2)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Straight On')\n# plt.imshow(z_ts_6_4[0][58, 124:168, 102:146], cmap='gray')\n# ax = plt.subplot(1, 4, 3)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Side View')\n# plt.imshow(np.transpose(z_ts_6_4[0], axes=(2,1,0))[124, 175:219, 36:80], cmap='gray')\n# ax = plt.subplot(1, 4, 4)\n# plt.xticks([])\n# plt.yticks([])\n# plt.title('Top View')\n# _ = plt.imshow(np.transpose(z_ts_6_4[0], axes=(1,0,2))[146, 36:80, 102:146], cmap='gray')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.200944Z","iopub.execute_input":"2024-12-11T07:57:57.201266Z","iopub.status.idle":"2024-12-11T07:57:57.211793Z","shell.execute_reply.started":"2024-12-11T07:57:57.201231Z","shell.execute_reply":"2024-12-11T07:57:57.210979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport cv2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.212718Z","iopub.execute_input":"2024-12-11T07:57:57.213005Z","iopub.status.idle":"2024-12-11T07:57:57.220662Z","shell.execute_reply.started":"2024-12-11T07:57:57.212980Z","shell.execute_reply":"2024-12-11T07:57:57.219898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!rm -rf mask\n!rm -rf image\n!rm -rf timage\n\n!mkdir image\n!mkdir mask\n!mkdir timage","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:57:57.221681Z","iopub.execute_input":"2024-12-11T07:57:57.222025Z","iopub.status.idle":"2024-12-11T07:58:03.372189Z","shell.execute_reply.started":"2024-12-11T07:57:57.221989Z","shell.execute_reply":"2024-12-11T07:58:03.371004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dir0=\"/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/\"\nnames=os.listdir(dir0)\nTSpaths=[]\nfor name in names:\n    TSpaths+=[os.path.join(dir0,name)]\nprint(TSpaths)\n\nPATHS=[]\nfor TSpath in TSpaths:\n    paths=[]\n    for dirname, _, filenames in os.walk(TSpath):\n        for filename in filenames:\n            paths+=[(os.path.join(dirname, filename))]\n    PATHS+=[paths]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:58:03.373797Z","iopub.execute_input":"2024-12-11T07:58:03.374235Z","iopub.status.idle":"2024-12-11T07:58:03.434569Z","shell.execute_reply.started":"2024-12-11T07:58:03.374205Z","shell.execute_reply":"2024-12-11T07:58:03.433728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import json\n\ndef load_from_json(file_path):\n    with open(file_path, 'r', encoding='utf-8') as f:\n        data = json.load(f)\n    return data\n\nfrom collections import defaultdict\ncnt = defaultdict(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:58:03.435547Z","iopub.execute_input":"2024-12-11T07:58:03.435861Z","iopub.status.idle":"2024-12-11T07:58:03.442238Z","shell.execute_reply.started":"2024-12-11T07:58:03.435803Z","shell.execute_reply":"2024-12-11T07:58:03.441245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LL=[]\nfor paths in PATHS:\n    L=[]\n    for i,path in enumerate(paths):\n        data=load_from_json(path)\n        cnt[i]=data['pickable_object_name']\n        X=[]\n        Y=[]\n        Z=[]\n        Z2=[]\n        for datai in data['points']:\n            xyz=datai['location']\n            X+=[xyz['x']]\n            Y+=[xyz['y']]\n            Z+=[xyz['z']]\n            Z2+=[xyz['z']//50]\n        L+=[(i,X,Y,Z,Z2)]\n    LL+=[L]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:58:03.443250Z","iopub.execute_input":"2024-12-11T07:58:03.443552Z","iopub.status.idle":"2024-12-11T07:58:03.595494Z","shell.execute_reply.started":"2024-12-11T07:58:03.443516Z","shell.execute_reply":"2024-12-11T07:58:03.594763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def z_filtered(L,k,j):\n    \n    fig = plt.figure(figsize=(8,8))\n    ax = fig.add_subplot(111)\n    \n    for data in L:\n        i, X, Y, Z, Z2 = data \n        filtered_indices = [idx for idx, z in enumerate(Z2) if z==j]\n        \n        # Apply the filter to X, Y, and Z2\n        filtered_X = [X[idx] for idx in filtered_indices]\n        filtered_Y = [Y[idx] for idx in filtered_indices]\n        filtered_Z2 = [Z2[idx] for idx in filtered_indices]\n        \n        # Plot the filtered data\n        scatter = ax.scatter(filtered_X, filtered_Y, c=[i] * len(filtered_X),\n                             vmin=0, vmax=5, cmap='viridis')\n    \n    ax.set_xlabel('X Label')\n    ax.set_ylabel('Y Label')\n    #cbar = fig.colorbar(scatter, ax=ax)\n    #cbar.set_label('Label')\n    plt.axis('off')\n    plt.savefig(f'mask/{k}_{j:02}.png', bbox_inches='tight', pad_inches=0)\n    plt.show()\nfor k,L in enumerate(LL):\n    for j in range(35):\n        print(k,j)\n        z_filtered(L,k,j)\n        print()\n        print('-----------------------------'*2)\n        print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:58:03.596510Z","iopub.execute_input":"2024-12-11T07:58:03.596780Z","iopub.status.idle":"2024-12-11T07:58:38.683266Z","shell.execute_reply.started":"2024-12-11T07:58:03.596753Z","shell.execute_reply":"2024-12-11T07:58:38.682385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport zarr\nfrom PIL import Image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:58:38.684266Z","iopub.execute_input":"2024-12-11T07:58:38.684617Z","iopub.status.idle":"2024-12-11T07:58:38.689086Z","shell.execute_reply.started":"2024-12-11T07:58:38.684574Z","shell.execute_reply":"2024-12-11T07:58:38.688207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dir1='/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns'\n\nnames=os.listdir(dir1)\nSpaths=[]\nfor name in names:\n    Spaths+=[os.path.join(dir1,name)]\nprint(Spaths)\n\nZARRS=[]\nfor sp in Spaths:\n    files=os.listdir(os.path.join(sp,'VoxelSpacing10.000'))\n    ZARR=[]\n    for file in files:\n        ZARR+=[os.path.join(sp,'VoxelSpacing10.000',file)]\n    ZARRS+=[ZARR]\n\nfor k,ZARR in enumerate(ZARRS):\n    for j,diri in enumerate(ZARR[0:1]):\n        print(ZARR[j].split('/')[-1]) \n        data = zarr.open(diri, mode='r') \n        fig = plt.figure(figsize=(12,18))\n    \n        for i in range(35):\n            ax = plt.subplot(7, 5, i + 1)\n            plt.axis('off')\n            image=data[2][i]\n            min_val, max_val = image.min(), image.max()\n            simage = ((image - min_val) / (max_val - min_val) * 255).astype('uint8')\n            plt.imshow(simage)\n            cv2.imwrite(f'image/{k}_{i:02}.png', simage)\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:58:38.689934Z","iopub.execute_input":"2024-12-11T07:58:38.690236Z","iopub.status.idle":"2024-12-11T07:58:58.578309Z","shell.execute_reply.started":"2024-12-11T07:58:38.690201Z","shell.execute_reply":"2024-12-11T07:58:58.576895Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dir2='/kaggle/input/czii-cryo-et-object-identification/test/static/ExperimentRuns'\n\nnames=os.listdir(dir2)\ntSpaths=[]\nfor name in names:\n    tSpaths+=[os.path.join(dir2,name)]\nprint(tSpaths)\n\ntZARRS=[]\nfor sp in tSpaths:\n    files=os.listdir(os.path.join(sp,'VoxelSpacing10.000'))\n    tZARR=[]\n    for file in files:\n        tZARR+=[os.path.join(sp,'VoxelSpacing10.000',file)]\n    tZARRS+=[tZARR]\n\nfor k,tZARR in enumerate(tZARRS):\n    for j,diri in enumerate(tZARR[0:1]):\n        print(tZARR[j].split('/')[-1]) \n        data = zarr.open(diri, mode='r') \n        fig = plt.figure(figsize=(12,18))\n    \n        for i in range(35):\n            ax = plt.subplot(7, 5, i + 1)\n            plt.axis('off')\n            image=data[2][i]\n            min_val, max_val = image.min(), image.max()\n            simage = ((image - min_val) / (max_val - min_val) * 255).astype('uint8')\n            plt.imshow(simage)\n            cv2.imwrite(f'timage/{k}_{i:02}.png', simage)\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:58:58.579642Z","iopub.execute_input":"2024-12-11T07:58:58.579973Z","iopub.status.idle":"2024-12-11T07:59:07.024138Z","shell.execute_reply.started":"2024-12-11T07:58:58.579942Z","shell.execute_reply":"2024-12-11T07:59:07.022704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install imantics --quiet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:07.025374Z","iopub.execute_input":"2024-12-11T07:59:07.025669Z","iopub.status.idle":"2024-12-11T07:59:17.173811Z","shell.execute_reply.started":"2024-12-11T07:59:07.025642Z","shell.execute_reply":"2024-12-11T07:59:17.172800Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport tensorflow as tf\nimport json\nimport os\nimport imantics\nfrom PIL import Image\nfrom skimage.transform import resize\nimport random\nfrom sklearn.model_selection import train_test_split\n%matplotlib inline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:17.175367Z","iopub.execute_input":"2024-12-11T07:59:17.175694Z","iopub.status.idle":"2024-12-11T07:59:28.546192Z","shell.execute_reply.started":"2024-12-11T07:59:17.175667Z","shell.execute_reply":"2024-12-11T07:59:28.545245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tracemalloc\ntracemalloc.start()\n\ndef print_memory():\n    snapshot = tracemalloc.take_snapshot()\n    total_memory = sum(stat.size for stat in snapshot.statistics('lineno')) \n    total_memory_mb = total_memory / 1024**2  \n    print(f\"{total_memory_mb:.2f} MB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:28.547334Z","iopub.execute_input":"2024-12-11T07:59:28.547976Z","iopub.status.idle":"2024-12-11T07:59:28.556885Z","shell.execute_reply.started":"2024-12-11T07:59:28.547945Z","shell.execute_reply":"2024-12-11T07:59:28.555401Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print_memory()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:28.559115Z","iopub.execute_input":"2024-12-11T07:59:28.559812Z","iopub.status.idle":"2024-12-11T07:59:28.577235Z","shell.execute_reply.started":"2024-12-11T07:59:28.559725Z","shell.execute_reply":"2024-12-11T07:59:28.575995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_dir = '/kaggle/working/'\nimages_dir = f'{base_dir}/image' \nmasks_dir = f'{base_dir}/mask' \ntimages_dir = f'{base_dir}/timage' ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:28.578950Z","iopub.execute_input":"2024-12-11T07:59:28.579397Z","iopub.status.idle":"2024-12-11T07:59:28.587792Z","shell.execute_reply.started":"2024-12-11T07:59:28.579353Z","shell.execute_reply":"2024-12-11T07:59:28.586312Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"images_listdir = os.listdir(images_dir)\nrandom_images = np.random.choice(images_listdir, size = 9, replace = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:28.589364Z","iopub.execute_input":"2024-12-11T07:59:28.589746Z","iopub.status.idle":"2024-12-11T07:59:28.602813Z","shell.execute_reply.started":"2024-12-11T07:59:28.589707Z","shell.execute_reply":"2024-12-11T07:59:28.600945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_size=512","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:28.604932Z","iopub.execute_input":"2024-12-11T07:59:28.605312Z","iopub.status.idle":"2024-12-11T07:59:28.611582Z","shell.execute_reply.started":"2024-12-11T07:59:28.605272Z","shell.execute_reply":"2024-12-11T07:59:28.610119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_image(path):\n    img = cv2.imread(path)\n    #img = cv2.imread(path,cv2.IMREAD_ANYDEPTH)\n    #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img2 = cv2.resize(img, (image_size, image_size))\n    #print(img.shape,img2.shape)\n    return img2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:28.613483Z","iopub.execute_input":"2024-12-11T07:59:28.614027Z","iopub.status.idle":"2024-12-11T07:59:28.624481Z","shell.execute_reply.started":"2024-12-11T07:59:28.613985Z","shell.execute_reply":"2024-12-11T07:59:28.622622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MASKS=np.zeros((1,image_size,image_size,3),dtype=np.uint8)\nIMAGES=np.zeros((1,image_size,image_size,3),dtype=np.uint8)\n\nfor j,file in enumerate(images_listdir[0:81]): ##the smaller, the faster\n    #print(j)\n    image = read_image(f\"{images_dir}/{file}\")\n    image_ex = np.expand_dims(image, axis=0)\n    IMAGES = np.vstack([IMAGES, image_ex])\n    \n    file2=file\n    mask = read_image(f\"{masks_dir}/{file2}\")\n    #mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY) #####\n    mask_ex = np.expand_dims(mask, axis=0)    \n    MASKS = np.vstack([MASKS, mask_ex])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T08:04:48.146294Z","iopub.execute_input":"2024-12-11T08:04:48.146869Z","iopub.status.idle":"2024-12-11T08:04:50.339056Z","shell.execute_reply.started":"2024-12-11T08:04:48.146788Z","shell.execute_reply":"2024-12-11T08:04:50.337315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"images=np.array(IMAGES)[1:81]\nmasks=np.array(MASKS)[1:81]\nprint(images.shape,masks.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T08:04:51.962725Z","iopub.execute_input":"2024-12-11T08:04:51.963295Z","iopub.status.idle":"2024-12-11T08:04:52.020178Z","shell.execute_reply.started":"2024-12-11T08:04:51.963242Z","shell.execute_reply":"2024-12-11T08:04:52.018665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"images_train, images_test, masks_train, masks_test = train_test_split(\n    images, masks, test_size=0.4, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T08:05:00.135378Z","iopub.execute_input":"2024-12-11T08:05:00.136583Z","iopub.status.idle":"2024-12-11T08:05:00.182622Z","shell.execute_reply.started":"2024-12-11T08:05:00.136518Z","shell.execute_reply":"2024-12-11T08:05:00.181055Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(len(images_train), len(masks_train))\nprint(len(images_test), len(masks_test))\n\n# images_train = images_train.reshape(-1, 1, 512, 512, 3)\n# masks_train = masks_train.reshape(-1, 1, 512, 512, 1) \n# images_test = images_test.reshape(-1, 1, 512, 512, 3)\n# masks_test = masks_test.reshape(-1, 1, 512, 512, 1) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T08:05:00.954601Z","iopub.execute_input":"2024-12-11T08:05:00.955732Z","iopub.status.idle":"2024-12-11T08:05:00.964907Z","shell.execute_reply.started":"2024-12-11T08:05:00.955673Z","shell.execute_reply":"2024-12-11T08:05:00.963378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import tensorflow as tf\n\n# def print_shape(name, tensor):\n#     \"\"\"Shape printing function for debugging\"\"\"\n#     print(f\"{name} shape:\", tensor.shape)\n#     return tensor\n\n# def encoder_block(input, num_filters, name):\n#     conv = conv_block(input, num_filters, f\"{name}_conv\")\n#     pool = tf.keras.layers.MaxPooling3D((2, 2, 2), strides=(2, 2, 2), padding=\"same\", name=f\"{name}_pool\")(conv)\n#     print_shape(f\"{name}_conv\", conv)\n#     print_shape(f\"{name}_pool\", pool)\n#     return conv, pool\n\n# def decoder_block(input, skip, num_filters, name):\n#     print_shape(f\"{name}_input\", input)\n#     print_shape(f\"{name}_skip\", skip)\n    \n#     # Upsample to match the shape of the skip connection\n#     target_shape = skip.shape\n#     current_shape = input.shape\n#     up_size = (\n#         target_shape[1] // current_shape[1],\n#         target_shape[2] // current_shape[2],\n#         target_shape[3] // current_shape[3]\n#     )\n    \n#     up = tf.keras.layers.UpSampling3D(size=up_size, name=f\"{name}_up\")(input)\n#     print_shape(f\"{name}_up\", up)\n    \n#     # Concatenate with the skip connection\n#     concat = tf.keras.layers.Concatenate(axis=-1, name=f\"{name}_concat\")([up, skip])\n#     print_shape(f\"{name}_concat\", concat)\n    \n#     conv = conv_block(concat, num_filters, f\"{name}_conv\")\n#     print_shape(f\"{name}_conv_out\", conv)\n#     return conv\n\n# def conv_block(input, num_filters, name):\n#     conv = tf.keras.layers.Conv3D(\n#         num_filters, 3, padding=\"same\", name=f\"{name}_conv1\")(input)\n#     conv = tf.keras.layers.BatchNormalization(name=f\"{name}_bn1\")(conv)\n#     conv = tf.keras.layers.ReLU(name=f\"{name}_relu1\")(conv)\n    \n#     conv = tf.keras.layers.Conv3D(\n#         num_filters, 3, padding=\"same\", name=f\"{name}_conv2\")(conv)\n#     conv = tf.keras.layers.BatchNormalization(name=f\"{name}_bn2\")(conv)\n#     conv = tf.keras.layers.ReLU(name=f\"{name}_relu2\")(conv)\n#     return conv\n\n# def Unet3D(input_shape):\n#     inputs = tf.keras.layers.Input(input_shape)\n#     print_shape(\"input\", inputs)\n    \n#     # Encoder path\n#     skip1, pool1 = encoder_block(inputs, 64, \"encoder1\")\n#     skip2, pool2 = encoder_block(pool1, 128, \"encoder2\")\n#     skip3, pool3 = encoder_block(pool2, 256, \"encoder3\")\n#     skip4, pool4 = encoder_block(pool3, 512, \"encoder4\")\n    \n#     # Bridge\n#     bridge = conv_block(pool4, 1024, \"bridge\")\n#     print_shape(\"bridge\", bridge)\n    \n#     # Decoder path\n#     decode1 = decoder_block(bridge, skip4, 512, \"decoder1\")\n#     decode2 = decoder_block(decode1, skip3, 256, \"decoder2\")\n#     decode3 = decoder_block(decode2, skip2, 128, \"decoder3\")\n#     decode4 = decoder_block(decode3, skip1, 64, \"decoder4\")\n    \n#     # Output layer\n#     outputs = tf.keras.layers.Conv3D(\n#         1, 1, padding=\"same\", activation=\"sigmoid\", name=\"output\")(decode4)\n#     print_shape(\"output\", outputs)\n    \n#     model = tf.keras.models.Model(inputs, outputs, name=\"U-Net3D\")\n#     return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:31.235620Z","iopub.execute_input":"2024-12-11T07:59:31.236067Z","iopub.status.idle":"2024-12-11T07:59:31.257734Z","shell.execute_reply.started":"2024-12-11T07:59:31.236023Z","shell.execute_reply":"2024-12-11T07:59:31.256481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# input_shape = (1, 256, 256, 3)  # (depth, height, width, channels)\n# model = Unet3D(input_shape)\n# model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n# model.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:31.259251Z","iopub.execute_input":"2024-12-11T07:59:31.259687Z","iopub.status.idle":"2024-12-11T07:59:31.271201Z","shell.execute_reply.started":"2024-12-11T07:59:31.259646Z","shell.execute_reply":"2024-12-11T07:59:31.269875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print_memory()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:31.273054Z","iopub.execute_input":"2024-12-11T07:59:31.273431Z","iopub.status.idle":"2024-12-11T07:59:31.281090Z","shell.execute_reply.started":"2024-12-11T07:59:31.273392Z","shell.execute_reply":"2024-12-11T07:59:31.279206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from tensorflow import keras\n\n# import numpy as np\n# import keras\n\n# class SaveMaskImagesCallback(keras.callbacks.Callback):\n#     def __init__(self, test_images, output_dir):\n#         super().__init__()\n#         self.test_images = test_images\n#         self.output_dir = output_dir\n        \n#     def on_epoch_end(self, epoch, logs=None):\n#         predictions = self.model.predict(self.test_images)\n#         for i, pred_mask in enumerate(predictions):\n#             # Squeeze out extra dimensions to get shape (512, 512, 1)\n#             pred_mask = np.squeeze(pred_mask)\n#             # Add channel dimension back if needed\n#             if pred_mask.ndim == 2:\n#                 pred_mask = pred_mask[..., np.newaxis]\n                \n#             output_path = os.path.join(self.output_dir, f\"mask_{i:02d}_ep_{epoch + 1:03d}.png\")\n#             pred_mask = (pred_mask * 255).astype(np.uint8)\n#             keras.preprocessing.image.save_img(output_path, pred_mask)\n            \n# output_directory = \"output_masks\"  \n# os.makedirs(output_directory, exist_ok=True)\n# mask_callback = SaveMaskImagesCallback(images_train, output_directory) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:31.283192Z","iopub.execute_input":"2024-12-11T07:59:31.283633Z","iopub.status.idle":"2024-12-11T07:59:31.295529Z","shell.execute_reply.started":"2024-12-11T07:59:31.283591Z","shell.execute_reply":"2024-12-11T07:59:31.294314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# images_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:31.297130Z","iopub.execute_input":"2024-12-11T07:59:31.297536Z","iopub.status.idle":"2024-12-11T07:59:31.309339Z","shell.execute_reply.started":"2024-12-11T07:59:31.297496Z","shell.execute_reply":"2024-12-11T07:59:31.308164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# masks_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:31.311191Z","iopub.execute_input":"2024-12-11T07:59:31.311636Z","iopub.status.idle":"2024-12-11T07:59:31.321672Z","shell.execute_reply.started":"2024-12-11T07:59:31.311574Z","shell.execute_reply":"2024-12-11T07:59:31.320225Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# images_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:31.323799Z","iopub.execute_input":"2024-12-11T07:59:31.324783Z","iopub.status.idle":"2024-12-11T07:59:31.334574Z","shell.execute_reply.started":"2024-12-11T07:59:31.324721Z","shell.execute_reply":"2024-12-11T07:59:31.333286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# unet_result = model.fit(\n#     images_train, masks_train, \n#     validation_split=0.2, batch_size=1, epochs=50,\n#     callbacks=[mask_callback]\n# )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:31.336562Z","iopub.execute_input":"2024-12-11T07:59:31.337619Z","iopub.status.idle":"2024-12-11T07:59:31.345329Z","shell.execute_reply.started":"2024-12-11T07:59:31.337541Z","shell.execute_reply":"2024-12-11T07:59:31.344029Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n\n# # Định nghĩa các callback\n# early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n# reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3)\n# model_checkpoint = ModelCheckpoint('best_model.keras', monitor='val_loss', save_best_only=True)\n\n# # Huấn luyện mô hình với các tham số đã chỉnh sửa\n# unet_result = model.fit(\n#     images_train, masks_train, \n#     validation_split=0.2, batch_size=8, epochs=50,\n#     callbacks=[early_stopping, reduce_lr, model_checkpoint]\n# )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:31.347405Z","iopub.execute_input":"2024-12-11T07:59:31.348962Z","iopub.status.idle":"2024-12-11T07:59:31.357978Z","shell.execute_reply.started":"2024-12-11T07:59:31.348836Z","shell.execute_reply":"2024-12-11T07:59:31.356661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n# from tensorflow.keras.optimizers import Adam\n\n# # Định nghĩa các callback\n# early_stopping = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True, min_delta=0.0001)\n# reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, min_delta=0.001)\n# model_checkpoint = ModelCheckpoint('best_model.keras', monitor='val_loss', save_best_only=True)\n\n# # Sử dụng optimizer Adam với learning rate điều chỉnh\n# optimizer = Adam(learning_rate=0.001)\n\n# # Huấn luyện mô hình với các tham số đã chỉnh sửa\n# unet_result = model.fit(\n#     images_train, masks_train, \n#     validation_split=0.2, batch_size=16, epochs=30,  # Batch size và epochs đã thay đổi\n#     callbacks=[early_stopping, reduce_lr, model_checkpoint],\n#     optimizer=optimizer\n# )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T07:59:31.359923Z","iopub.execute_input":"2024-12-11T07:59:31.360417Z","iopub.status.idle":"2024-12-11T07:59:31.371485Z","shell.execute_reply.started":"2024-12-11T07:59:31.360376Z","shell.execute_reply":"2024-12-11T07:59:31.370097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def conv_block(input, num_filters):\n    conv = tf.keras.layers.Conv2D(num_filters, 3, padding=\"same\")(input)\n    conv = tf.keras.layers.BatchNormalization()(conv)\n    conv = tf.keras.layers.Activation(\"relu\")(conv)\n    conv = tf.keras.layers.Conv2D(num_filters, 3, padding=\"same\")(conv)\n    conv = tf.keras.layers.BatchNormalization()(conv)\n    conv = tf.keras.layers.Activation(\"relu\")(conv)\n    return conv\n\ndef encoder_block(input, num_filters):\n    skip = conv_block(input, num_filters)\n    pool = tf.keras.layers.MaxPool2D((2,2))(skip)\n    return skip, pool\n\ndef decoder_block(input, skip, num_filters):\n    up_conv = tf.keras.layers.Conv2DTranspose(num_filters, (2,2), strides=2, padding=\"same\")(input)\n    conv = tf.keras.layers.Concatenate()([up_conv, skip])\n    conv = conv_block(conv, num_filters)\n    return conv\n\ndef Unet(input_shape):\n    inputs = tf.keras.layers.Input(input_shape)\n\n    skip1, pool1 = encoder_block(inputs, 64)\n    skip2, pool2 = encoder_block(pool1, 128)\n    skip3, pool3 = encoder_block(pool2, 256)\n    skip4, pool4 = encoder_block(pool3, 512)\n\n    bridge = conv_block(pool4, 1024)\n\n    decode1 = decoder_block(bridge, skip4, 512)\n    decode2 = decoder_block(decode1, skip3, 256)\n    decode3 = decoder_block(decode2, skip2, 128)\n    decode4 = decoder_block(decode3, skip1, 64)\n\n    outputs = tf.keras.layers.Conv2D(3, 1, padding=\"same\", activation=\"sigmoid\")(decode4) #####\n\n    model = tf.keras.models.Model(inputs, outputs, name=\"U-Net\")\n    return model\n\nunet_model = Unet((512,512,3)) # Unet((512,512,3)) # actual size(1041,1511,3)\nunet_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\nunet_model.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T08:05:17.959572Z","iopub.execute_input":"2024-12-11T08:05:17.960182Z","iopub.status.idle":"2024-12-11T08:05:18.830761Z","shell.execute_reply.started":"2024-12-11T08:05:17.960127Z","shell.execute_reply":"2024-12-11T08:05:18.829486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow import keras\n\nclass SaveMaskImagesCallback(keras.callbacks.Callback):\n    def __init__(self, output_dir, images, masks, save_every_n_epochs=1):\n        super(SaveMaskImagesCallback, self).__init__()\n        self.output_dir = output_dir\n        self.images = images\n        self.masks = masks\n        self.save_every_n_epochs = save_every_n_epochs\n\n    def on_epoch_end(self, epoch, logs=None):\n        if (epoch + 1) % self.save_every_n_epochs == 0:\n            predictions = self.model.predict(self.images)\n            for i, pred_mask in enumerate(predictions):\n                output_path = os.path.join(self.output_dir, f\"mask_{i:02d}_ep_{epoch + 1:03d}.png\")\n                pred_mask = (pred_mask * 255).astype(np.uint8)\n                keras.preprocessing.image.save_img(output_path, pred_mask)\n\noutput_directory = \"output_masks\"  \nos.makedirs(output_directory, exist_ok=True)\nmask_callback = SaveMaskImagesCallback(output_directory, images_train, masks_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T08:05:22.322736Z","iopub.execute_input":"2024-12-11T08:05:22.323317Z","iopub.status.idle":"2024-12-11T08:05:22.342041Z","shell.execute_reply.started":"2024-12-11T08:05:22.323262Z","shell.execute_reply":"2024-12-11T08:05:22.340566Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unet_result = unet_model.fit(\n    images_train, masks_train, \n    validation_split=0.2, batch_size=2, epochs=200,\n    callbacks=[mask_callback]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T08:05:24.232552Z","iopub.execute_input":"2024-12-11T08:05:24.233688Z","iopub.status.idle":"2024-12-11T09:05:19.180921Z","shell.execute_reply.started":"2024-12-11T08:05:24.233628Z","shell.execute_reply":"2024-12-11T09:05:19.179075Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unet_predict = unet_model.predict(images_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T09:05:19.184393Z","iopub.execute_input":"2024-12-11T09:05:19.184900Z","iopub.status.idle":"2024-12-11T09:05:22.333412Z","shell.execute_reply.started":"2024-12-11T09:05:19.184843Z","shell.execute_reply":"2024-12-11T09:05:22.331640Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_result(idx, og, unet, target, p):\n    \n    fig, axs = plt.subplots(1, 3, figsize=(12,12))\n    axs[0].set_title(\"Original \"+str(idx))\n    axs[0].imshow(og)\n    axs[0].axis('off')\n    \n    axs[1].set_title(\"U-Net: p>\"+str(p))\n    axs[1].imshow(unet)\n    axs[1].axis('off')\n    \n    axs[2].set_title(\"Ground Truth\")\n    axs[2].imshow(target)\n    axs[2].axis('off')\n\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T09:05:22.336062Z","iopub.execute_input":"2024-12-11T09:05:22.336517Z","iopub.status.idle":"2024-12-11T09:05:22.350501Z","shell.execute_reply.started":"2024-12-11T09:05:22.336469Z","shell.execute_reply":"2024-12-11T09:05:22.348685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"r1,r2,r3,r4=0.7,0.8,0.9,0.99","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T09:05:22.354281Z","iopub.execute_input":"2024-12-11T09:05:22.354707Z","iopub.status.idle":"2024-12-11T09:05:22.362861Z","shell.execute_reply.started":"2024-12-11T09:05:22.354664Z","shell.execute_reply":"2024-12-11T09:05:22.361452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unet_predict1 = (unet_predict > r1).astype(np.uint8)\nunet_predict2 = (unet_predict > r2).astype(np.uint8)\nunet_predict3 = (unet_predict > r3).astype(np.uint8)\nunet_predict4 = (unet_predict > r4).astype(np.uint8)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T09:05:22.364641Z","iopub.execute_input":"2024-12-11T09:05:22.365934Z","iopub.status.idle":"2024-12-11T09:05:22.443556Z","shell.execute_reply.started":"2024-12-11T09:05:22.365861Z","shell.execute_reply":"2024-12-11T09:05:22.442132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"show_test_idx = random.sample(range(len(unet_predict)), 3)\nfor idx in show_test_idx: \n    show_result(idx, images_test[idx], unet_predict1[idx], masks_test[idx], r1)\n    show_result(idx, images_test[idx], unet_predict2[idx], masks_test[idx], r2)\n    show_result(idx, images_test[idx], unet_predict3[idx], masks_test[idx], r3)\n    show_result(idx, images_test[idx], unet_predict4[idx], masks_test[idx], r4)\n    print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T09:05:22.445968Z","iopub.execute_input":"2024-12-11T09:05:22.446464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}