{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"},{"sourceId":9862305,"sourceType":"datasetVersion","datasetId":6052780},{"sourceId":9867543,"sourceType":"datasetVersion","datasetId":6040935},{"sourceId":10066269,"sourceType":"datasetVersion","datasetId":6203847},{"sourceId":10445850,"sourceType":"datasetVersion","datasetId":6465904},{"sourceId":206640467,"sourceType":"kernelVersion"},{"sourceId":211097053,"sourceType":"kernelVersion"},{"sourceId":198003,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":168871,"modelId":191221},{"sourceId":203751,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":173829,"modelId":196168},{"sourceId":203836,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":173905,"modelId":196241},{"sourceId":203893,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":173953,"modelId":196292},{"sourceId":204497,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":174469,"modelId":196817},{"sourceId":204554,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":174515,"modelId":196865},{"sourceId":205497,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":175252,"modelId":197606},{"sourceId":206984,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":176462,"modelId":198783},{"sourceId":207090,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":176559,"modelId":198874},{"sourceId":207521,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":176911,"modelId":199206},{"sourceId":232934,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":198816,"modelId":220641},{"sourceId":234359,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":200180,"modelId":222002},{"sourceId":234728,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":200502,"modelId":222317},{"sourceId":234887,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":200641,"modelId":222464},{"sourceId":235150,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":200869,"modelId":222682}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from IPython.display import clear_output\n!tar xfvz /kaggle/input/ultralytics-for-offline-install/archive.tar.gz\n!pip install --no-index --find-links=./packages ultralytics\n!rm -rf ./packages\n\n\n\n\n\nclear_output()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:05:30.052061Z","iopub.execute_input":"2025-01-25T12:05:30.052897Z","iopub.status.idle":"2025-01-25T12:06:02.180133Z","shell.execute_reply.started":"2025-01-25T12:05:30.052864Z","shell.execute_reply":"2025-01-25T12:06:02.179079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try:\n    import zarr\nexcept: \n    !cp -r '/kaggle/input/hengck-czii-cryo-et-01/wheel_file' '/kaggle/working/'\n    !pip install /kaggle/working/wheel_file/asciitree-0.3.3/asciitree-0.3.3\n    !pip install --no-index --find-links=/kaggle/working/wheel_file zarr\n    !pip install --no-index --find-links=/kaggle/working/wheel_file connected-components-3d\nfrom datetime import datetime\nimport pytz\nimport sys\nsys.path.append('/kaggle/input/hengck-czii-cryo-et-01')\nfrom czii_helper import *\nfrom dataset import *\nfrom model2 import *\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\nfrom czii_helper import *\n#import lightning.pytorch as pl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:06:09.175307Z","iopub.execute_input":"2025-01-25T12:06:09.175835Z","iopub.status.idle":"2025-01-25T12:07:15.775625Z","shell.execute_reply.started":"2025-01-25T12:06:09.175771Z","shell.execute_reply":"2025-01-25T12:07:15.774904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from typing import List, Tuple, Union\ndeps_path = '/kaggle/input/czii-cryoet-dependencies'\n! pip install -q --no-index --find-links {deps_path} --requirement {deps_path}/requirements.txt\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:07:24.451530Z","iopub.execute_input":"2025-01-25T12:07:24.451889Z","iopub.status.idle":"2025-01-25T12:07:42.295976Z","shell.execute_reply.started":"2025-01-25T12:07:24.451856Z","shell.execute_reply":"2025-01-25T12:07:42.294812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nimport time\nimport sys\nimport warnings\nimport math\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport torch\nfrom tqdm import tqdm\nfrom ultralytics import YOLO\nimport zarr\nfrom scipy.spatial import cKDTree\nfrom collections import defaultdict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:07:57.933847Z","iopub.execute_input":"2025-01-25T12:07:57.934233Z","iopub.status.idle":"2025-01-25T12:08:00.305896Z","shell.execute_reply.started":"2025-01-25T12:07:57.934198Z","shell.execute_reply":"2025-01-25T12:08:00.304969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_path = '/kaggle/input/czii-yolo-l-trained-with-synthetic-data/best_synthetic.pt'\nmodel = YOLO(model_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:10:08.222369Z","iopub.execute_input":"2025-01-25T12:10:08.223442Z","iopub.status.idle":"2025-01-25T12:10:08.801832Z","shell.execute_reply.started":"2025-01-25T12:10:08.223404Z","shell.execute_reply":"2025-01-25T12:10:08.801069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"runs_path = '/kaggle/input/czii-cryo-et-object-identification/test/static/ExperimentRuns/*'\nruns = sorted(glob.glob(runs_path))\nruns = [os.path.basename(run) for run in runs]\nsp = len(runs)//2\nruns1 = runs[:sp]\nruns1[:5]\n\n#add by @minfuka\nruns2 = runs[sp:]\nruns2[:5]\n\n#add by @minfuka\nassert torch.cuda.device_count() == 2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:10:32.879975Z","iopub.execute_input":"2025-01-25T12:10:32.880370Z","iopub.status.idle":"2025-01-25T12:10:32.913586Z","shell.execute_reply.started":"2025-01-25T12:10:32.880340Z","shell.execute_reply":"2025-01-25T12:10:32.912935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"particle_names = [\n    'apo-ferritin',\n    'beta-amylase',\n    'beta-galactosidase',\n    'ribosome',\n    'thyroglobulin',\n    'virus-like-particle'\n]\n\nparticle_to_index = {\n    'apo-ferritin': 0,\n    'beta-amylase': 1,\n    'beta-galactosidase': 2,\n    'ribosome': 3,\n    'thyroglobulin': 4,\n    'virus-like-particle': 5\n}\n\nindex_to_particle = {index: name for name, index in particle_to_index.items()}\n\nparticle_radius = {\n    'apo-ferritin': 60,\n    'beta-amylase': 65,\n    'beta-galactosidase': 90,\n    'ribosome': 150,\n    'thyroglobulin': 130,\n    'virus-like-particle': 135,\n}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:11:21.201885Z","iopub.execute_input":"2025-01-25T12:11:21.202282Z","iopub.status.idle":"2025-01-25T12:11:21.208059Z","shell.execute_reply.started":"2025-01-25T12:11:21.202250Z","shell.execute_reply":"2025-01-25T12:11:21.206918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# add by @sesasj\nclass UnionFind:\n    def __init__(self, size):\n        self.parent = np.arange(size)\n        self.rank = np.zeros(size, dtype=int)\n\n    def find(self, u):\n        if self.parent[u] != u:\n            self.parent[u] = self.find(self.parent[u])  \n        return self.parent[u]\n\n    def union(self, u, v):\n        u_root = self.find(u)\n        v_root = self.find(v)\n        if u_root == v_root:\n            return\n            \n        if self.rank[u_root] < self.rank[v_root]:\n            self.parent[u_root] = v_root\n        else:\n            self.parent[v_root] = u_root\n            if self.rank[u_root] == self.rank[v_root]:\n                self.rank[u_root] += 1\n\nclass PredictionAggregator:\n    def __init__(self, first_conf=0.2, conf_coef=0.75):\n        self.first_conf = first_conf\n        self.conf_coef = conf_coef\n        self.particle_confs = np.array([0.5, 0.0, 0.2, 0.5, 0.2, 0.5])\n        \n    def convert_to_8bit(self, volume):\n        lower, upper = np.percentile(volume, (0.5, 99.5))\n        clipped = np.clip(volume, lower, upper)\n        scaled = ((clipped - lower) / (upper - lower + 1e-12) * 255).astype(np.uint8)\n        return scaled\n\n    def make_predictions(self, run_id, model, device_no):\n        volume_path = f'/kaggle/input/czii-cryo-et-object-identification/test/static/ExperimentRuns/{run_id}/VoxelSpacing10.000/denoised.zarr'\n        volume = zarr.open(volume_path, mode='r')[0]\n        volume_8bit = self.convert_to_8bit(volume)\n        num_slices = volume_8bit.shape[0]\n\n        detections = {\n            'particle_type': [],\n            'confidence': [],\n            'x': [],\n            'y': [],\n            'z': []\n        }\n\n        for slice_idx in range(num_slices):\n            \n            img = volume_8bit[slice_idx]\n            input_image = cv2.resize(np.stack([img]*3, axis=-1), (640, 640))\n\n            results = model.predict(\n                input_image,\n                save=False,\n                imgsz=640,\n                conf=self.first_conf,\n                device=device_no,\n                batch=1,\n                verbose=False,\n            )\n\n            for result in results:\n                boxes = result.boxes\n                if boxes is None:\n                    continue\n                cls = boxes.cls.cpu().numpy().astype(int)\n                conf = boxes.conf.cpu().numpy()\n                xyxy = boxes.xyxy.cpu().numpy()\n\n                xc = ((xyxy[:, 0] + xyxy[:, 2]) / 2.0) * 10 * (63/64) # 63/64 because of the resize\n                yc = ((xyxy[:, 1] + xyxy[:, 3]) / 2.0) * 10 * (63/64)\n                zc = np.full(xc.shape, slice_idx * 10 + 5)\n\n                particle_types = [index_to_particle[c] for c in cls]\n\n                detections['particle_type'].extend(particle_types)\n                detections['confidence'].extend(conf)\n                detections['x'].extend(xc)\n                detections['y'].extend(yc)\n                detections['z'].extend(zc)\n\n        if not detections['particle_type']:\n            return pd.DataFrame()  \n\n        particle_types = np.array(detections['particle_type'])\n        confidences = np.array(detections['confidence'])\n        xs = np.array(detections['x'])\n        ys = np.array(detections['y'])\n        zs = np.array(detections['z'])\n\n        aggregated_data = []\n\n        for idx, particle in enumerate(particle_names):\n            if particle == 'beta-amylase':\n                continue \n\n            mask = (particle_types == particle)\n            if not np.any(mask):\n                continue  \n                \n            particle_confidences = confidences[mask]\n            particle_xs = xs[mask]\n            particle_ys = ys[mask]\n            particle_zs = zs[mask]\n            # -------------modified by @sersasj ------------------------\n            coords = np.vstack((particle_xs, particle_ys, particle_zs)).T\n\n           \n            z_distance = 30 # How many slices can you \"jump\" to aggregate predictions 10 = 1, 20 = 2...\n            xy_distance = 20 # xy_tol_p2 in original code by ITK8191\n            \n            max_distance = math.sqrt(z_distance**2 + xy_distance**2)\n            tree = cKDTree(coords)            \n            pairs = tree.query_pairs(r=max_distance, p=2)\n\n            \n            uf = UnionFind(len(coords))\n            \n            coords_xy = coords[:, :2]\n            coords_z = coords[:, 2]\n            for u, v in pairs:\n                z_diff = abs(coords_z[u] - coords_z[v])\n                if z_diff > z_distance:\n                    continue  \n\n                xy_diff = np.linalg.norm(coords_xy[u] - coords_xy[v])\n                if xy_diff > xy_distance:\n                    continue  \n\n                uf.union(u, v)\n\n            roots = np.array([uf.find(i) for i in range(len(coords))])\n            unique_roots, inverse_indices, counts = np.unique(roots, return_inverse=True, return_counts=True)\n            conf_sums = np.bincount(inverse_indices, weights=particle_confidences)\n            \n            aggregated_confidences = conf_sums / (counts ** self.conf_coef)\n            cluster_per_particle = [4,1,2,9,4,8]\n            valid_clusters = (counts >= cluster_per_particle[idx]) & (aggregated_confidences > self.particle_confs[idx])\n\n            if not np.any(valid_clusters):\n                continue  \n\n            cluster_ids = unique_roots[valid_clusters]\n\n            centers_x = np.bincount(inverse_indices, weights=particle_xs) / counts\n            centers_y = np.bincount(inverse_indices, weights=particle_ys) / counts\n            centers_z = np.bincount(inverse_indices, weights=particle_zs) / counts\n\n            centers_x = centers_x[valid_clusters]\n            centers_y = centers_y[valid_clusters]\n            centers_z = centers_z[valid_clusters]\n\n            aggregated_df = pd.DataFrame({\n                'experiment': [run_id] * len(centers_x),\n                'particle_type': [particle] * len(centers_x),\n                'x': centers_x,\n                'y': centers_y,\n                'z': centers_z\n            })\n\n            aggregated_data.append(aggregated_df)\n\n        if aggregated_data:\n            return pd.concat(aggregated_data, axis=0)\n        else:\n            return pd.DataFrame()  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:11:36.624109Z","iopub.execute_input":"2025-01-25T12:11:36.624427Z","iopub.status.idle":"2025-01-25T12:11:36.644764Z","shell.execute_reply.started":"2025-01-25T12:11:36.624402Z","shell.execute_reply":"2025-01-25T12:11:36.643872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# instance main class\naggregator = PredictionAggregator(first_conf=0.19,  conf_coef=0.34) #Update\naggregated_results = []\n#add by @minfuka\nfrom concurrent.futures import ProcessPoolExecutor #add by @minfuka\n\n#add by @minfuka\ndef inference(runs, model, device_no):\n    subs = []\n    for r in tqdm(runs, total=len(runs)):\n        df = aggregator.make_predictions(r, model, device_no)\n        subs.append(df)\n    \n    return subs\nstart_time = time.time()\n\nwith ProcessPoolExecutor(max_workers=2) as executor:\n    results = list(executor.map(inference, (runs1, runs2), (model, model), (\"0\", \"1\")))\n\n\nend_time = time.time()\n\nestimated_total_time = (end_time - start_time) / len(runs) * 500  \nprint(f'estimated total prediction time for 500 runs: {estimated_total_time:.4f} seconds')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:11:56.994171Z","iopub.execute_input":"2025-01-25T12:11:56.995101Z","iopub.status.idle":"2025-01-25T12:12:24.344261Z","shell.execute_reply.started":"2025-01-25T12:11:56.995063Z","shell.execute_reply":"2025-01-25T12:12:24.343293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#change by @minfuka\nsubmission0 = pd.concat(results[0])\nsubmission1 = pd.concat(results[1])\nsubmission_ = pd.concat([submission0, submission1]).reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:13:02.551892Z","iopub.execute_input":"2025-01-25T12:13:02.552819Z","iopub.status.idle":"2025-01-25T12:13:02.559041Z","shell.execute_reply.started":"2025-01-25T12:13:02.552779Z","shell.execute_reply":"2025-01-25T12:13:02.558199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_.insert(0, 'id', range(len(submission_)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:13:13.929983Z","iopub.execute_input":"2025-01-25T12:13:13.930370Z","iopub.status.idle":"2025-01-25T12:13:13.939516Z","shell.execute_reply.started":"2025-01-25T12:13:13.930340Z","shell.execute_reply":"2025-01-25T12:13:13.938527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nchannels = (48, 64, 80, 80)\nstrides_pattern = (2, 2, 1)\nnum_res_units = 1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:15:24.082047Z","iopub.execute_input":"2025-01-25T12:15:24.082439Z","iopub.status.idle":"2025-01-25T12:15:24.086801Z","shell.execute_reply.started":"2025-01-25T12:15:24.082410Z","shell.execute_reply":"2025-01-25T12:15:24.085928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_3d_patches_minimal_overlap(arrays: List[np.ndarray], patch_size: int) -> Tuple[List[np.ndarray], List[Tuple[int, int, int]]]:\n    if not arrays or not isinstance(arrays, list):\n        raise ValueError(\"Input must be a non-empty list of arrays\")\n    \n    # Verify all arrays have the same shape\n    shape = arrays[0].shape\n    if not all(arr.shape == shape for arr in arrays):\n        raise ValueError(\"All input arrays must have the same shape\")\n    \n    if patch_size > min(shape):\n        raise ValueError(f\"patch_size ({patch_size}) must be smaller than smallest dimension {min(shape)}\")\n    \n    m, n, l = shape\n    patches = []\n    coordinates = []\n    \n    # Calculate starting positions for each dimension\n    x_starts = calculate_patch_starts(m, patch_size)\n    y_starts = calculate_patch_starts(n, patch_size)\n    z_starts = calculate_patch_starts(l, patch_size)\n    \n    # Extract patches from each array\n    for arr in arrays:\n        for x in x_starts:\n            for y in y_starts:\n                for z in z_starts:\n                    patch = arr[\n                        x:x + patch_size,\n                        y:y + patch_size,\n                        z:z + patch_size\n                    ]\n                    patches.append(patch)\n                    coordinates.append((x, y, z))\n    \n    return patches, coordinates\ndef reconstruct_array(patches: List[np.ndarray], \n                     coordinates: List[Tuple[int, int, int]], \n                     original_shape: Tuple[int, int, int]) -> np.ndarray:\n    reconstructed = np.zeros(original_shape, dtype=np.int64)  # To track overlapping regions\n    \n    patch_size = patches[0].shape[0]\n    \n    for patch, (x, y, z) in zip(patches, coordinates):\n        reconstructed[\n            x:x + patch_size,\n            y:y + patch_size,\n            z:z + patch_size\n        ] = patch\n        \n    \n    return reconstructed\ndef calculate_patch_starts(dimension_size: int, patch_size: int) -> List[int]:\n    if dimension_size <= patch_size:\n        return [0]\n        \n    # Calculate number of patches needed\n    n_patches = np.ceil(dimension_size / patch_size)\n    \n    if n_patches == 1:\n        return [0]\n    \n    # Calculate overlap\n    total_overlap = (n_patches * patch_size - dimension_size) / (n_patches - 1)\n    \n    # Generate starting positions\n    positions = []\n    for i in range(int(n_patches)):\n        pos = int(i * (patch_size - total_overlap))\n        if pos + patch_size > dimension_size:\n            pos = dimension_size - patch_size\n        if pos not in positions:  # Avoid duplicates\n            positions.append(pos)\n    \n    return positions\nimport pandas as pd\n\ndef dict_to_df(coord_dict, experiment_name):\n    # Create lists to store data\n    all_coords = []\n    all_labels = []\n    \n    # Process each label and its coordinates\n    for label, coords in coord_dict.items():\n        all_coords.append(coords)\n        all_labels.extend([label] * len(coords))\n    \n    # Concatenate all coordinates\n    all_coords = np.vstack(all_coords)\n    \n    df = pd.DataFrame({\n        'experiment': experiment_name,\n        'particle_type': all_labels,\n        'x': all_coords[:, 0],\n        'y': all_coords[:, 1],\n        'z': all_coords[:, 2]\n    })\n\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:15:26.550210Z","iopub.execute_input":"2025-01-25T12:15:26.551189Z","iopub.status.idle":"2025-01-25T12:15:26.564226Z","shell.execute_reply.started":"2025-01-25T12:15:26.551138Z","shell.execute_reply":"2025-01-25T12:15:26.563216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\nimport timm\n\n#from timm.models.convnext import *\n\n\"\"\"\n3D Image Segmentation Network Implementation\n-----------------------------------------\nThis module implements a 3D image segmentation network using PyTorch. The network\narchitecture combines a 2D encoder (various backbones supported) with a 3D decoder\nfor volumetric image segmentation tasks.\n\nKey Features:\n- Supports multiple backbone architectures (ResNet, ConvNext, EfficientNet, etc.)\n- Handles 3D data processing with efficient depth pooling\n- Implements skip connections for better feature preservation\n- Supports mixed precision training\n- Configurable for different input sizes and number of classes\n\"\"\"\n\n\n\n\"\"\"\nDecoder Implementation\n--------------------\nThis module implements both 2D and 3D UNet-style decoders for image segmentation.\nThe decoders use skip connections and progressive upsampling to recover spatial\nresolution while maintaining feature information.\n\"\"\"\n\nclass MyDecoderBlock(nn.Module):\n    \"\"\"\n    2D Decoder block implementing skip connections and progressive upsampling.\n    \n    Attributes:\n        conv1 (nn.Sequential): First convolution block with batch norm and ReLU\n        attention1 (nn.Module): Optional attention mechanism after skip connection\n        conv2 (nn.Sequential): Second convolution block with batch norm and ReLU\n        attention2 (nn.Module): Optional attention mechanism after second conv\n    \"\"\"\n    \n    def __init__(self, in_channel, skip_channel, out_channel):\n        \"\"\"\n        Initialize the decoder block.\n        \n        Args:\n            in_channel (int): Number of input channels\n            skip_channel (int): Number of channels in skip connection\n            out_channel (int): Number of output channels\n        \"\"\"\n        super().__init__()\n        self.conv1 = nn.Sequential(\n            nn.Conv2d(in_channel + skip_channel, out_channel, \n                     kernel_size=3, padding=1, bias=False),\n            nn.BatchNorm2d(out_channel),\n            nn.ReLU(inplace=True),\n        )\n        self.attention1 = nn.Identity()\n        self.conv2 = nn.Sequential(\n            nn.Conv2d(out_channel, out_channel, \n                     kernel_size=3, padding=1, bias=False),\n            nn.BatchNorm2d(out_channel),\n            nn.ReLU(inplace=True),\n        )\n        self.attention2 = nn.Identity()\n\n    def forward(self, x, skip=None):\n        \"\"\"\n        Forward pass of the decoder block.\n        \n        Args:\n            x (torch.Tensor): Input tensor\n            skip (torch.Tensor, optional): Skip connection tensor\n        \n        Returns:\n            torch.Tensor: Processed feature map\n        \"\"\"\n        x = F.interpolate(x, scale_factor=2, mode='nearest')\n        if skip is not None:\n            x = torch.cat([x, skip], dim=1)\n            x = self.attention1(x)\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.attention2(x)\n        return x\n\nclass MyUnetDecoder(nn.Module):\n    \"\"\"\n    2D UNet decoder implementation with skip connections.\n    \n    Attributes:\n        center (nn.Module): Optional center processing block\n        block (nn.ModuleList): List of decoder blocks\n    \"\"\"\n    \n    def __init__(self, in_channel, skip_channel, out_channel):\n        \"\"\"\n        Initialize the UNet decoder.\n        \n        Args:\n            in_channel (int): Number of input channels\n            skip_channel (list): List of skip connection channels\n            out_channel (list): List of output channels for each block\n        \"\"\"\n        super().__init__()\n        self.center = nn.Identity()\n\n        i_channel = [in_channel, ] + out_channel[:-1]\n        s_channel = skip_channel\n        o_channel = out_channel\n        block = [\n            MyDecoderBlock(i, s, o)\n            for i, s, o in zip(i_channel, s_channel, o_channel)\n        ]\n        self.block = nn.ModuleList(block)\n\n    def forward(self, feature, skip):\n        \"\"\"\n        Forward pass of the UNet decoder.\n        \n        Args:\n            feature (torch.Tensor): Input feature tensor\n            skip (list): List of skip connection tensors\n        \n        Returns:\n            tuple: (Final output tensor, List of intermediate decoder outputs)\n        \"\"\"\n        d = self.center(feature)\n        decode = []\n        for i, block in enumerate(self.block):\n            s = skip[i]\n            d = block(d, s)\n            decode.append(d)\n        last = d\n        return last, decode\n\nclass MyDecoderBlock3d(nn.Module):\n    \"\"\"\n    3D Decoder block implementing skip connections and progressive upsampling.\n    Extends the 2D decoder block concept to handle volumetric data.\n    \n    Attributes:\n        conv1 (nn.Sequential): First 3D convolution block with batch norm and ReLU\n        attention1 (nn.Module): Optional attention mechanism after skip connection\n        conv2 (nn.Sequential): Second 3D convolution block with batch norm and ReLU\n        attention2 (nn.Module): Optional attention mechanism after second conv\n    \"\"\"\n    \n    def __init__(self, in_channel, skip_channel, out_channel):\n        \"\"\"\n        Initialize the 3D decoder block.\n        \n        Args:\n            in_channel (int): Number of input channels\n            skip_channel (int): Number of channels in skip connection\n            out_channel (int): Number of output channels\n        \"\"\"\n        super().__init__()\n        self.conv1 = nn.Sequential(\n            nn.Conv3d(in_channel + skip_channel, out_channel, \n                     kernel_size=3, padding=1, bias=False),\n            nn.BatchNorm3d(out_channel),\n            nn.ReLU(inplace=True),\n        )\n        self.attention1 = nn.Identity()\n        self.conv2 = nn.Sequential(\n            nn.Conv3d(out_channel, out_channel, \n                     kernel_size=3, padding=1, bias=False),\n            nn.BatchNorm3d(out_channel),\n            nn.ReLU(inplace=True),\n        )\n        self.attention2 = nn.Identity()\n\n    def forward(self, x, skip=None, depth_scaling=2):\n        \"\"\"\n        Forward pass of the 3D decoder block.\n        \n        Args:\n            x (torch.Tensor): Input tensor\n            skip (torch.Tensor, optional): Skip connection tensor\n            depth_scaling (int): Scaling factor for depth dimension upsampling\n        \n        Returns:\n            torch.Tensor: Processed feature volume\n        \"\"\"\n        # Upsample with different scaling for depth dimension\n        x = F.interpolate(x, scale_factor=(depth_scaling,2,2), mode='nearest')\n        if skip is not None:\n            x = torch.cat([x, skip], dim=1)\n            x = self.attention1(x)\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.attention2(x)\n        return x\n\nclass MyUnetDecoder3d(nn.Module):\n    \"\"\"\n    3D UNet decoder implementation with skip connections.\n    Handles volumetric data with separate scaling for depth dimension.\n    \n    Attributes:\n        center (nn.Module): Optional center processing block\n        block (nn.ModuleList): List of 3D decoder blocks\n    \"\"\"\n    \n    def __init__(self, in_channel, skip_channel, out_channel):\n        \"\"\"\n        Initialize the 3D UNet decoder.\n        \n        Args:\n            in_channel (int): Number of input channels\n            skip_channel (list): List of skip connection channels\n            out_channel (list): List of output channels for each block\n        \"\"\"\n        super().__init__()\n        self.center = nn.Identity()\n\n        i_channel = [in_channel, ] + out_channel[:-1]\n        s_channel = skip_channel\n        o_channel = out_channel\n        block = [\n            MyDecoderBlock3d(i, s, o)\n            for i, s, o in zip(i_channel, s_channel, o_channel)\n        ]\n        self.block = nn.ModuleList(block)\n\n    def forward(self, feature, skip, depth_scaling=[2,2,2,2,2,2]):\n        \"\"\"\n        Forward pass of the 3D UNet decoder.\n        \n        Args:\n            feature (torch.Tensor): Input feature tensor\n            skip (list): List of skip connection tensors\n            depth_scaling (list): List of depth scaling factors for each block\n        \n        Returns:\n            tuple: (Final output tensor, List of intermediate decoder outputs)\n        \"\"\"\n        d = self.center(feature)\n        decode = []\n        for i, block in enumerate(self.block):\n            s = skip[i]\n            d = block(d, s, depth_scaling[i])\n            decode.append(d)\n        last = d\n        return last, decode\n        \n\n#------------------------------------------------\n# processing\n\ndef encode_for_resnet(e, x, B, depth_scaling=[2,2,2,2,1]):\n\n    def pool_in_depth(x, depth_scaling):\n        bd, c, h, w = x.shape\n        x1 = x.reshape(B, -1, c, h, w).permute(0, 2, 1, 3, 4)\n        x1 = F.avg_pool3d(x1, kernel_size=(depth_scaling, 1, 1), stride=(depth_scaling, 1, 1), padding=0)\n        x = x1.permute(0, 2, 1, 3, 4).reshape(-1, c, h, w)\n        return x, x1\n\n    encode=[]\n    x = e.conv1(x)\n    x = e.bn1(x)\n    x = e.act1(x)\n    x, x1 = pool_in_depth(x, depth_scaling[0])\n    encode.append(x1)\n    #print(x.shape)\n    #x = e.maxpool(x)\n    x = F.avg_pool2d(x,kernel_size=2,stride=2)\n\n    x = e.layer1(x)\n    x, x1 = pool_in_depth(x, depth_scaling[1])\n    encode.append(x1)\n    #print(x.shape)\n\n    x = e.layer2(x)\n    x, x1 = pool_in_depth(x, depth_scaling[2])\n    encode.append(x1)\n    #print(x.shape)\n\n    x = e.layer3(x)\n    x, x1 = pool_in_depth(x, depth_scaling[3])\n    encode.append(x1)\n    #print(x.shape)\n\n    x = e.layer4(x)\n    x, x1 = pool_in_depth(x, depth_scaling[4])\n    encode.append(x1)\n    #print(x.shape)\n\n    return encode\n\n\nclass Net(nn.Module):\n    def __init__(self, pretrained=False, cfg=None):\n        super(Net, self).__init__()\n        self.output_type = ['infer', 'loss', ]\n        self.register_buffer('D', torch.tensor(0))\n\n        num_class=6+1\n\n        self.arch = 'resnet34d'\n        if cfg is not None:\n            self.arch = cfg.arch\n\n        encoder_dim = {\n            'resnet18': [64, 64, 128, 256, 512, ],\n            'resnet18d': [64, 64, 128, 256, 512, ],\n            'resnet34d': [64, 64, 128, 256, 512, ],\n            'resnet50d': [64, 256, 512, 1024, 2048, ],\n            'seresnext26d_32x4d': [64, 256, 512, 1024, 2048, ],\n            'convnext_small.fb_in22k': [96, 192, 384, 768],\n            'convnext_tiny.fb_in22k': [96, 192, 384, 768],\n            'convnext_base.fb_in22k': [128, 256, 512, 1024],\n            'tf_efficientnet_b4.ns_jft_in1k':[32, 56, 160, 448],\n            'tf_efficientnet_b5.ns_jft_in1k':[40, 64, 176, 512],\n            'tf_efficientnet_b6.ns_jft_in1k':[40, 72, 200, 576],\n            'tf_efficientnet_b7.ns_jft_in1k':[48, 80, 224, 640],\n            'pvt_v2_b1': [64, 128, 320, 512],\n            'pvt_v2_b2': [64, 128, 320, 512],\n            'pvt_v2_b4': [64, 128, 320, 512],\n        }.get(self.arch, [768])\n        decoder_dim = \\\n              [256, 128, 64, 32, 16]\n\n        self.encoder = timm.create_model(\n            model_name=self.arch, pretrained=pretrained, in_chans=3, num_classes=0, global_pool='', features_only=True,\n        )\n        self.decoder = MyUnetDecoder3d(\n            in_channel=encoder_dim[-1],\n            skip_channel=encoder_dim[:-1][::-1]+[0],\n            out_channel=decoder_dim,\n        )\n        self.mask = nn.Conv3d(decoder_dim[-1],num_class, kernel_size=1)\n\n    def forward(self, batch):\n        device = self.D.device\n\n        #image = batch['image'].to(device)\n        B, C, D, H, W = batch.shape\n        #batch = batch[:,0,:]\n        image = batch.reshape(B*D, 1, H, W)\n\n        x = (image.float() - 0.5) / 0.5\n        x = x.expand(-1, 3, -1, -1)\n\n        #encode = self.encoder(x)[-5:]\n        encode = encode_for_resnet(self.encoder, x, B, depth_scaling=[2,2,2,2,1])\n        #[print(f'encode_{i}', e.shape) for i,e in enumerate(encode)]\n\n        #[print(f'encode_{i}', e.shape) for i, e in enumerate(encode)]\n        last, decode = self.decoder(\n            feature=encode[-1], skip=encode[:-1][::-1]+[None], depth_scaling=[1,2,2,2,2]\n        )\n        #print(f'last', last.shape)\n\n        logit = self.mask(last)\n        #print('logit', logit.shape)\n\n        # output = {}\n        # if 'loss' in self.output_type:\n        #     #<todo> weighted cross entropy\n        #     output['mask_loss'] = F.cross_entropy(logit, batch['mask'].to(device))\n\n        # if 'infer' in self.output_type:\n        #     output['particle'] = F.softmax(logit,1)\n\n        output = logit\n        return output\n\n\n\n\ndevice = torch.device(\"cuda:0\")\nnet = Net(pretrained=False, cfg=None).to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:15:30.341248Z","iopub.execute_input":"2025-01-25T12:15:30.341583Z","iopub.status.idle":"2025-01-25T12:15:31.011737Z","shell.execute_reply.started":"2025-01-25T12:15:30.341555Z","shell.execute_reply":"2025-01-25T12:15:31.011042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"root_dir = '/kaggle/input/2d3d_jan18_unet/pytorch/default/1/'\nnet = torch.nn.DataParallel(net)\nnet.load_state_dict(torch.load(os.path.join(root_dir, \"2d3d_jan18_best_metric_model.pth\")))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:15:39.071418Z","iopub.execute_input":"2025-01-25T12:15:39.071768Z","iopub.status.idle":"2025-01-25T12:15:40.915310Z","shell.execute_reply.started":"2025-01-25T12:15:39.071738Z","shell.execute_reply":"2025-01-25T12:15:40.914223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#from monai.networks.nets import UNet\n# from monai.networks.layers import Norm\n# device = torch.device(\"cuda:0\")\n\n# root_dir = '/kaggle/input/myunet_jan17b/pytorch/default/1/'\n\n# net = UNet(\n#     spatial_dims=3,\n#     in_channels=1,\n#     out_channels=7,  \n#     #channels=(16, 32, 64, 128, 256),  \n#     #strides=(2, 2, 2, 2),  \n#     #num_res_units=2, \n#     #dropout=0.1, \n#     channels=(48, 64, 80, 80),  \n#     strides=(2, 2, 1),  \n#     num_res_units=2, \n#     dropout=0.3, \n#     norm=Norm.BATCH,\n# )\n# #net = torch.nn.DataParallel(net)\n# net.load_state_dict(torch.load(os.path.join(root_dir, \"jan17_best_metric_model2.pth\")))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-18T10:03:44.146707Z","iopub.execute_input":"2025-01-18T10:03:44.147301Z","iopub.status.idle":"2025-01-18T10:04:14.420829Z","shell.execute_reply.started":"2025-01-18T10:03:44.147267Z","shell.execute_reply":"2025-01-18T10:04:14.419975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#from monai.networks.nets import UNet\n# from monai.networks.layers import Norm\n# device = torch.device(\"cuda:0\")\n# root_dir = '/kaggle/input/myunet9/pytorch/default/1/'\n# net = UNet(\n#     spatial_dims=3,\n#     in_channels=1,\n#     out_channels=7,  \n#     #channels=(16, 32, 64, 128, 256),  \n#     #strides=(2, 2, 2, 2),  \n#     #num_res_units=2, \n#     #dropout=0.1, \n#     channels=(48, 64, 80, 80),  \n#     strides=(2, 2, 1),  \n#     num_res_units=2, \n#     dropout=0.3, \n#     norm=Norm.BATCH,\n# ).to(device)\n# net.load_state_dict(torch.load(os.path.join(root_dir, \"dec23_best_metric_model_scale.pth\")))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T17:21:12.482120Z","iopub.execute_input":"2025-01-17T17:21:12.482953Z","iopub.status.idle":"2025-01-17T17:21:12.728858Z","shell.execute_reply.started":"2025-01-17T17:21:12.482918Z","shell.execute_reply":"2025-01-17T17:21:12.728094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from typing import List, Tuple, Union\nimport numpy as np\nimport torch\nfrom monai.data import DataLoader, Dataset, CacheDataset, decollate_batch\nfrom monai.transforms import (\n    Compose, \n    EnsureChannelFirstd, \n    Orientationd,  \n    AsDiscrete,  \n    RandFlipd, \n    RandRotate90d, \n    NormalizeIntensityd,\n    RandCropByLabelClassesd,\n    ScaleIntensityd,\n)\nTRAIN_DATA_DIR = \"/kaggle/input/create-numpy-dataset-exp-name\"\n#net = torch.load('//kaggle/input/ana_unet1/pytorch/default/1/best_metric_model.pth')\nnet.to('cuda')\nimport json\ncopick_config_path = TRAIN_DATA_DIR + \"/copick.config\"\n\nwith open(copick_config_path) as f:\n    copick_config = json.load(f)\n\ncopick_config['static_root'] = '/kaggle/input/czii-cryo-et-object-identification/test/static'\n\ncopick_test_config_path = 'copick_test.config'\n\nwith open(copick_test_config_path, 'w') as outfile:\n    json.dump(copick_config, outfile)\nimport copick\n\nroot = copick.from_file(copick_test_config_path)\n\ncopick_user_name = \"copickUtils\"\ncopick_segmentation_name = \"paintedPicks\"\nvoxel_size = 10\ntomo_type = \"denoised\"\ninference_transforms = Compose([\n    EnsureChannelFirstd(keys=[\"image\"], channel_dim=\"no_channel\"),\n    Orientationd(keys=[\"image\"], axcodes=\"RAS\"),\n    #NormalizeIntensityd(keys=\"image\"),\n    ScaleIntensityd(keys=\"image\", minv=0, maxv=1.0),\n])\nimport cc3d\n\nid_to_name = {1: \"apo-ferritin\", \n              2: \"beta-amylase\",\n              3: \"beta-galactosidase\", \n              4: \"ribosome\", \n              5: \"thyroglobulin\", \n              6: \"virus-like-particle\"}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:15:55.321607Z","iopub.execute_input":"2025-01-25T12:15:55.321968Z","iopub.status.idle":"2025-01-25T12:16:26.780465Z","shell.execute_reply.started":"2025-01-25T12:15:55.321937Z","shell.execute_reply":"2025-01-25T12:16:26.779530Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def ensemble_prediction_tta(model_, input_tensor, threshold=0.5):\n    # TTA\n    data_copy0 = torch.flip(input_tensor.clone(), dims=[2])\n    data_copy1 = torch.flip(input_tensor.clone(), dims=[3])\n    data_copy2 = torch.flip(input_tensor.clone(), dims=[4])\n    data_copy3 = input_tensor.clone().rot90(1, dims=[3, 4])\n    \n    with torch.no_grad():\n        model_output0 = model_(input_tensor)\n        model_output1 = model_(data_copy0)\n        model_output1 = torch.flip(model_output1, dims=[2])\n        model_output2 = model_(data_copy1)\n        model_output2 = torch.flip(model_output2, dims=[3])\n        model_output3 = model_(data_copy2)\n        model_output3 = torch.flip(model_output3, dims=[4])\n        \n        probs0 = torch.softmax(model_output0[0], dim=0)\n        probs1 = torch.softmax(model_output1[0], dim=0)\n        probs2 = torch.softmax(model_output2[0], dim=0)\n        probs3 = torch.softmax(model_output3[0], dim=0)\n        \n        avg_probs = torch.mean(torch.stack([probs0, probs1, probs2, probs3]), dim=0)\n        thresh_probs = avg_probs > threshold\n        _, max_classes = thresh_probs.max(dim=0)\n    return max_classes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:16:45.922173Z","iopub.execute_input":"2025-01-25T12:16:45.922853Z","iopub.status.idle":"2025-01-25T12:16:45.929661Z","shell.execute_reply.started":"2025-01-25T12:16:45.922816Z","shell.execute_reply":"2025-01-25T12:16:45.928715Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BLOB_THRESHOLD = 10 #200\nCERTAINTY_THRESHOLD = 0.15\n\nclasses = [1, 2, 3, 4, 5, 6]\n\ndef simple_scale(input_image):\n    input_image -= np.min(input_image)\n    input_image = input_image / np.max(input_image)\n    return input_image\n    \nwith torch.no_grad():\n    location_df = []\n    for run in root.runs:\n        print(run)\n\n        tomo = run.get_voxel_spacing(10)\n        tomo = tomo.get_tomogram(tomo_type).numpy()\n\n        tomo = simple_scale(tomo)\n\n        tomo_patches, coordinates  = extract_3d_patches_minimal_overlap([tomo], 96)\n\n        tomo_patched_data = [{\"image\": img} for img in tomo_patches]\n\n        tomo_ds = CacheDataset(data=tomo_patched_data, transform=inference_transforms, cache_rate=1.0)\n\n        pred_masks = []\n\n        for i in range(len(tomo_ds)):\n            input_tensor = tomo_ds[i]['image'].unsqueeze(0).to(\"cuda\")\n            \n            #model_output = net(input_tensor)\n            #probs = torch.softmax(model_output[0], dim=0)\n            #thresh_probs = probs > CERTAINTY_THRESHOLD\n            #_, max_classes = thresh_probs.max(dim=0)\n\n            max_classes = ensemble_prediction_tta(net, input_tensor, threshold=CERTAINTY_THRESHOLD)\n            pred_masks.append(max_classes.cpu().numpy())\n            \n\n        reconstructed_mask = reconstruct_array(pred_masks, coordinates, tomo.shape)\n        \n        location = {}\n\n        for c in classes:\n            cc = cc3d.connected_components(reconstructed_mask == c)\n            stats = cc3d.statistics(cc)\n            zyx=stats['centroids'][1:]*10.012444 #https://www.kaggle.com/competitions/czii-cryo-et-object-identification/discussion/544895#3040071\n            zyx_large = zyx[stats['voxel_counts'][1:] > BLOB_THRESHOLD]\n            xyz =np.ascontiguousarray(zyx_large[:,::-1])\n\n            location[id_to_name[c]] = xyz\n\n\n        df = dict_to_df(location, run.name)\n        location_df.append(df)\n    \n    location_df = pd.concat(location_df)\nlocation_df.insert(loc=0, column='id', value=np.arange(len(location_df)))\nlocation_df.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:16:53.067565Z","iopub.execute_input":"2025-01-25T12:16:53.068468Z","iopub.status.idle":"2025-01-25T12:18:36.289964Z","shell.execute_reply.started":"2025-01-25T12:16:53.068431Z","shell.execute_reply":"2025-01-25T12:18:36.289032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.cluster import DBSCAN\n\n\ndf = pd.concat([submission_,location_df], ignore_index=True)\n\nparticle_names = ['apo-ferritin', 'beta-amylase', 'beta-galactosidase', 'ribosome', 'thyroglobulin', 'virus-like-particle']\nparticle_radius = {\n    'apo-ferritin': 60,\n    'beta-amylase': 65,\n    'beta-galactosidase': 90,\n    'ribosome': 150,\n    'thyroglobulin': 130,\n    'virus-like-particle': 135,\n}\n\nfinal = []\nfor pidx, p in enumerate(particle_names):\n    pdf = df[df['particle_type'] == p].reset_index(drop=True)\n    p_rad = particle_radius[p]\n    \n    grouped = pdf.groupby(['experiment'])\n    \n    for exp, group in grouped:\n        group = group.reset_index(drop=True)\n        \n        coords = group[['x', 'y', 'z']].values\n        db = DBSCAN(eps=p_rad, min_samples=2, metric='euclidean').fit(coords)\n        labels = db.labels_\n        \n        group['cluster'] = labels\n        \n        for cluster_id in np.unique(labels):\n            if cluster_id == -1:\n                continue\n            \n            cluster_points = group[group['cluster'] == cluster_id]\n            \n            avg_x = cluster_points['x'].mean()\n            avg_y = cluster_points['y'].mean()\n            avg_z = cluster_points['z'].mean()\n            \n            group.loc[group['cluster'] == cluster_id, ['x', 'y', 'z']] = avg_x, avg_y, avg_z\n            group = group.drop_duplicates(subset=['x', 'y', 'z'])\n        final.append(group)\n\ndf_save = pd.concat(final, ignore_index=True)\ndf_save = df_save.drop(columns=['cluster'])\ndf_save = df_save.sort_values(by=['experiment', 'particle_type']).reset_index(drop=True)\ndf_save['id'] = np.arange(0, len(df_save))\ndf_save.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:19:31.923617Z","iopub.execute_input":"2025-01-25T12:19:31.924272Z","iopub.status.idle":"2025-01-25T12:19:33.214087Z","shell.execute_reply.started":"2025-01-25T12:19:31.924239Z","shell.execute_reply":"2025-01-25T12:19:33.213330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#from monai.networks.nets import UNet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T13:33:32.663269Z","iopub.execute_input":"2024-12-03T13:33:32.663986Z","iopub.status.idle":"2024-12-03T13:33:32.668047Z","shell.execute_reply.started":"2024-12-03T13:33:32.663952Z","shell.execute_reply":"2024-12-03T13:33:32.667166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# net2 = UNet(spatial_dims=3,\n#     in_channels=1,\n#     out_channels=7,\n#     channels=channels,\n#     strides=strides_pattern,\n#     num_res_units=num_res_units)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T13:36:00.850726Z","iopub.execute_input":"2024-12-03T13:36:00.851469Z","iopub.status.idle":"2024-12-03T13:36:00.883615Z","shell.execute_reply.started":"2024-12-03T13:36:00.851436Z","shell.execute_reply":"2024-12-03T13:36:00.882744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#df_save","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:20:01.201459Z","iopub.execute_input":"2025-01-25T12:20:01.201847Z","iopub.status.idle":"2025-01-25T12:20:01.218562Z","shell.execute_reply.started":"2025-01-25T12:20:01.201797Z","shell.execute_reply":"2025-01-25T12:20:01.217685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# DATA_KAGGLE_DIR = \"/kaggle/input/czii-cryo-et-object-identification\"\n# import glob\n\n# MODE='local'\n\n# if MODE=='local':\n#     valid_dir =f'{DATA_KAGGLE_DIR}/train'\n#     valid_id = ['TS_69_2',  ] #'TS_6_6'\n    \n# if MODE=='submit':\n#     valid_dir =f'{DATA_KAGGLE_DIR}/test' \n#     valid_id = glob.glob(f'{valid_dir}/static/ExperimentRuns/*')\n#     print(valid_id)\n#     valid_id = [f.split('/')[-1] for f in valid_id]\n    \n\n# #test_experiment_dir = \"/kaggle/input/czii-cryo-et-object-identification/test/static/ExperimentRuns/\"\n# #exps = os.listdir(test_experiment_dir)\n\n\n# print('valid_id:',len(valid_id), valid_id)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:20:09.443251Z","iopub.execute_input":"2025-01-25T12:20:09.443598Z","iopub.status.idle":"2025-01-25T12:20:09.449670Z","shell.execute_reply.started":"2025-01-25T12:20:09.443570Z","shell.execute_reply":"2025-01-25T12:20:09.448761Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from scipy.optimize import linear_sum_assignment\n# from matplotlib import pyplot as plt\n\n# #valid_dir = '/kaggle/input/czii-cryo-et-object-identification/train'\n# #MODE='local'\n\n# def do_one_eval(truth, predict, threshold):\n#     P=len(predict)\n#     T=len(truth)\n\n#     if P==0:\n#         hit=[[],[]]\n#         miss=np.arange(T).tolist()\n#         fp=[]\n#         metric = [P,T,len(hit[0]),len(miss),len(fp)]\n#         return hit, fp, miss, metric\n\n#     if T==0:\n#         hit=[[],[]]\n#         fp=np.arange(P).tolist()\n#         miss=[]\n#         metric = [P,T,len(hit[0]),len(miss),len(fp)]\n#         return hit, fp, miss, metric\n\n#     #---\n#     distance = predict.reshape(P,1,3)-truth.reshape(1,T,3)\n#     distance = distance**2\n#     distance = distance.sum(axis=2)\n#     distance = np.sqrt(distance)\n#     p_index, t_index = linear_sum_assignment(distance)\n\n#     valid = distance[p_index, t_index] <= threshold\n#     p_index = p_index[valid]\n#     t_index = t_index[valid]\n#     hit = [p_index.tolist(), t_index.tolist()]\n#     miss = np.arange(T)\n#     miss = miss[~np.isin(miss,t_index)].tolist()\n#     fp = np.arange(P)\n#     fp = fp[~np.isin(fp,p_index)].tolist()\n\n#     metric = [P,T,len(hit[0]),len(miss),len(fp)] #for lb metric F-beta copmutation\n#     return hit, fp, miss, metric\n\n\n# def compute_lb(submit_df, overlay_dir):\n#     valid_id = list(submit_df['experiment'].unique())\n#     print(f\"Computing score for experiment: {valid_id}\")\n\n#     eval_df = []\n#     for id in valid_id:\n#         truth = read_one_truth(id, overlay_dir) #=f'{valid_dir}/overlay/ExperimentRuns')\n#         id_df = submit_df[submit_df['experiment'] == id]\n#         for p in PARTICLE:\n#             p = dotdict(p)\n#             print('\\r', id, p.name, end='', flush=True)\n#             xyz_truth = truth[p.name]\n#             xyz_predict = id_df[id_df['particle_type'] == p.name][['x', 'y', 'z']].values\n#             hit, fp, miss, metric = do_one_eval(xyz_truth, xyz_predict, p.radius* 0.5)\n#             eval_df.append(dotdict(\n#                 id=id, particle_type=p.name,\n#                 P=metric[0], T=metric[1], hit=metric[2], miss=metric[3], fp=metric[4],\n#             ))\n#     print('')\n#     eval_df = pd.DataFrame(eval_df)\n#     gb = eval_df.groupby('particle_type').agg('sum').drop(columns=['id'])\n#     gb.loc[:, 'precision'] = gb['hit'] / gb['P']\n#     gb.loc[:, 'precision'] = gb['precision'].fillna(0)\n#     gb.loc[:, 'recall'] = gb['hit'] / gb['T']\n#     gb.loc[:, 'recall'] = gb['recall'].fillna(0)\n#     gb.loc[:, 'f-beta4'] = 17 * gb['precision'] * gb['recall'] / (16 * gb['precision'] + gb['recall'])\n#     gb.loc[:, 'f-beta4'] = gb['f-beta4'].fillna(0)\n\n#     gb = gb.sort_values('particle_type').reset_index(drop=False)\n#     # https://www.kaggle.com/competitions/czii-cryo-et-object-identification/discussion/544895\n#     gb.loc[:, 'weight'] = [1, 0, 2, 1, 2, 1]\n#     lb_score = (gb['f-beta4'] * gb['weight']).sum() / gb['weight'].sum()\n#     return gb, lb_score\n\n\n# #debug\n# if 1:\n#     if MODE=='local':\n#     #if 1:\n#         submit_df=pd.read_csv(\n#            'submission.csv'\n#             # '/kaggle/input/hengck-czii-cryo-et-weights-01/submission.csv'\n#         )\n#         gb, lb_score = compute_lb(submit_df, f'{valid_dir}/overlay/ExperimentRuns')\n#         print(gb)\n#         print('lb_score:',lb_score)\n#         print('')\n\n\n#         #show one ----------------------------------\n#         fig = plt.figure(figsize=(18, 8))\n\n#         id = valid_id[0]\n#         truth = read_one_truth(id,overlay_dir=f'{valid_dir}/overlay/ExperimentRuns')\n\n#         submit_df = submit_df[submit_df['experiment']==id]\n#         for p in PARTICLE:\n#             p = dotdict(p)\n#             xyz_truth = truth[p.name]\n#             xyz_predict = submit_df[submit_df['particle_type']==p.name][['x','y','z']].values\n#             hit, fp, miss, _ = do_one_eval(xyz_truth, xyz_predict, p.radius)\n#             print(id, p.name)\n#             print('\\t num truth   :',len(xyz_truth) )\n#             print('\\t num predict :',len(xyz_predict) )\n#             print('\\t num hit  :',len(hit[0]) )\n#             print('\\t num fp   :',len(fp) )\n#             print('\\t num miss :',len(miss) )\n\n#             ax = fig.add_subplot(2, 3, p.label, projection='3d')\n#             if hit[0]:\n#                 pt = xyz_predict[hit[0]]\n#                 ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2], alpha=0.5, color='r')\n#                 pt = xyz_truth[hit[1]]\n#                 ax.scatter(pt[:,0], pt[:,1], pt[:,2], s=80, facecolors='none', edgecolors='r')\n#             if fp:\n#                 pt = xyz_predict[fp]\n#                 ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2], alpha=1, color='k')\n#             if miss:\n#                 pt = xyz_truth[miss]\n#                 ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2], s=160, alpha=1, facecolors='none', edgecolors='k')\n\n#             ax.set_title(f'{p.name} ({p.difficulty})')\n\n#         plt.tight_layout()\n#         plt.show()\n        \n#         #--- \n#         zz=0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T12:20:11.550303Z","iopub.execute_input":"2025-01-25T12:20:11.550621Z","iopub.status.idle":"2025-01-25T12:20:12.775168Z","shell.execute_reply.started":"2025-01-25T12:20:11.550595Z","shell.execute_reply":"2025-01-25T12:20:12.774315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}