{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"},{"sourceId":10413823,"sourceType":"datasetVersion","datasetId":6351434},{"sourceId":10476214,"sourceType":"datasetVersion","datasetId":6486959},{"sourceId":10505091,"sourceType":"datasetVersion","datasetId":6488425},{"sourceId":10540638,"sourceType":"datasetVersion","datasetId":6497627},{"sourceId":10599554,"sourceType":"datasetVersion","datasetId":6405875},{"sourceId":10655272,"sourceType":"datasetVersion","datasetId":6347637},{"sourceId":10656868,"sourceType":"datasetVersion","datasetId":6523649},{"sourceId":10657117,"sourceType":"datasetVersion","datasetId":6523674},{"sourceId":214223778,"sourceType":"kernelVersion"},{"sourceId":220058047,"sourceType":"kernelVersion"},{"sourceId":220196471,"sourceType":"kernelVersion"},{"sourceId":220206210,"sourceType":"kernelVersion"}],"dockerImageVersionId":30823,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install /kaggle/input/tensorrt-10-1-0/nvidia_cuda_runtime_cu12-12.2.140-py3-none-manylinux1_x86_64.whl\n!pip install /kaggle/input/tensorrt-10-1-0/tensorrt_cu12_bindings-10.1.0-cp310-none-manylinux_2_17_x86_64.whl\n!pip install /kaggle/input/tensorrt-10-1-0/tensorrt_cu12_libs-10.1.0-py2.py3-none-manylinux_2_17_x86_64.whl\n!pip install /kaggle/input/tensorrt-10-1-0/tensorrt_cu12-10.1.0-py2.py3-none-any.whl\n!pip install /kaggle/input/tensorrt-10-1-0/tensorrt-10.1.0-py2.py3-none-any.whl\n!pip install /kaggle/input/tensorrt-10-1-0/polygraphy-0.49.14-py2.py3-none-any.whl\n!cp -r /kaggle/input/tensorrt-10-1-0/torch2trt-master /kaggle/working/torch2trt\n!pip install /kaggle/working/torch2trt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:37:58.557015Z","iopub.execute_input":"2025-02-02T15:37:58.557315Z","iopub.status.idle":"2025-02-02T15:38:59.596013Z","shell.execute_reply.started":"2025-02-02T15:37:58.557292Z","shell.execute_reply":"2025-02-02T15:38:59.594981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nsys.path.append(\"/kaggle/input/czii-src/kaggle-czii-main\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:38:59.59736Z","iopub.execute_input":"2025-02-02T15:38:59.597591Z","iopub.status.idle":"2025-02-02T15:38:59.601312Z","shell.execute_reply.started":"2025-02-02T15:38:59.597572Z","shell.execute_reply":"2025-02-02T15:38:59.600588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"! pip install -q zarr segmentation_models_pytorch omegaconf --no-index --find-links=/kaggle/input/czii-libs\n! pip install -q -U timm --no-index --find-links=/kaggle/input/czii-libs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:38:59.603382Z","iopub.execute_input":"2025-02-02T15:38:59.60361Z","iopub.status.idle":"2025-02-02T15:39:13.521479Z","shell.execute_reply.started":"2025-02-02T15:38:59.603591Z","shell.execute_reply":"2025-02-02T15:39:13.520658Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nos.chdir(\"/kaggle/working/\")\n! cp -r /kaggle/input/czii-cryo-et-object-identification-main/czii-cryo-et-object-identification-main .\nos.chdir(\"/kaggle/input/czii-cryo-et-object-identification-main/czii-cryo-et-object-identification-main/src/exp122\")\n! ls \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:39:13.52278Z","iopub.execute_input":"2025-02-02T15:39:13.523006Z","iopub.status.idle":"2025-02-02T15:39:14.314612Z","shell.execute_reply.started":"2025-02-02T15:39:13.522988Z","shell.execute_reply":"2025-02-02T15:39:14.313807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from evaluation import CZIIInferenceDataset\nfrom train import (\n    EXP_ID,\n    CZIILightningModel,\n    fbeta_comp_fn,\n    # find_local_maxima,\n    find_threshold_fixed,\n    find_threshold_percentile,\n    get_patch_weight,\n    get_transform,\n    particles_fbeta_one_class,\n    radius,\n    weights,\n)\nimport json\nimport torch\nfrom glob import glob\nfrom tqdm.auto import tqdm\nimport zarr\nimport numpy as np\nfrom torch.utils.data import DataLoader\nfrom scipy.ndimage import maximum_filter\nimport pandas as pd\nimport gc\nfrom scipy.optimize import minimize\nfrom functools import partial\nfrom torch.nn import functional as F\n\nfrom threading import Thread\nfrom collections import defaultdict\n\nimport os\nimport torch\nimport multiprocessing as mp\nimport zarr\nfrom tqdm import tqdm\nfrom glob import glob\nfrom torch2trt import TRTModule\nimport tensorrt as trt\nfrom torch.utils.data import DataLoader\nfrom cuda import cudart\n\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\n# from tqdm.notebook import tqdm\nimport zarr\nimport cv2\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport segmentation_models_pytorch as smp\nfrom omegaconf import OmegaConf\nfrom scipy.ndimage import maximum_filter\n\nfrom src.model import get_model_from_cfg, EnsembleModel\nfrom src.pl_module import get_patch_weight\nfrom src.metric import experiments, score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:39:14.31562Z","iopub.execute_input":"2025-02-02T15:39:14.315965Z","iopub.status.idle":"2025-02-02T15:39:56.55271Z","shell.execute_reply.started":"2025-02-02T15:39:14.315943Z","shell.execute_reply":"2025-02-02T15:39:56.551993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, cfg, x):\n        assert mode in [\"train\", \"val\", \"test\"]\n        self.cfg = cfg\n        self.x = x  # 184, 630, 630\n        self.mode = mode\n        self.indices = self.get_indices()\n\n    \"\"\"\n    @staticmethod\n    def get_stride(w, h, d, w_p, h_p, d_p, stride_scale=0.5):\n        w_s = int(w_p * stride_scale)\n        h_s = int(h_p * stride_scale)\n        d_s = int(d_p * stride_scale)\n        t_x = (w - w_p) // w_s + 1\n        t_y = (h - h_p) // h_s + 1\n        t_z = (d - d_p) // d_s + 1\n        return w_s, h_s, d_s, t_x, t_y, t_z\n    \"\"\"\n    @staticmethod\n    def get_stride(img_w, img_h, img_d, tile_size_x, tile_size_y, tile_size_z, stride_scale=0.5):\n        tmp_stride_x = tile_size_x * stride_scale\n        tmp_stride_y = tile_size_y * stride_scale\n        tmp_stride_z = tile_size_z * stride_scale\n        tile_num_x = 12 # max(round((img_w - tile_size_x) / tmp_stride_x + 1), 1)\n        tile_num_y = 12 # max(round((img_h - tile_size_y) / tmp_stride_y + 1), 1)\n        tile_num_z = max(round((img_d - tile_size_z) / tmp_stride_z + 1), 1)\n        stride_x = (img_w - tile_size_x) // (tile_num_x - 1) if tile_num_x - 1 > 0 else 0\n        stride_y = (img_h - tile_size_y) // (tile_num_y - 1) if tile_num_y - 1 > 0 else 0\n        stride_z = (img_d - tile_size_z) // (tile_num_z - 1) if tile_num_z - 1 > 0 else 0\n        return stride_x, stride_y, stride_z, tile_num_x, tile_num_y, tile_num_z\n\n    def get_indices(self):\n        img_size = self.cfg.model.img_size\n        img_depth = self.cfg.model.img_depth\n        tile_size_x = img_size\n        tile_size_y = img_size\n        tile_size_z = img_depth\n        stride_scale = 0.5 # if self.mode != \"test\" else 0.25\n        indices = []\n\n        img_d, img_h, img_w = self.x.shape\n        s_x, s_y, s_z, tile_x, tile_y, tile_z = self.get_stride(img_w, img_h, img_d, tile_size_x, tile_size_y,\n                                                                tile_size_z, stride_scale=stride_scale)\n        for iz in range(tile_z):\n            for iy in range(tile_y):\n                for ix in range(tile_x):\n                    sx = ix * s_x\n                    sy = iy * s_y\n                    sz = iz * s_z\n                    indices.append((sx, sy, sz))\n        return np.array(indices)\n\n    def __len__(self):\n        return len(self.indices)\n\n    def __getitem__(self, idx):\n        img_size = self.cfg.model.img_size\n        img_depth = self.cfg.model.img_depth\n        sx, sy, sz = self.indices[idx]\n        x = self.x  # d, h, w\n        x = x[sz:sz + img_depth, sy:sy + img_size, sx:sx + img_size]\n        x = x.unsqueeze(0)\n\n        return x, sx, sy, sz","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:39:56.553544Z","iopub.execute_input":"2025-02-02T15:39:56.554201Z","iopub.status.idle":"2025-02-02T15:39:56.562625Z","shell.execute_reply.started":"2025-02-02T15:39:56.554175Z","shell.execute_reply":"2025-02-02T15:39:56.561789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"root_dir = Path(\"/kaggle/input/czii-cryo-et-object-identification\")\nmode = \"test\"\n# mode = \"train\"\ncfg = OmegaConf.load(\"/kaggle/input/czii-src/kaggle-czii-main/src/config.yaml\")\ncfg.model.backbone = \"tf_efficientnet_b0\"\ncfg.model.arch = \"timm3d3\"\ncfg.model.in_channels = 5\ncfg.model.class_num = 6\n\ncfg.data.fold_id = 2\nimg_size = cfg.model.img_size\nimg_depth = cfg.model.img_depth","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:39:56.563482Z","iopub.execute_input":"2025-02-02T15:39:56.563778Z","iopub.status.idle":"2025-02-02T15:39:56.605856Z","shell.execute_reply.started":"2025-02-02T15:39:56.563743Z","shell.execute_reply":"2025-02-02T15:39:56.605237Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def find_local_maxima(arr, threshold, a):\n    # max_filtered = maximum_filter(arr, size=a, mode=\"constant\")\n    max_filtered = F.max_pool3d(arr[None, None, ...], a, stride=1, padding=a // 2)[0]\n    local_maxima = (arr == max_filtered) & (arr >= threshold)\n    #coordinates = np.argwhere(local_maxima)\n    coordinates = torch.argwhere(local_maxima)\n    return coordinates","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:39:56.607626Z","iopub.execute_input":"2025-02-02T15:39:56.607869Z","iopub.status.idle":"2025-02-02T15:39:56.611623Z","shell.execute_reply.started":"2025-02-02T15:39:56.607848Z","shell.execute_reply":"2025-02-02T15:39:56.610979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mode = \"test\"\ndebug = False\n\n# folds = [0, 1, 2, 3, 4, 5, 6]\n# folds = [2]\n# folds = [2, 3, 4, 6]\nfolds = [0, 1, 2]\ncrop_size = 128\nseq_len = 32\nnum_workers = 0\nbatch_size = 8\nnms_filter_size = 7\n\nfold_exp_mapping = {\n    0: 'TS_5_4', \n    1: 'TS_69_2',\n    2: 'TS_6_4',\n    3: 'TS_6_6', \n    4: 'TS_73_6', \n    5: 'TS_86_3', \n    6: 'TS_99_9'\n}\nDATA_DIR = \"/kaggle/input/czii-cryo-et-object-identification\"\nzarr_paths = sorted(glob(f\"{DATA_DIR}/{mode}/static/ExperimentRuns/**/denoised.zarr\", recursive=True))\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\noof_results = defaultdict(list)\nfor fold in folds:\n    logdir = \"/kaggle/input/czii-weights4/exp122/resnetrs50_32x1x128x128_ia_fastfcn_mixup_ep25_bs8x4\"\n    oof_result = json.load(open(f\"{logdir}/{fold_exp_mapping[fold]}/result.json\"))[\"results\"]\n    # oof_result = json.load(open(f\"{logdir}/{fold_exp_mapping[fold]}/result_best_auc.json\"))[\"results\"]\n    for key in oof_result:\n        oof_results[key].append(oof_result[key])\noof_results = {key: sum(oof_results[key]) / len(oof_results[key]) for key in oof_results}\n# print(oof_results)\n# oof_results = {\n#     'threshold_apo-ferritin': 0.6,\n#     'threshold_beta-amylase': 0.5,\n#     'threshold_beta-galactosidase': 0.65,\n#     'threshold_ribosome': 0.6,\n#     'threshold_thyroglobulin': 0.7,\n#     'threshold_virus-like-particle': 0.5,\n# }\n\n\nclass Net:\n    def __init__(self, weights, device=0):\n        cudart.cudaSetDevice(device)\n        logger = trt.Logger(trt.Logger.INFO)\n        trt.init_libnvinfer_plugins(logger, \"\")\n        runtime = trt.Runtime(logger)\n        with open(weights, \"rb\") as f:\n            engine = runtime.deserialize_cuda_engine(f.read())\n\n        self.trt_model = TRTModule(input_names=['images'],\n                                   output_names=['output'],\n                                   engine=engine)\n        self.device = device\n\n    def __del__(self):\n        del self.trt_model\n\n    def __call__(self, images):\n        output = self.trt_model(images.cuda(self.device))  # key名修正\n        return output\n\ndef get_models_tattaka(device):\n    models = []\n    for fold in folds:\n        logdir = \"/kaggle/input/converting-pytorch-checkpoints-to-tensorrt-models/czii-weights4/exp122/resnetrs50_32x1x128x128_ia_fastfcn_mixup_ep25_bs8x4\"\n        ckpt_path = glob(f\"{logdir}/{fold_exp_mapping[fold]}/**/best_metric.engine\", recursive=True)[0]\n        model = Net(ckpt_path, device)\n        models.append(model)\n    return models\noof_results","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:39:56.612775Z","iopub.execute_input":"2025-02-02T15:39:56.612996Z","iopub.status.idle":"2025-02-02T15:39:56.899039Z","shell.execute_reply.started":"2025-02-02T15:39:56.612975Z","shell.execute_reply":"2025-02-02T15:39:56.898264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_models_yu4u(device):\n    models = []\n\n    for ckpt_path in sorted(Path(\"/kaggle/input/czii-tensorrt-convert-1\").glob(\"*.engine\")):\n        print(f\"loading {str(ckpt_path)}\")\n        model = Net(ckpt_path, device)\n        models.append(model)\n\n    \"\"\"\n    for ckpt_path in sorted(Path(\"/kaggle/input/czii-tensorrt-convert-2\").glob(\"*.engine\")):\n        print(f\"loading {str(ckpt_path)}\")\n        model = Net(ckpt_path, device)\n        models.append(model)\n    \"\"\"\n\n    return models","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:39:56.899842Z","iopub.execute_input":"2025-02-02T15:39:56.90007Z","iopub.status.idle":"2025-02-02T15:39:56.904205Z","shell.execute_reply.started":"2025-02-02T15:39:56.900051Z","shell.execute_reply":"2025-02-02T15:39:56.903421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# thresholds = [0.3, 0.3, 0.25, 0.3, 0.2]\n# thresholds = [0.35, 0.35, 0.3, 0.35, 0.25]\n\n# thresholds = [0.4, 0.35, 0.4, 0.35, 0.3]\n\n# thresholds = [0.5, 0.5, 0.5, 0.55, 0.4] # ensemble\nthresholds = [0.5, 0.53, 0.5, 0.55, 0.4] # ensemble\n\nnms_filter_size = 7\n\nparticle_types = [\n    \"apo-ferritin\",\n    \"beta-galactosidase\",\n    \"ribosome\",\n    \"thyroglobulin\",\n    \"virus-like-particle\",\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:39:56.904967Z","iopub.execute_input":"2025-02-02T15:39:56.905182Z","iopub.status.idle":"2025-02-02T15:39:56.917053Z","shell.execute_reply.started":"2025-02-02T15:39:56.905161Z","shell.execute_reply":"2025-02-02T15:39:56.916272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def gen_df(zarr_paths, results, device_id):\n    device = torch.device(f\"cuda:{device_id}\")\n    torch.cuda.set_device(device)\n    models_yu4u = get_models_yu4u(device_id)\n    models_tattaka = get_models_tattaka(device_id)\n    patch_weight_yu4u = torch.from_numpy(get_patch_weight(img_size, img_depth)).to(device)\n    patch_weight_tattaka = torch.from_numpy(get_patch_weight(crop_size, seq_len)).to(device)\n    for zarr_path in tqdm(zarr_paths):\n        exp_name = zarr_path.split(\"/\")[7]\n        zarr_file = zarr.open(str(zarr_path))\n        tomogram = zarr_file[\"0\"][:]\n        tomogram = (tomogram - 5.2577832e-08) / 7.199929e-06\n        tomogram = np.pad(tomogram, ((4, 4), (10, 10), (10, 10)), mode=\"constant\", constant_values=0)\n        tomogram = tomogram.astype(np.float32)\n        tomogram = torch.from_numpy(tomogram).to(device)\n        preds = torch.zeros((5, *tomogram.shape), dtype=float, device=device)\n        weights = torch.zeros((1, *tomogram.shape), dtype=float, device=device)\n        # yu4u part\n        cfg.model.img_depth = img_depth\n        cfg.model.img_size = img_size\n        dataset = MyDataset(cfg, tomogram)\n        dataloader = DataLoader(dataset, batch_size=16, collate_fn=None, shuffle=False, drop_last=False,\n                                num_workers=0)\n        for xi, sx, sy, sz in tqdm(dataloader):\n            with torch.no_grad():\n                with torch.autocast(device_type=\"cuda\", dtype=torch.float16), torch.no_grad():\n                    outputs = []\n                    \n                    for mi, model in enumerate(models_yu4u):\n                        if mi == 0:\n                            output = model(xi)\n                        elif mi == 1:\n                            output = model(xi.flip(3))\n                            output = output.flip(3)\n                        elif mi == 2:\n                            output = model(xi.flip(4))\n                            output = output.flip(4)\n                        elif mi == 3:\n                            output = model(xi.flip(3).flip(4))\n                            output = output.flip(3).flip(4)\n                        elif mi == 4:\n                            output = model(xi.rot90(k=1, dims=(3, 4))).rot90(k=3, dims=(3, 4))\n                        elif mi == 5:\n                            output = model(xi.rot90(k=1, dims=(3, 4)).flip(3))\n                            output = output.flip(3).rot90(k=3, dims=(3, 4))\n                        elif mi == 6:\n                            output = model(xi.rot90(k=1, dims=(3, 4)).flip(4))\n                            output = output.flip(4).rot90(k=3, dims=(3, 4))\n                        else:\n                            raise ValueError(\"invalid mi\")\n\n                        outputs.append(output)\n\n                    output = torch.mean(torch.stack(outputs), dim=0)\n                        \n                    for pred, x, y, z in zip(output, sx, sy, sz):\n                        preds[:, z:z + img_depth, y:y + img_size, x:x + img_size] += pred[:5] * patch_weight_yu4u\n                        weights[:, z:z + img_depth, y:y + img_size, x:x + img_size] += patch_weight_yu4u\n        del dataloader, dataset\n        # tattaka part\n        cfg.model.img_depth = seq_len\n        cfg.model.img_size = crop_size\n        dataset = MyDataset(cfg, tomogram)\n        dataloader = DataLoader(dataset, batch_size=8, collate_fn=None, shuffle=False, drop_last=False,\n                                num_workers=0)\n        for xi, sx, sy, sz in tqdm(dataloader):\n            xi = xi.transpose(1, 2)\n            with torch.no_grad():\n                with torch.autocast(device_type=\"cuda\", dtype=torch.float16), torch.no_grad():\n                    outputs = []\n                    \n                    for mi, model in enumerate(models_tattaka):\n                        if mi == 0:\n                            output = model(xi)\n                        elif mi == 1:\n                            output = model(xi.flip(3))\n                            output = output.flip(3)\n                        elif mi == 2:\n                            output = model(xi.flip(4))\n                            output = output.flip(4)\n                        elif mi == 3:\n                            output = model(xi.flip(3).flip(4))\n                            output = output.flip(3).flip(4)\n                        elif mi == 4:\n                            output = model(xi.rot90(k=1, dims=(3, 4))).rot90(k=3, dims=(3, 4))\n                        elif mi == 5:\n                            output = model(xi.rot90(k=1, dims=(3, 4)).flip(3))\n                            output = output.flip(3).rot90(k=3, dims=(3, 4))\n                        elif mi == 6:\n                            output = model(xi.rot90(k=1, dims=(3, 4)).flip(4))\n                            output = output.flip(4).rot90(k=3, dims=(3, 4))\n                        else:\n                            raise ValueError(\"invalid mi\")\n                        \n                        outputs.append(output)\n\n                    output = torch.mean(torch.stack(outputs), dim=0)\n                    \n                    for pred, x, y, z in zip(output, sx, sy, sz):\n                        preds[:, z:z + seq_len, y:y + crop_size, x:x + crop_size] += pred[[0, 2, 3, 4, 5]] * patch_weight_tattaka\n                        weights[:, z:z + seq_len, y:y + crop_size, x:x + crop_size] += patch_weight_tattaka\n        weights.clamp_min_(1.0)\n        preds /= weights\n        preds = preds[:, 4:-4, 10:640, 10:640]\n    \n        for i, particle_type in enumerate(particle_types):\n            threshold = thresholds[i]\n            coordinates = find_local_maxima(preds[i], threshold, nms_filter_size)\n            \n            for _, z, y, x in coordinates.cpu().numpy():\n                point = ((x + 0.5 - 1) * 10.012444537618887, (y + 0.5 - 1) * 10.012444196428572, (z + 0.5 - 1) * 10.012444196428572)\n                result = [exp_name, particle_type, *point]\n                results.append(result)\n        del tomogram, preds, weights, dataloader, dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:39:56.917825Z","iopub.execute_input":"2025-02-02T15:39:56.918111Z","iopub.status.idle":"2025-02-02T15:39:56.938311Z","shell.execute_reply.started":"2025-02-02T15:39:56.91809Z","shell.execute_reply":"2025-02-02T15:39:56.937465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"manager = mp.Manager()\nresults1 = manager.list()\nresults2 = manager.list()\n\n# zarr_paths = zarr_paths + zarr_paths + zarr_paths + zarr_paths # debug\n\n# データセットの分割\nN = len(zarr_paths)\nhalf = N // 2\nzarr_paths1 = zarr_paths[:half]\nzarr_paths2 = zarr_paths[half:]\n\n# 並列実行\np0 = mp.Process(target=gen_df, args=(zarr_paths1, results1, 0))  # GPU 0\np1 = mp.Process(target=gen_df, args=(zarr_paths2, results2, 1))  # GPU 1\n\np0.start()\np1.start()\n\np0.join()\np1.join()\n\n# 結果を統合\nresults = list(results1) + list(results2)\nprint(\"Inference completed!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:39:56.93908Z","iopub.execute_input":"2025-02-02T15:39:56.939309Z","iopub.status.idle":"2025-02-02T15:45:47.42675Z","shell.execute_reply.started":"2025-02-02T15:39:56.939284Z","shell.execute_reply":"2025-02-02T15:45:47.42557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.chdir(\"/kaggle/working/\")\ndf = pd.DataFrame(results, columns=[\"experiment\", \"particle_type\", \"x\", \"y\", \"z\"])\ndf.index.name = \"id\"\ndf.to_csv(\"submission.csv\")\ndf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T15:45:47.428358Z","iopub.execute_input":"2025-02-02T15:45:47.428741Z","iopub.status.idle":"2025-02-02T15:45:47.476934Z","shell.execute_reply.started":"2025-02-02T15:45:47.428694Z","shell.execute_reply":"2025-02-02T15:45:47.476267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"! rm -r /kaggle/working/czii-cryo-et-object-identification-main","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-31T14:36:23.297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if mode == \"train\":\n    from src.metric import experiments, score\n    ! cp /kaggle/input/czii-src/kaggle-czii-main/03_create_solution.py .\n    ! ln -s /kaggle/input/czii-cryo-et-object-identification input\n    ! python 03_create_solution.py\n    solution = pd.read_csv(\"output/solution.csv\")\n    for fold in folds:\n        solution = solution[solution[\"experiment\"] == fold_exp_mapping[fold]]\n        df = df[df[\"experiment\"] == fold_exp_mapping[fold]]\n        display(solution.head())\n        print(score(solution, df, distance_multiplier=0.5, beta=4))","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-31T14:36:23.297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def func_fixed(\n#     y_true: np.ndarray,\n#     y_pred: np.ndarray,\n#     y_pred_score: np.ndarray,\n#     metric_fn,\n#     r: int,\n#     t,\n# ):\n#     score = metric_fn(\n#         y_pred[y_pred_score > np.clip(t[0], 0, 1)],\n#         y_true,\n#         r,\n#     )\n#     return -score\n    \n# def find_threshold_fixed(\n#     y_true: np.ndarray,\n#     y_pred: np.ndarray,\n#     y_pred_score: np.ndarray,\n#     metric_fn,\n#     r: int,\n# ):\n#     x0 = [0.5]\n#     threshold = minimize(\n#         partial(func_fixed, y_true, y_pred, y_pred_score, metric_fn, r),\n#         x0,\n#         method=\"nelder-mead\",\n#     ).x[0]\n#     return np.clip(threshold, 0, 1)\n\n\n# def func_percentile(\n#     y_true: np.ndarray,\n#     y_pred: np.ndarray,\n#     y_pred_score: np.ndarray,\n#     metric_fn,\n#     r: int,\n#     t,\n# ):\n#     print(t[0])\n#     score = metric_fn(\n#         y_pred[\n#             y_pred_score\n#             > np.quantile(\n#                 y_pred_score,\n#                 np.clip(t[0], 0, 1),\n#             )\n#         ],\n#         y_true,\n#         r,\n#     )\n#     return -score\n\n# def find_threshold_percentile(\n#     y_true: np.ndarray,\n#     y_pred: np.ndarray,\n#     y_pred_score: np.ndarray,\n#     metric_fn,\n#     r: int,\n# ):\n#     x0 = [1]\n#     threshold = minimize(\n#         partial(func_percentile, y_true, y_pred, y_pred_score, metric_fn, r),\n#         x0,\n#         method=\"nelder-mead\",\n#     ).x[0]\n#     return np.clip(threshold, 0, 1)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-31T14:36:23.297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if mode == \"train\":\n#     oof_results_tmp = {\n#         \"threshold_apo-ferritin\": 0.,\n#         \"threshold_beta-galactosidase\": 0.,\n#         \"threshold_ribosome\": 0.,\n#         \"threshold_thyroglobulin\": 0.,\n#         \"threshold_virus-like-particle\": 0.,\n#     }\n#     pred_points_cand, pred_scores_cand = find_local_maxima(logit_volume, oof_results_tmp, nms_filter_size)\n#     for key in pred_points_cand:\n#         pred_points_cand[key] = pred_points_cand[key][:, ::-1]\n#     thresholds = {}\n#     metrics = {}\n    \n#     for key in pred_points_cand:\n#         thresholds[key] = find_threshold_fixed(\n#             solution[solution.particle_type == key].loc[:, [\"x\", \"y\", \"z\"]].values,\n#             pred_points_cand[key],\n#             pred_scores_cand[key],\n#             fbeta_comp_fn,\n#             r=radius[key],\n#         )\n#         # thresholds[key] = find_threshold_percentile(\n#         #     solution[solution.particle_type == key].loc[:, [\"x\", \"y\", \"z\"]].values,\n#         #     pred_points_cand[key],\n#         #     pred_scores_cand[key],\n#         #     fbeta_comp_fn,\n#         #     r=radius[key],\n#         # )\n#         print(key, pred_points_cand[key].shape)\n#         pred_points = pred_points_cand[key][pred_scores_cand[key] > thresholds[key]]\n#         # pred_points = pred_points_cand[key][\n#         #     pred_scores_cand[key]\n#         #     > np.quantile(\n#         #         pred_scores_cand[key],\n#         #         np.clip(thresholds[key], 0, 1),\n#         #     )\n#         # ]\n#         metrics[key] = particles_fbeta_one_class(\n#             pred_points, solution[solution.particle_type == key].loc[:, [\"x\", \"y\", \"z\"]].values, radius[key]\n#         )\n#     log_metric = {}\n#     for key in metrics:\n#         log_metric[f\"val_fbeta_{key}\"] = metrics[key]\n#     for key in thresholds:\n#         log_metric[f\"threshold_{key}\"] = thresholds[key]\n#     log_metric[\"val_aggregate_fbeta\"] = sum(\n#         [metrics[key] * weights[key] for key in metrics]\n#     ) / sum(weights.values())\n#     display(log_metric)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-31T14:36:23.297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def score_tp_fp_fn(tp, fp, fn):\n#     beta = 4\n#     precision = tp / (tp + fp) if tp + fp > 0 else 0\n#     recall = tp / (tp + fn) if tp + fn > 0 else 0\n#     fbeta = (\n#         (1 + beta**2) * (precision * recall) / (beta**2 * precision + recall)\n#         if (precision + recall) > 0\n#         else 0.0\n#     )\n#     return fbeta","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-31T14:36:23.297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}