{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":61446,"databundleVersionId":6962461,"sourceType":"competition"},{"sourceId":7229399,"sourceType":"datasetVersion","datasetId":4185566},{"sourceId":7229919,"sourceType":"datasetVersion","datasetId":4185960},{"sourceId":7229957,"sourceType":"datasetVersion","datasetId":4185994},{"sourceId":7273531,"sourceType":"datasetVersion","datasetId":4216702},{"sourceId":7391777,"sourceType":"datasetVersion","datasetId":4297118},{"sourceId":7394332,"sourceType":"datasetVersion","datasetId":4298881},{"sourceId":7396595,"sourceType":"datasetVersion","datasetId":4300581},{"sourceId":7399537,"sourceType":"datasetVersion","datasetId":4302483},{"sourceId":7405312,"sourceType":"datasetVersion","datasetId":4306459},{"sourceId":7528824,"sourceType":"datasetVersion","datasetId":4385142},{"sourceId":7528849,"sourceType":"datasetVersion","datasetId":4385159},{"sourceId":7530678,"sourceType":"datasetVersion","datasetId":4386138},{"sourceId":7537888,"sourceType":"datasetVersion","datasetId":4389454},{"sourceId":7538471,"sourceType":"datasetVersion","datasetId":4389749},{"sourceId":7540531,"sourceType":"datasetVersion","datasetId":4390942},{"sourceId":7544644,"sourceType":"datasetVersion","datasetId":4393620}],"dockerImageVersionId":30636,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport math\nimport glob\nimport tqdm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as T\nimport torchvision.transforms.functional as TF\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom scipy.signal import argrelextrema\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-06T11:07:19.283897Z","iopub.execute_input":"2024-02-06T11:07:19.284579Z","iopub.status.idle":"2024-02-06T11:07:24.898130Z","shell.execute_reply.started":"2024-02-06T11:07:19.284534Z","shell.execute_reply":"2024-02-06T11:07:24.896903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ENCODER = \"tu-seresnext50_32x4d\"\nWEIGHTS = \"imagenet\"\nDEPTH = 1\nPATCH_SIZE = 512\nBATCH_SIZE = 2\nrotations = [0,1,2,3]\nmodel_path = '/kaggle/input/p1-p99-tu-seresnext50-32x4d-imagenet-512-30-1e-3/SMP_2D_MIXED_tu-seresnext50_32x4d_imagenet_512'\nckp_path = '/kaggle/input/p1-p99-tu-seresnext50-32x4d-imagenet-512-30-1e-3/models/model_20.pth'\n#model_path = '/kaggle/input/smp-2d-mixed-tu-seresnext50-32x4d-imagenet-512-30/SMP_2D_MIXED_tu-seresnext50_32x4d_imagenet_512'\n#std_model_path = '/kaggle/input/std-tu-seresnext50-32x4d-imagenet-512-30-1e-3/SMP_2D_MIXED_tu-seresnext50_32x4d_imagenet_512'\n#path = '/kaggle/input/smp-2d-mixed-resnet50-imagenet-512-volnorm/SMP_2D_MIXED_resnet50_imagenet_512 (2)'\n#path = '/kaggle/input/smp-2d-mixed-resnet50-imagenet-512-30/SMP_2D_MIXED_resnet50_imagenet_512 (1)'\n#path = '/kaggle/input/smp-2d-mixed-resnet50-imagenet-512-10/SMP_2D_MIXED_resnet50_imagenet_512'\n#path = '/kaggle/input/smp-resnet50-imagenet-512-2-of-4/SMP_resnet50_imagenet_512_2_of_4'\n#path = '/kaggle/input/smp-2d-resnet50-imagenet-512-anydepth-norot/SMP_2D_resnet50_imagenet_512'\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"execution":{"iopub.status.busy":"2024-02-06T11:07:24.900465Z","iopub.execute_input":"2024-02-06T11:07:24.901424Z","iopub.status.idle":"2024-02-06T11:07:24.911514Z","shell.execute_reply.started":"2024-02-06T11:07:24.901379Z","shell.execute_reply":"2024-02-06T11:07:24.908022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Datasets = glob.glob(\"/kaggle/input/blood-vessel-segmentation/test/*\")","metadata":{"execution":{"iopub.status.busy":"2024-02-06T11:07:24.913161Z","iopub.execute_input":"2024-02-06T11:07:24.913614Z","iopub.status.idle":"2024-02-06T11:07:24.941385Z","shell.execute_reply.started":"2024-02-06T11:07:24.913579Z","shell.execute_reply":"2024-02-06T11:07:24.939953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ref.: https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    \n    mystr = ' '.join(str(x) for x in runs)\n    if mystr == \"\":\n        mystr = \"1 0\"\n    return mystr","metadata":{"execution":{"iopub.status.busy":"2024-02-06T11:07:24.944721Z","iopub.execute_input":"2024-02-06T11:07:24.945633Z","iopub.status.idle":"2024-02-06T11:07:24.954337Z","shell.execute_reply.started":"2024-02-06T11:07:24.945574Z","shell.execute_reply":"2024-02-06T11:07:24.953085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_volume(dataset, labeled=True, slice_range=None):\n    ''' Load slices into a volume. Keeps the memory requirement\n        as low as possible by using uint8 and uint16 in CPU memory.\n    '''\n    if labeled:\n        path = os.path.join(dataset, \"labels\", \"*.tif\")\n    else:\n        path = os.path.join(dataset, \"images\", \"*.tif\")\n        \n    dataset = sorted(glob.glob(path))\n    volume = None\n    target = None\n    keys = []\n    offset = 0 if slice_range is None else slice_range[0]\n    depth = len(dataset) if slice_range is None else slice_range[1]-slice_range[0]\n    \n    for z, path in enumerate(tqdm.tqdm(dataset)):\n        if slice_range is not None:\n            if z < slice_range[0]: continue\n            if z >= slice_range[1]: continue\n        \n        parts = path.split(os.path.sep)\n        key = parts[-3] + \"_\" + parts[-1].split(\".\")[0]\n        keys.append(key)\n                \n        if labeled:\n            label = cv2.imread(path, cv2.IMREAD_ANYDEPTH)\n            label = np.array(label,dtype=np.uint8)\n            if target is None:\n                target = np.zeros((1,depth, *label.shape[-2:]), dtype=np.uint8)\n            target[:,z-offset] = label\n        \n        path = path.replace(\"labels\",\"images\")\n        path = path.replace(\"kidney_3_dense\",\"kidney_3_sparse\")\n        image = cv2.imread(path, cv2.IMREAD_ANYDEPTH)\n        image = np.array(image,dtype=np.uint16)\n        \n        if volume is None:\n            volume = np.zeros((1,depth, *image.shape[-2:]), dtype=np.uint16)\n        volume[:,z-offset] = image\n    \n    return volume, target, keys","metadata":{"execution":{"iopub.status.busy":"2024-02-06T11:07:24.955586Z","iopub.execute_input":"2024-02-06T11:07:24.956075Z","iopub.status.idle":"2024-02-06T11:07:24.971085Z","shell.execute_reply.started":"2024-02-06T11:07:24.955932Z","shell.execute_reply":"2024-02-06T11:07:24.969859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scan3D(volume,BATCH_SIZE,model,device):\n    pmin,pmax = np.percentile(volume,(1,99))\n    Z,X,Y = volume.shape[-3:]\n    SHAPE = np.array([Z,X,Y])\n#   PREDS = torch.zeros((2,Z,X,Y))\n    PREDS = torch.zeros((Z,X,Y))\n    for axis in [0,1,2]:\n        D = SHAPE[axis]\n        for d in tqdm.tqdm(range(0,D,BATCH_SIZE)):\n            END = min([d+BATCH_SIZE,D])\n            BATCH = volume[0].take(range(d,END), axis).swapaxes(0,axis)\n            H,W = BATCH.shape[-2:]\n            S = np.max([H,W])\n            P = 0\n            x = S/32\n            N = S//32\n            if x > N: P = 32*(N + 1) - S\n            HP = P + S - H\n            WP = P + S - W\n            BATCH = np.pad(BATCH,((0,0),(HP//2,HP-HP//2),(WP//2,WP-WP//2)),'reflect').astype(np.float32)\n            BATCH = (BATCH - pmin)/(pmax - pmin)\n            BATCH = torch.from_numpy(BATCH).unsqueeze(1).to(device)\n            for rot in rotations:\n                PRED = model(\n                    torch.rot90(BATCH,rot,[-2,-1])\n                )\n                PRED = torch.rot90(PRED,-rot,[-2,-1])\n                PRED = PRED[:,:,HP//2:HP//2+H,WP//2:WP//2+W].swapaxes(0,1).swapaxes(1,1+axis)\n                PRED = nn.Softmax(0)(PRED)\n                PRED = PRED[1]/(PRED[0]+PRED[1])\n                if axis == 0:\n#                   PREDS[:,d:END,:,:] += PRED.cpu()\n                    PREDS[d:END,:,:] += PRED.cpu()\n                elif axis == 1:\n#                   PREDS[:,:,d:END,:] += PRED.cpu()\n                    PREDS[:,d:END,:] += PRED.cpu()\n                else:\n#                   PREDS[:,:,:,d:END] += PRED.cpu()\n                    PREDS[:,:,d:END] += PRED.cpu()\n                    \n            del BATCH\n    \n    return PREDS","metadata":{"execution":{"iopub.status.busy":"2024-02-06T11:07:24.972466Z","iopub.execute_input":"2024-02-06T11:07:24.972855Z","iopub.status.idle":"2024-02-06T11:07:24.990272Z","shell.execute_reply.started":"2024-02-06T11:07:24.972824Z","shell.execute_reply":"2024-02-06T11:07:24.989192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Detailed_scan3D(volume,PREDS,model,device):          \n    TH = 6\n    Z,X,Y = SHAPE = volume.shape[-3:]\n    labels = np.zeros((Z,X,Y), dtype=bool)\n    for i in range(Z):\n        labels[i] = PREDS[i] > TH\n        PREDS[i] = 0\n#-----------------------------------------------------------------------------------------------\n#   Shaman standardization\n#-----------------------------------------------------------------------------------------------\n#   Let's try to standardize targets subspace\n#-----------------------------------------------------------------------------------------------\n    SV = volume[0][labels]\n    mean = np.mean(SV)\n    std = np.std(SV)\n#   Let's scan volume values at\n#   labels neighborhood subspace       \n    for axis in [0,1,2]:\n        D = SHAPE[axis]\n        for d in tqdm.tqdm(range(D)):\n            nonzero = np.argwhere(labels.take(d, axis) > 0)\n            if len(nonzero) > 0:\n                PLANE = volume[0].take(d, axis)\n                H,W = PLANE.shape\n                h,w = np.min(nonzero,axis=0)\n                h_,w_ = np.max(nonzero,axis=0)\n                hc,wc = np.sum(nonzero,axis=0)//len(nonzero)\n                s = np.max([hc-h,h_-hc,wc-w,w_-wc,PATCH_SIZE//2])\n                S = 2*s\n                P = 0\n                x = S/32\n                N = S//32\n                if x > N: P = 32*(N + 1) - S\n                h = hc - s - P//2\n                h_ = h + S + P\n                BP = 0\n                if h < 0:\n                    BP = - h\n                    h = 0\n                TP = 0\n                if h_ > H:\n                    TP = h_ - H\n                    h_ = H\n                w = wc - s - P//2\n                w_ = w + S + P\n                LP = 0\n                if w < 0:\n                    LP = - w\n                    w = 0\n                RP = 0\n                if w_ > W:\n                    RP = w_ - W\n                    w_ = W\n                PLANE = np.pad(PLANE[h:h_,w:w_],((BP,TP),(LP,RP)),'reflect').astype(np.float32)\n                PLANE = torch.from_numpy(PLANE).unsqueeze(0).unsqueeze(0).to(device)\n                PLANE = (PLANE - mean)/std\n                for rot in rotations:\n                    PRED = model(\n                        torch.rot90(PLANE,rot,[-2,-1])\n                    )\n                    PRED = torch.rot90(PRED,-rot,[-2,-1])\n                    PRED = PRED[0,:,BP:BP + h_ - h,LP:LP + w_ - w]\n                    PRED = nn.Softmax(0)(PRED)\n                    PRED = PRED[1]/(PRED[0]+PRED[1])\n                    if axis == 0:\n                        PREDS[d,h:h_,w:w_] += PRED.cpu()\n                    elif axis == 1:\n                        PREDS[h:h_,d,w:w_] += PRED.cpu()\n                    else:\n                        PREDS[h:h_,w:w_,d] += PRED.cpu()\n                        \n                del PLANE\n    \n    return PREDS","metadata":{"execution":{"iopub.status.busy":"2024-02-06T11:07:24.992164Z","iopub.execute_input":"2024-02-06T11:07:24.992602Z","iopub.status.idle":"2024-02-06T11:07:25.015254Z","shell.execute_reply.started":"2024-02-06T11:07:24.992553Z","shell.execute_reply":"2024-02-06T11:07:25.014112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"All_keys = []\nrle = []\nif len(glob.glob(Datasets[0]+'/images/*')) != 3:\n    import sys\n    sys.path.append(\"/kaggle/input/segmentation-models-pytorch/segmentation_models.pytorch-master/\")\n    sys.path.append(\"/kaggle/input/pretrainedmodels-0-7-4/pretrainedmodels-0.7.4\")\n    sys.path.append(\"/kaggle/input/efficientnet-pytorch-0-7-1/efficientnet_pytorch-0.7.1\")\n    import efficientnet_pytorch\n    import pretrainedmodels\n    import segmentation_models_pytorch\n    \n    model = torch.load(model_path)\n#   model = torch.load(model_path,map_location=torch.device('cpu'))\n    model.load_state_dict(torch.load(ckp_path))\n#   model.load_state_dict(torch.load(ckp_path,map_location=torch.device('cpu')))\n     \n    for dataset in Datasets:\n        volume, _, keys = load_volume(dataset, labeled=False, slice_range=None)\n        All_keys += keys\n        with torch.no_grad():\n#           First prediction normalizing between p1,p99 percentiles\n#           with a model trained in that behavior\n            PREDS = scan3D(volume,BATCH_SIZE,model,device)\n#           Second prediction standardizing target subspace\n#           PREDS = Detailed_scan3D(volume,PREDS,std_model,device)\n        \n        for i in range(len(keys)):\n#           PRED = torch.argmax(PREDS[:,i],0)\n#           PRED = nn.Softmax(0)(PREDS[:,i])\n#           PRED = (PRED[1]/(PRED[0]+PRED[1]))>.9\n            PRED = PREDS[i] > 5\n            rle += [rle_encode(PRED)]\n        \n        del volume,keys,PREDS\n    \nelse:\n    for dataset in Datasets:\n        for path in sorted(glob.glob(dataset+'/images/*')):\n            path = path.split('/')\n            All_keys.append(path[-3]+'_'+path[-1][:-4])\n            rle.append('1 0')\n            \npd.DataFrame({'id':All_keys, 'rle':rle}).to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T11:07:25.017083Z","iopub.execute_input":"2024-02-06T11:07:25.017489Z","iopub.status.idle":"2024-02-06T11:07:25.046977Z","shell.execute_reply.started":"2024-02-06T11:07:25.017448Z","shell.execute_reply":"2024-02-06T11:07:25.046026Z"},"trusted":true},"execution_count":null,"outputs":[]}]}