{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport os\nfrom scipy import ndimage\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A \nfrom tqdm import tqdm\nfrom albumentations.pytorch.transforms import ToTensorV2\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nfrom PIL import Image \nfrom matplotlib import pyplot as plt \nimport json \nfrom skimage import io\nfrom skimage import color\nimport torch\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-13T07:41:01.413302Z","iopub.execute_input":"2023-06-13T07:41:01.413798Z","iopub.status.idle":"2023-06-13T07:41:09.071767Z","shell.execute_reply.started":"2023-06-13T07:41:01.413752Z","shell.execute_reply":"2023-06-13T07:41:09.070661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /tmp/pip/cache/\n!cp ../input/smpmodels/pretrainedmodels-0.7.4.tar.gz /tmp/pip/cache/pretrainedmodels-0.7.4.tar.gz\n!cp ../input/smpmodels/efficientnet_pytorch-0.6.3.tar.gz /tmp/pip/cache/efficientnet_pytorch-0.6.3.tar.gz\n\n!cp ../input/smpmodels/segmentation_models_pytorch-0.2.1-py3-none-any.whl /tmp/pip/cache/\n\n\n!pip install ../input/smpmodels/timm-0.4.12-py3-none-any.whl\n!pip install --no-index --find-links /tmp/pip/cache/ segmentation-models-pytorch\n#!pip install /kaggle/input/segmentation-models-pytorch/pretrainedmodels-0.7.4-py3-none-any.whl\n#!pip install /kaggle/input/segmentation-models-pytorch/efficientnet_pytorch-0.7.1-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-06-13T07:41:09.073663Z","iopub.execute_input":"2023-06-13T07:41:09.074036Z","iopub.status.idle":"2023-06-13T07:42:06.260079Z","shell.execute_reply.started":"2023-06-13T07:41:09.074006Z","shell.execute_reply":"2023-06-13T07:42:06.258506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/packages\n!cp -r /kaggle/input/pycocotools/* /kaggle/working/packages\nos.chdir(\"/kaggle/working/packages/pycocotools-2.0.6/\")\n!python setup.py install\n!pip install . --no-index --find-links /kaggle/working/packages/\nos.chdir(\"/kaggle/working\")","metadata":{"execution":{"iopub.status.busy":"2023-06-13T07:42:06.262647Z","iopub.execute_input":"2023-06-13T07:42:06.264001Z","iopub.status.idle":"2023-06-13T07:43:06.712874Z","shell.execute_reply.started":"2023-06-13T07:42:06.263954Z","shell.execute_reply":"2023-06-13T07:43:06.711521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nimport segmentation_models_pytorch as smp\n\ndef build_model():\n    model = smp.Unet(\n        encoder_name='efficientnet-b0',      # choose encoder, e.g. mobilenet_v2 or efficientnet-b7\n        encoder_weights=None,     # use `imagenet` pre-trained weights for encoder initialization\n        in_channels=3,                  # model input channels (1 for gray-scale images, 3 for RGB, etc.)\n        classes=5,        # model output channels (number of classes in your dataset)\n        activation=None,\n    )\n    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n    model = model.to(device) \n    return model\n\nmodel = build_model()\nmodelWeight= \"/kaggle/input/hubmapmodelssmp/model_balance_0.pt\"\nmodel.load_state_dict(torch.load(modelWeight))","metadata":{"execution":{"iopub.status.busy":"2023-06-13T07:43:06.716739Z","iopub.execute_input":"2023-06-13T07:43:06.717209Z","iopub.status.idle":"2023-06-13T07:43:13.296772Z","shell.execute_reply.started":"2023-06-13T07:43:06.717154Z","shell.execute_reply":"2023-06-13T07:43:13.295607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_transform = A.Compose([\n    A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n    ToTensorV2()\n])\nclass kidneyDatasetaTest(Dataset):\n    def __init__(self, trainFiles, trainFolder,transforms):\n        self.trainFiles = trainFiles \n        self.trainFolder = trainFolder \n        self.transforms = transforms \n    def __getitem__(self, idx):\n        trainId = self.trainFiles[idx]\n        inputPath = os.path.join(self.trainFolder, trainId)\n        im  = Image.open(inputPath)\n        #mask = np.argmax(mask,axis=-1)\n        if self.transforms is not None: \n            transformed = self.transforms(image = np.array(im))\n            return trainId.split(\".\")[0], transformed['image']\n    def __len__(self):\n        return len(self.trainFiles)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T07:43:13.299359Z","iopub.execute_input":"2023-06-13T07:43:13.300526Z","iopub.status.idle":"2023-06-13T07:43:13.309450Z","shell.execute_reply.started":"2023-06-13T07:43:13.300489Z","shell.execute_reply":"2023-06-13T07:43:13.308412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testFolder = \"/kaggle/input/hubmap-hacking-the-human-vasculature/test\"\ntestFiles = os.listdir(testFolder) \ntest_dataset = kidneyDatasetaTest(testFiles,testFolder, valid_transform)\ntest_loader = DataLoader(test_dataset, batch_size=2, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T07:43:13.311653Z","iopub.execute_input":"2023-06-13T07:43:13.312118Z","iopub.status.idle":"2023-06-13T07:43:13.333433Z","shell.execute_reply.started":"2023-06-13T07:43:13.312070Z","shell.execute_reply":"2023-06-13T07:43:13.332309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import base64\nimport numpy as np\nimport typing as t\nimport zlib\n\nimport numpy as np\nfrom itertools import groupby\n\ndef binary_mask_to_rle(binary_mask):\n    rle = {'counts': [], 'size': list(binary_mask.shape)}\n    counts = rle.get('counts')\n    for i, (value, elements) in enumerate(groupby(binary_mask.ravel(order='F'))):\n        if i == 0 and value == 1:\n            counts.append(0)\n        counts.append(len(list(elements)))\n    return rle\n\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    encoded_mask  = binary_mask_to_rle(img)\n    encoded_mask = encoded_mask[\"counts\"]\n    encoded_mask  = [ str(x) for x in encoded_mask] \n    encoded_mask = \" \".join(encoded_mask)\n    binary_str = zlib.compress(encoded_mask.encode(), zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str.decode() ","metadata":{"execution":{"iopub.status.busy":"2023-06-13T07:43:13.335210Z","iopub.execute_input":"2023-06-13T07:43:13.335648Z","iopub.status.idle":"2023-06-13T07:43:13.346113Z","shell.execute_reply.started":"2023-06-13T07:43:13.335614Z","shell.execute_reply":"2023-06-13T07:43:13.344998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nfrom skimage.measure import label, regionprops\n\nimport base64\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\n\ndef encode_binary_mask(mask: np.ndarray) -> t.Text:\n    # check input mask --\n    if mask.dtype != bool:\n        raise ValueError(\n            \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n            mask.dtype)\n\n    mask = np.squeeze(mask)\n    if len(mask.shape) != 2:\n        raise ValueError(\n            \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n            mask.shape)\n\n    # convert input mask to expected COCO API input --\n    mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    mask_to_encode = mask_to_encode.astype(np.uint8)\n    mask_to_encode = np.asfortranarray(mask_to_encode)\n\n    # RLE encode mask --\n    encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n    # compress and base64 encoding --\n    binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str\n\n# taken from Sample Submission \n#test_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/test\"\n#sample_submission = pd.read_csv('/kaggle/input/hubmap-hacking-the-human-vasculature/sample_submission.csv')\n\n#ids = []\n#heights = []\n#widths = []\n#prediction_strings = []\n#for img_name in os.listdir(test_path):\n#     img = cv2.imread(f\"{test_path}/{img_name}\", 0)\n#     h, w = img.shape\n    \n#     mask = np.ones((h, w), dtype=np.bool)\n#  encoded = encode_binary_mask(mask)\n    \n#     ids.append(img_name.split('.')[0])\n#     heights.append(h)\n#     widths.append(w)\n#     prediction_strings.append(f\"0 1.0 {encoded.decode('utf-8')}\")\n\n# submission = pd.DataFrame()\n# submission['id'] = ids\n# submission['height'] = heights\n# submission['width'] = widths\n# submission['prediction_string'] = prediction_strings\ndef convertOutput_v2(outputs,idx):\n    \"\"\"\n    Return numpy array for outputs for only blood vessel \n    \"\"\"\n    blood_vessel = torch.argmax(outputs, 1) # N, H, W\n    blood_vessel = blood_vessel == 1# multiple images are expected\n    blood_vessel = blood_vessel * 1\n    blood_vessel = blood_vessel.cpu().numpy()\n    all_encode = {} \n    for i in range(blood_vessel.shape[0]):\n        predString = \"\"\n        sliceImage = blood_vessel[i,:,:]\n        sliceImage= label(sliceImage)\n        areaStats = regionprops(sliceImage) \n        allObjs = np.nonzero(np.unique(sliceImage))\n        for n in list(allObjs[0]):\n            null_image = np.zeros_like(sliceImage)\n            null_image = sliceImage == n\n            areaObj =  areaStats[n-1].area\n            if areaObj > 50 : \n                coded_len = encode_binary_mask(null_image)\n                if not predString:\n                    predString += f\"0 1.0 {coded_len.decode('utf-8')}\"\n                else:\n                    predString +=  f\" 0 1.0 {coded_len.decode('utf-8')}\"\n\n        all_encode[idx[i]] =predString\n    return all_encode\n\n# def convertOutput(outputs,idx):\n#     \"\"\"\n#     Return numpy array for outputs for only blood vessel \n#     \"\"\"\n#     blood_vessel = torch.argmax(outputs, 1) # N, H, W\n#     blood_vessel = blood_vessel == 1# multiple images are expected\n#     blood_vessel = blood_vessel * 1\n#     blood_vessel = blood_vessel.cpu().numpy()\n#     all_encode = {} \n#     for i in range(blood_vessel.shape[0]):\n#         list_encode = []\n#         sliceImage = blood_vessel[i,:,:]\n#         sliceImage= label(sliceImage)\n#         for n in np.unique(sliceImage):\n#             if n != 0 :\n#                 null_image = np.zeros_like(sliceImage)\n#                 null_image = sliceImage == n\n#                 coded_len = rle_encode(null_image)\n#                 list_encode.append(coded_len)\n                \n#         all_encode[idx[i]] = \" \".join([\"0 1.0 \" + x for x in list_encode])\n#     return all_encode\ndef convertOutput_v3(outputs,idx):\n    \"\"\"\n    Return numpy array for outputs for only blood vessel \n    \"\"\"\n    blood_vessel = torch.argmax(outputs, 1) # N, H, W\n    blood_vessel = blood_vessel == 1# multiple images are expected\n    blood_vessel = blood_vessel * 1\n    \n    blood_vessel = blood_vessel.cpu().numpy()\n    all_encode = {} \n    for i in range(blood_vessel.shape[0]):\n        list_encode = []\n        sliceImage = blood_vessel[i,:,:]\n        plt.imshow(sliceImage)\n        plt.show()\n        binarized = sliceImage > 0\n        coded_len = encode_binary_mask(binarized)\n        list_encode.append(coded_len)\n        all_encode[idx[i]] =list_encode\n    return all_encode\n# submission = submission.set_index('id')\n# print(submission)\n# submission.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-06-13T07:43:13.347852Z","iopub.execute_input":"2023-06-13T07:43:13.348778Z","iopub.status.idle":"2023-06-13T07:43:13.371848Z","shell.execute_reply.started":"2023-06-13T07:43:13.348739Z","shell.execute_reply":"2023-06-13T07:43:13.370676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device =  torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\noutSoft = torch.nn.Softmax2d() \nOutputDict ={}\nwith torch.no_grad():\n    for  idx, images in tqdm(test_loader):\n        images = images.to(device)\n        outputs = model(images.float())\n        outputs = outSoft(outputs)\n        encoded = convertOutput_v2(outputs, idx)\n        for key in encoded: \n            OutputDict[key] = \"\"#encoded[key]","metadata":{"execution":{"iopub.status.busy":"2023-06-13T07:43:13.374109Z","iopub.execute_input":"2023-06-13T07:43:13.375314Z","iopub.status.idle":"2023-06-13T07:43:18.791117Z","shell.execute_reply.started":"2023-06-13T07:43:13.375279Z","shell.execute_reply":"2023-06-13T07:43:18.789977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(OutputDict.items(), columns= [\"id\", \"prediction_string\"]) \nsubmission[\"height\"] = 512\nsubmission[\"width\"] = 512 \nsubmission= submission[[\"id\",\"height\",\"width\",\"prediction_string\"]]\nsubmission.to_csv(\"submission.csv\", index= False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T07:43:18.794421Z","iopub.execute_input":"2023-06-13T07:43:18.795359Z","iopub.status.idle":"2023-06-13T07:43:18.828282Z","shell.execute_reply.started":"2023-06-13T07:43:18.795313Z","shell.execute_reply":"2023-06-13T07:43:18.827321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['prediction_string'][0]","metadata":{"execution":{"iopub.status.busy":"2023-06-13T07:43:18.829939Z","iopub.execute_input":"2023-06-13T07:43:18.830353Z","iopub.status.idle":"2023-06-13T07:43:18.837798Z","shell.execute_reply.started":"2023-06-13T07:43:18.830309Z","shell.execute_reply":"2023-06-13T07:43:18.836137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}