{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"# Installing segmentation_models_pytorch\n!mkdir -p /tmp/pip/cache/\n!cp ../input/segmentationmodelspytorch/segmentation_models/efficientnet_pytorch-0.6.3.xyz /tmp/pip/cache/efficientnet_pytorch-0.6.3.tar.gz\n!cp ../input/segmentationmodelspytorch/segmentation_models/pretrainedmodels-0.7.4.xyz /tmp/pip/cache/pretrainedmodels-0.7.4.tar.gz\n!cp ../input/segmentationmodelspytorch/segmentation_models/segmentation-models-pytorch-0.1.2.xyz /tmp/pip/cache/segmentation_models_pytorch-0.1.2.tar.gz\n!cp ../input/segmentationmodelspytorch/segmentation_models/timm-0.1.20-py3-none-any.whl /tmp/pip/cache/\n!cp ../input/segmentationmodelspytorch/segmentation_models/timm-0.2.1-py3-none-any.whl /tmp/pip/cache/\n!pip install --no-index --find-links /tmp/pip/cache/ efficientnet-pytorch\n!pip install --no-index --find-links /tmp/pip/cache/ segmentation-models-pytorch","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-07T16:51:36.515425Z","iopub.execute_input":"2021-11-07T16:51:36.516476Z","iopub.status.idle":"2021-11-07T16:51:59.238576Z","shell.execute_reply.started":"2021-11-07T16:51:36.516276Z","shell.execute_reply":"2021-11-07T16:51:59.237667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import GroupKFold\n\nfrom tqdm import tqdm\nimport os, gc\nimport random\nfrom PIL import Image\nimport tifffile as tiff\nimport cv2\nimport zipfile\nimport collections\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom skimage import exposure\nfrom bokeh.plotting import figure as bokeh_figure\nfrom bokeh.io import output_notebook, show, output_file\nfrom bokeh.models import ColumnDataSource, HoverTool, Panel\nfrom bokeh.models.widgets import Tabs\nfrom PIL import Image\nfrom sklearn import preprocessing\nfrom random import randint\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.optim as optim\n\nfrom segmentation_models_pytorch.unet import Unet\nfrom segmentation_models_pytorch.encoders import get_preprocessing_fn\n\nimport torchvision\nfrom torchvision import transforms\nfrom albumentations import *\nimport albumentations as A\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-07T16:51:59.241133Z","iopub.execute_input":"2021-11-07T16:51:59.24149Z","iopub.status.idle":"2021-11-07T16:52:04.25473Z","shell.execute_reply.started":"2021-11-07T16:51:59.241441Z","shell.execute_reply":"2021-11-07T16:52:04.253874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \n \nseed = 2020\nseed_everything(seed)\nsz = 512\nNFOLDS = 5\n\n#ImageNet\nmean = np.array([[[0.485, 0.456, 0.406]]])\nstd = np.array([[[0.229, 0.224, 0.225]]])","metadata":{"execution":{"iopub.status.busy":"2021-11-07T16:52:04.257249Z","iopub.execute_input":"2021-11-07T16:52:04.257753Z","iopub.status.idle":"2021-11-07T16:52:04.267179Z","shell.execute_reply.started":"2021-11-07T16:52:04.257714Z","shell.execute_reply":"2021-11-07T16:52:04.266294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DataFrame","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv('../input/sartorius-cell-instance-segmentation/sample_submission.csv')\ntest_df","metadata":{"execution":{"iopub.status.busy":"2021-11-07T16:52:04.268839Z","iopub.execute_input":"2021-11-07T16:52:04.26925Z","iopub.status.idle":"2021-11-07T16:52:04.303007Z","shell.execute_reply.started":"2021-11-07T16:52:04.269211Z","shell.execute_reply":"2021-11-07T16:52:04.302287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class Sartorius_Seg_Dataset(Dataset):\n    def __init__(self, df, preprocess_input=None, transform=None):\n        self.df = df\n        self.preprocess_input = preprocess_input\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        image_id = self.df.iloc[idx, 0]\n        img = cv2.imread(f'../input/sartorius-cell-instance-segmentation/test/{image_id}.png').astype(np.float32)\n        img = cv2.resize(img, (sz, sz)).astype(np.float32)\n                \n        if self.transform:\n            img = self.transform(image=img)['image']\n            \n        if self.preprocess_input:\n            img = self.preprocess_input(image=img)['image']\n        \n        img = img.transpose((2, 0, 1))\n        img = torch.from_numpy(img)\n            \n        return img, image_id","metadata":{"execution":{"iopub.status.busy":"2021-11-07T16:52:04.304134Z","iopub.execute_input":"2021-11-07T16:52:04.304436Z","iopub.status.idle":"2021-11-07T16:52:04.312301Z","shell.execute_reply.started":"2021-11-07T16:52:04.304404Z","shell.execute_reply":"2021-11-07T16:52:04.311128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"ENCODER_NAME = 'efficientnet-b0'\npreprocessing_fn = Lambda(image=get_preprocessing_fn(encoder_name=ENCODER_NAME, pretrained='imagenet'))\n\nclass Model(nn.Module):\n    def __init__(self):\n        super(Model, self).__init__()\n        self.model = Unet(\n            encoder_name='efficientnet-b0', \n            encoder_weights=None, \n            classes=1, \n            activation=None\n        )\n\n    def forward(self, images):\n        img_masks = self.model(images)\n        return img_masks","metadata":{"execution":{"iopub.status.busy":"2021-11-07T16:52:04.313691Z","iopub.execute_input":"2021-11-07T16:52:04.314223Z","iopub.status.idle":"2021-11-07T16:52:04.325917Z","shell.execute_reply.started":"2021-11-07T16:52:04.314187Z","shell.execute_reply":"2021-11-07T16:52:04.325055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)","metadata":{"execution":{"iopub.status.busy":"2021-11-07T16:52:04.327221Z","iopub.execute_input":"2021-11-07T16:52:04.327693Z","iopub.status.idle":"2021-11-07T16:52:04.335688Z","shell.execute_reply.started":"2021-11-07T16:52:04.327657Z","shell.execute_reply":"2021-11-07T16:52:04.334764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TH = 0.5\ntest_ds = Sartorius_Seg_Dataset(df=test_df, preprocess_input=preprocessing_fn)\ntest_dl = DataLoader(dataset=test_ds, batch_size=16, shuffle=False, num_workers=2)\npred_dict = {}\nseed_everything(seed)\n\nfor i, (img_btch, image_id) in enumerate(test_dl):\n    img_btch = img_btch.to(device, dtype=torch.float)\n    pred_mask_btch = 0\n    \n    for fold in range(NFOLDS):\n        print(f'===============Fold:{fold}===============')\n        model = Model()\n        model.load_state_dict(torch.load(f'../input/sartorius-efnetb0-unet-ver001/fold_{fold}.pth'))\n        model.to(device)\n        model.eval()\n        \n        with torch.no_grad():\n            pred_mask_btch += nn.Sigmoid()(model(img_btch.float())) / NFOLDS\n            \n    preds = (pred_mask_btch >= TH).cpu().numpy().astype(np.uint8)\n    \n    del img_btch, pred_mask_btch\n    gc.collect()\n            \n    for b in range(preds.shape[0]):\n        pred = preds[b]\n        pred = cv2.resize(np.squeeze(pred), (704, 520), interpolation=cv2.INTER_NEAREST)\n        pred = cv2.connectedComponents(pred.astype(np.uint8))[1]\n                \n        for p in range(1, pred.max()+1):\n            pred2 = np.where(pred==p, 1, 0)\n            x1 = np.min(np.where(pred2==1)[1])\n            x2 = np.max(np.where(pred2==1)[1])\n            y1 = np.min(np.where(pred2==1)[0])\n            y2 = np.max(np.where(pred2==1)[0])\n            if (x1 == x2) | (y1 == y2):\n                continue \n            rle = rle_encode(pred2)\n            pred_dict[f'{image_id[b]}_{p}'] = rle","metadata":{"execution":{"iopub.status.busy":"2021-11-07T16:57:13.8894Z","iopub.execute_input":"2021-11-07T16:57:13.889771Z","iopub.status.idle":"2021-11-07T16:57:18.616193Z","shell.execute_reply.started":"2021-11-07T16:57:13.889738Z","shell.execute_reply":"2021-11-07T16:57:18.615238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.DataFrame.from_dict(pred_dict, orient='index').reset_index().rename(columns={'index': 'id', 0: 'predicted'})\ntest_df.id = test_df.id.apply(lambda x: x.split('_')[0])\ntest_df = test_df.sort_values('id').reset_index(drop=True)\ntest_df","metadata":{"execution":{"iopub.status.busy":"2021-11-07T17:04:26.27805Z","iopub.execute_input":"2021-11-07T17:04:26.278367Z","iopub.status.idle":"2021-11-07T17:04:26.293888Z","shell.execute_reply.started":"2021-11-07T17:04:26.278337Z","shell.execute_reply":"2021-11-07T17:04:26.293028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-07T17:05:43.133567Z","iopub.execute_input":"2021-11-07T17:05:43.133913Z","iopub.status.idle":"2021-11-07T17:05:43.146704Z","shell.execute_reply.started":"2021-11-07T17:05:43.133877Z","shell.execute_reply":"2021-11-07T17:05:43.145956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}