{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Install deepflash2 and dependencies\nimport sys\nsys.path.append(\"../input/zarrkaggleinstall\")\nsys.path.append(\"../input/segmentation-models-pytorch-install\")\n!pip install -q --no-deps ../input/deepflash2-lfs\nimport cv2, torch, zarr, tifffile, pandas as pd, gc\nfrom fastai.vision.all import *\nfrom deepflash2.all import *\nimport segmentation_models_pytorch as smp","metadata":{"_uuid":"28ffcd74-04c1-46b4-9d7b-560aa3046c0f","_cell_guid":"65a7a917-d4f4-4919-b0e6-b66339bb3a46","collapsed":false,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/bguberfain/memory-aware-rle-encoding\n#with transposed mask\ndef rle_encode_less_memory(img):\n    #the image should be transposed\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)\n\ndef load_model_weights(model, file, strict=True):\n    state = torch.load(file, map_location='cpu')\n    stats = state['stats']\n    model_state = state['model']\n    model.load_state_dict(model_state, strict=strict)\n    return model, stats","metadata":{"_uuid":"af737c4d-d53d-47d0-aadf-6a76f3b74434","_cell_guid":"0a86ba45-ff1b-4e60-940a-afe2f2ba2f0c","collapsed":false,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://matjesg.github.io/deepflash2/data.html#BaseDataset\n# Handling of different input shapes\n@patch\ndef read_img(self:BaseDataset, *args, **kwargs):\n    image = tifffile.imread(args[0])\n    if len(image.shape) == 5:\n        image = image.squeeze().transpose(1, 2, 0)\n    elif image.shape[0] == 3:\n        image = image.transpose(1, 2, 0)\n    return image\n\n# https://matjesg.github.io/deepflash2/data.html#DeformationField\n# Adding normalization (divide by 255)\n@patch\ndef apply(self:DeformationField, data, offset=(0, 0), pad=(0, 0), order=1):\n    \"Apply deformation field to image using interpolation\"\n    outshape = tuple(int(s - p) for (s, p) in zip(self.shape, pad))\n    coords = [np.squeeze(d).astype('float32').reshape(*outshape) for d in self.get(offset, pad)]\n    # Get slices to avoid loading all data (.zarr files)\n    sl = []\n    for i in range(len(coords)):\n        cmin, cmax = int(coords[i].min()), int(coords[i].max())\n        dmax = data.shape[i]\n        if cmin<0: \n            cmax = max(-cmin, cmax)\n            cmin = 0 \n        elif cmax>dmax:\n            cmin = min(cmin, 2*dmax-cmax)\n            cmax = dmax\n            coords[i] -= cmin\n        else: coords[i] -= cmin\n        sl.append(slice(cmin, cmax))    \n    if len(data.shape) == len(self.shape) + 1:\n        \n        ## Channel order change in V12\n        tile = np.empty((*outshape, data.shape[-1]))\n        for c in range(data.shape[-1]):\n            # Adding divide\n            tile[..., c] = cv2.remap(data[sl[0],sl[1], c]/255, coords[1],coords[0], interpolation=order, borderMode=cv2.BORDER_REFLECT)\n    else:\n        tile = cv2.remap(data[sl[0], sl[1]], coords[1], coords[0], interpolation=order, borderMode=cv2.BORDER_REFLECT)\n    return tile","metadata":{"_uuid":"1505be1c-0641-438d-8af5-795d6687b3c1","_cell_guid":"053d4672-735c-404f-8200-3497faf4c751","collapsed":false,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CONFIG():\n    \n    # data paths\n    data_path = Path('../input/hubmap-kidney-segmentation')\n    models_path = Path('../input/hubmap-efficient-sampling-deepflash2-train')\n    models_file = np.array([x for x in models_path.iterdir() if x.name.startswith('u')])\n    \n    # deepflash2 dataset (https://matjesg.github.io/deepflash2/data.html#TileDataset)\n    scale = 3 # zoom facor (zoom out)\n    tile_shape = (512, 512)\n    padding = (100,100) # Border overlap for prediction\n\n    # pytorch model (https://github.com/qubvel/segmentation_models.pytorch)\n    encoder_name = \"efficientnet-b4\"\n    encoder_weights = None\n    in_channels = 3\n    classes = 2\n    \n    # dataloader \n    batch_size = 16\n    \n    # prediction threshold\n    threshold = 0.5\n    \ncfg = CONFIG()","metadata":{"_uuid":"bc547395-2534-4a65-9eab-2cc741118abc","_cell_guid":"a0183676-4249-4baf-9b0c-492878ae62f7","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cfg.models_file)\nprint(len(cfg.models_file))\n\n# Sample submissions for ids\ndf_sample = pd.read_csv(cfg.data_path/'sample_submission.csv',  index_col='id')\nnames,preds = [],[]\nsub = None","metadata":{"_uuid":"f3c3f744-4286-49e9-8269-8c53e94b1e61","_cell_guid":"24643b81-c8fe-482c-82a8-7de8d4feea5a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"names,preds = [],[]\n\n\nfor idx, _ in df_sample.iterrows():\n    print(f'###### File {idx} ######')\n    f = cfg.data_path/'test'/f'{idx}.tiff'\n    \n    # Create deepflash2 dataset (including tiling and file conversion)\n    ds = TileDataset([f], scale=cfg.scale, tile_shape=cfg.tile_shape, padding=cfg.padding)\n    shape = ds.data[f.name].shape\n    print('Shape:', shape)\n    \n    names.append(idx)\n    msk = None\n    \n    print('Prediction')\n    for model_path in cfg.models_file:\n        model = smp.Unet(encoder_name=cfg.encoder_name, \n                         encoder_weights=cfg.encoder_weights, \n                         in_channels=cfg.in_channels, \n                         classes=cfg.classes)\n        model, stats = load_model_weights(model, model_path)\n        batch_tfms = [Normalize.from_stats(*stats)]\n        \n        dls = DataLoaders.from_dsets(ds, batch_size=cfg.batch_size, after_batch=batch_tfms, shuffle=False, drop_last=False)\n        if torch.cuda.is_available(): dls.cuda(), model.cuda()\n        \n        learn = Learner(dls, model, loss_func='')\n        \n        # Predict tiles, see https://matjesg.github.io/deepflash2/learner.html#Learner.predict_tiles\n        res = learn.predict_tiles(dl=dls.train, path='/kaggle/temp/', use_tta=False, uncertainty_estimates=False)\n        if msk is None:\n            msk = res[0][f.name][..., 1]\n        else:\n            msk += res[0][f.name][..., 1]\n#         print(msk[:3, :3])\n        \n        del model, stats, learn\n    \n    msk = msk/len(cfg.models_file)\n    msk = (msk > cfg.threshold).astype(np.uint8)\n    \n    # Resize image and create RLE\n    print('Rezising')\n    msk = cv2.resize(msk, (shape[1], shape[0]))\n    rle = rle_encode_less_memory(msk)\n    preds.append(rle)\n        \n    # Plot Result\n    print('Plotting')\n    fig, ax = plt.subplots(figsize=(15,15))\n    ax.imshow(cv2.resize(msk, (1024, 1024)))\n    plt.show()\n    \n    # Overwrite store (reduce disk usage)\n    _ = [shutil.rmtree(p, ignore_errors=True) for p in Path('/kaggle/temp/').iterdir()]\n    _ = [shutil.rmtree(p, ignore_errors=True) for p in Path('/tmp/').iterdir() if p.name.startswith('zarr')]","metadata":{"_uuid":"1714472b-dd86-48d3-837e-85a81d1af08c","_cell_guid":"5d89daea-c839-4ee1-89e9-6435f847510d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'id':names,'predicted':preds}).set_index('id')\ndf_sample.loc[df.index.values] = df.values  \ndf_sample.to_csv('submission.csv')","metadata":{"_uuid":"afaaf56b-a4de-4a19-aa87-16276dfd5c0d","_cell_guid":"6f791ac9-8548-4794-ac6d-9ad467a6306a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}