{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/torch-1.10.0+cu111-cp37-cp37m-linux_x86_64.whl\n!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/torchvision-0.11.0+cu111-cp37-cp37m-linux_x86_64.whl\n!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/mmcv_full-1.5.3-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/mmcls-0.23.1-py2.py3-none-any.whl\n!pip install ../input/mmdetection/addict-2.4.0-py3-none-any.whl\n!pip install ../input/mmdetection/yapf-0.31.0-py2.py3-none-any.whl\n!pip install ../input/mmdetection/terminaltables-3.1.0-py3-none-any.whl\n!pip install ../input/mmdetection/einops-0.4.1-py3-none-any.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-07T14:54:15.649792Z","iopub.execute_input":"2022-09-07T14:54:15.650314Z","iopub.status.idle":"2022-09-07T14:57:05.686779Z","shell.execute_reply.started":"2022-09-07T14:54:15.650212Z","shell.execute_reply":"2022-09-07T14:57:05.685552Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p ./mmseg_data/images\n!mkdir -p ./mmseg_data/labels\n!mkdir -p ./mmseg_data/splits","metadata":{"execution":{"iopub.status.busy":"2022-09-07T14:57:33.807076Z","iopub.execute_input":"2022-09-07T14:57:33.807780Z","iopub.status.idle":"2022-09-07T14:57:36.785995Z","shell.execute_reply.started":"2022-09-07T14:57:33.807739Z","shell.execute_reply":"2022-09-07T14:57:36.784407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def enc2mask(mask_rle, shape):\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0::2], s[1::2])]\n    starts -= 1\n    ends = starts + lengths\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T","metadata":{"execution":{"iopub.status.busy":"2022-09-07T14:57:52.089629Z","iopub.execute_input":"2022-09-07T14:57:52.090381Z","iopub.status.idle":"2022-09-07T14:57:52.097572Z","shell.execute_reply.started":"2022-09-07T14:57:52.090339Z","shell.execute_reply":"2022-09-07T14:57:52.096630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport cv2\nimport numpy as np\n\n\ntrain = pd.read_csv(\"../input/hubmap-organ-segmentation/train.csv\")\nclasses_to_num = {'background':0, 'kidney':1, 'prostate':2, 'largeintestine':3, 'spleen':4, 'lung':5}\nfor i in range(len(train)):\n    row = train.iloc[i]\n    mask_rle = row.rle\n    organ = row.organ\n    class_num = classes_to_num[organ]\n    \n    m = enc2mask(mask_rle, (row.img_height, row.img_width))\n    m *= class_num\n    \n    cv2.imwrite('./mmseg_data/labels/' + str(row.id )+ '.png', m)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T14:57:56.249575Z","iopub.execute_input":"2022-09-07T14:57:56.249951Z","iopub.status.idle":"2022-09-07T14:58:15.123228Z","shell.execute_reply.started":"2022-09-07T14:57:56.249919Z","shell.execute_reply":"2022-09-07T14:58:15.122214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\nplt.imshow(cv2.imread(\"../input/hubmap-organ-segmentation/train_images/10044.tiff\", -1))","metadata":{"execution":{"iopub.status.busy":"2022-09-07T14:58:28.249482Z","iopub.execute_input":"2022-09-07T14:58:28.249883Z","iopub.status.idle":"2022-09-07T14:58:30.026753Z","shell.execute_reply.started":"2022-09-07T14:58:28.249837Z","shell.execute_reply":"2022-09-07T14:58:30.025837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(cv2.imread(\"./mmseg_data/labels/10044.png\",  cv2.IMREAD_GRAYSCALE))","metadata":{"execution":{"iopub.status.busy":"2022-09-07T14:58:34.632235Z","iopub.execute_input":"2022-09-07T14:58:34.632608Z","iopub.status.idle":"2022-09-07T14:58:35.557048Z","shell.execute_reply.started":"2022-09-07T14:58:34.632575Z","shell.execute_reply":"2022-09-07T14:58:35.556127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r ../input/hubmap-organ-segmentation/train_images/* /kaggle/working/mmseg_data/images/","metadata":{"execution":{"iopub.status.busy":"2022-09-07T14:59:44.025894Z","iopub.execute_input":"2022-09-07T14:59:44.026290Z","iopub.status.idle":"2022-09-07T15:01:17.266123Z","shell.execute_reply.started":"2022-09-07T14:59:44.026258Z","shell.execute_reply":"2022-09-07T15:01:17.264699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nimport numpy as np\nimport cv2\nimport os\nfrom sklearn.model_selection import StratifiedKFold\n\nFold = 5\nall_mask_files = glob(\"./mmseg_data/labels/*\")\nmasks = []\nnum_mask = np.zeros((6,Fold))\n\nfor i in range(len(all_mask_files)):\n    mask = cv2.imread(all_mask_files[i])\n    masks.append(mask.max())\n\nsplit = list(StratifiedKFold(n_splits=Fold, shuffle=True, random_state=2022).split(all_mask_files, masks))\nfor fold, (train_idx, valid_idx) in enumerate(split):\n    for i in valid_idx:\n        num_mask[masks[i]]+=1\n    with open(f\"./mmseg_data/splits/fold_{fold}.txt\", \"w\") as f:\n        for idx in train_idx:\n            f.write(os.path.basename(all_mask_files[idx])[:-4] + \"\\n\")\n    with open(f\"./mmseg_data/splits/valid_{fold}.txt\", \"w\") as f:\n        for idx in valid_idx:\n            f.write(os.path.basename(all_mask_files[idx])[:-4] + \"\\n\")\nprint(num_mask)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T15:01:25.260271Z","iopub.execute_input":"2022-09-07T15:01:25.260919Z","iopub.status.idle":"2022-09-07T15:01:53.429395Z","shell.execute_reply.started":"2022-09-07T15:01:25.260877Z","shell.execute_reply":"2022-09-07T15:01:53.427217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Newest version of mmsegmentation.","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/mmsegm/mmsegmentation-master /kaggle/working/ && cd /kaggle/working/mmsegmentation-master && pip install -e . && cd ..","metadata":{"execution":{"iopub.status.busy":"2022-09-07T15:02:22.014675Z","iopub.execute_input":"2022-09-07T15:02:22.015173Z","iopub.status.idle":"2022-09-07T15:02:41.086905Z","shell.execute_reply.started":"2022-09-07T15:02:22.015126Z","shell.execute_reply":"2022-09-07T15:02:41.085668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Larger backbone and more iters may lead to better score!","metadata":{}},{"cell_type":"code","source":"%%bash\n\ncat <<EOT >> /kaggle/working/fold_0.py\n\nnorm_cfg = dict(type='SyncBN', requires_grad=True)\nmodel = dict(\n    type='EncoderDecoder',\n    pretrained=None,\n    backbone=dict(\n        type='MixVisionTransformer',\n        in_channels=3,\n        embed_dims=32,\n        num_stages=4,\n        num_layers=[2, 2, 2, 2],\n        num_heads=[1, 2, 5, 8],\n        patch_sizes=[7, 3, 3, 3],\n        sr_ratios=[8, 4, 2, 1],\n        out_indices=(0, 1, 2, 3),\n        mlp_ratio=4,\n        qkv_bias=True,\n        drop_rate=0.0,\n        attn_drop_rate=0.0,\n        drop_path_rate=0.1),\n    decode_head=dict(\n        type='SegformerHead',\n        in_channels=[32, 64, 160, 256],\n        in_index=[0, 1, 2, 3],\n        channels=256,\n        dropout_ratio=0.1,\n        num_classes=6,\n        norm_cfg=norm_cfg,\n        align_corners=False,\n        loss_decode=dict(type='DiceLoss', loss_name='loss_dice', loss_weight=1.0)),\n    # model training and testing settings\n    train_cfg=dict(),\n#     test_cfg=dict(mode='whole')\n    )\n\n# dataset settings\ndataset_type = 'CustomDataset'\ndata_root = '/kaggle/working/mmseg_data/'\nclasses = ['background', 'kidney', 'prostate', 'largeintestine', 'spleen', 'lung']\npalette = [[0,0,0], [255,0,0], [0,255,0], [0,0,255], [255,255,0], [255,0,255]]\nimg_norm_cfg = dict(mean=[196.869, 190.186, 194.802], std=[63.010, 66.765, 65.745], to_rgb=True)\nsize = 512\n\ntrain_pipeline = [\n    dict(type='LoadImageFromFile'),\n    dict(type='LoadAnnotations'),\n    dict(type='Resize', img_scale=(size*2, size*2), keep_ratio=True),\n    dict(type='RandomCrop', crop_size=(size, size), cat_max_ratio=0.75),\n    dict(type='RandomFlip', prob=0.5, direction='horizontal'),\n    dict(type='RandomFlip', prob=0.5, direction='vertical'),\n    dict(type='Normalize', **img_norm_cfg),\n    dict(type='DefaultFormatBundle'),\n    dict(type='Collect', keys=['img', 'gt_semantic_seg']),\n]\ntest_pipeline = [\n    dict(type='LoadImageFromFile'),\n    dict(\n        type='MultiScaleFlipAug',\n        img_scale=(size*2, size*2),\n        flip=False,\n        transforms=[\n            dict(type='Resize', keep_ratio=True),\n            dict(type='RandomFlip'),\n            dict(type='Normalize', **img_norm_cfg),\n            dict(type='ImageToTensor', keys=['img']),\n            dict(type='Collect', keys=['img']),\n        ])\n]\ntest_cfg = dict(mode='slide', crop_size=(size, size), stride=(size//2, size//2))\n\ndata = dict(\n    samples_per_gpu=8,\n    workers_per_gpu=4,\n    train=dict(\n        type=dataset_type,\n        data_root=data_root,\n        img_dir='images',\n        ann_dir='labels',\n        img_suffix=\".tiff\",\n        seg_map_suffix='.png',\n        split=\"splits/fold_0.txt\",\n        classes=classes,\n        palette=palette,\n        pipeline=train_pipeline),\n    val=dict(\n        type=dataset_type,\n        data_root=data_root,\n        img_dir='images',\n        ann_dir='labels',\n        img_suffix=\".tiff\",\n        seg_map_suffix='.png',\n        split=\"splits/valid_0.txt\",\n        classes=classes,\n        palette=palette,\n        pipeline=test_pipeline),\n    test=dict(\n        type=dataset_type,\n        data_root=data_root,\n        test_mode=True,\n        img_dir='test/images',\n        ann_dir='test/labels',\n        img_suffix=\".tiff\",\n        seg_map_suffix='.png',\n        classes=classes,\n        palette=palette,\n        pipeline=test_pipeline))\n\n# yapf:disable\nlog_config = dict(\n    interval=1000,\n    hooks=[\n        dict(type='TextLoggerHook', by_epoch=False),\n    ])\n# yapf:enable\ndist_params = dict(backend='nccl')\nlog_level = 'INFO'\nload_from = None\nresume_from = None\nworkflow = [('train', 1)]\ncudnn_benchmark = True\n\ntotal_iters = 12000\n# optimizer\noptimizer = dict(type='AdamW', lr=5e-4, betas=(0.9, 0.999), weight_decay=0.05)\noptimizer_config = dict(type='Fp16OptimizerHook', loss_scale='dynamic')\n# learning policy\nlr_config = dict(policy='poly',\n                 warmup='linear',\n                 warmup_iters=1000,\n                 warmup_ratio=1e-6,\n                 power=1.0, min_lr=0.0, by_epoch=False)\n# runtime settings\nfind_unused_parameters = True\nrunner = dict(type = 'IterBasedRunner', max_iters = total_iters)\ncheckpoint_config = dict(by_epoch=False, interval=-1, save_optimizer=False)\nevaluation = dict(by_epoch=False, interval=200, metric='mDice', pre_eval=True, save_best='mDice')\nfp16 = dict()\nwork_dir = './fold_0'\nEOT","metadata":{"execution":{"iopub.status.busy":"2022-09-07T15:06:59.769117Z","iopub.execute_input":"2022-09-07T15:06:59.769534Z","iopub.status.idle":"2022-09-07T15:06:59.803506Z","shell.execute_reply.started":"2022-09-07T15:06:59.769498Z","shell.execute_reply":"2022-09-07T15:06:59.802276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train with api","metadata":{}},{"cell_type":"code","source":"!python /kaggle/working/mmsegmentation-master/tools/train.py /kaggle/working/fold_0.py","metadata":{"execution":{"iopub.status.busy":"2022-09-07T15:07:40.868661Z","iopub.execute_input":"2022-09-07T15:07:40.870528Z","iopub.status.idle":"2022-09-07T15:31:37.396303Z","shell.execute_reply.started":"2022-09-07T15:07:40.870477Z","shell.execute_reply":"2022-09-07T15:31:37.394051Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]}]}