{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Begin with mmsegmentation🥳\n* Simple baseline for training\n* Look forward to your upvote if it helps!🤗","metadata":{}},{"cell_type":"code","source":"!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/torch-1.10.0+cu111-cp37-cp37m-linux_x86_64.whl\n!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/torchvision-0.11.0+cu111-cp37-cp37m-linux_x86_64.whl\n!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/mmcv_full-1.5.3-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/mmcls-0.23.1-py2.py3-none-any.whl\n!pip install ../input/mmdetection/addict-2.4.0-py3-none-any.whl\n!pip install ../input/mmdetection/yapf-0.31.0-py2.py3-none-any.whl\n!pip install ../input/mmdetection/terminaltables-3.1.0-py3-none-any.whl\n!pip install ../input/mmdetection/einops-0.4.1-py3-none-any.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-24T16:21:13.683680Z","iopub.execute_input":"2022-07-24T16:21:13.684376Z","iopub.status.idle":"2022-07-24T16:24:38.956128Z","shell.execute_reply.started":"2022-07-24T16:21:13.684285Z","shell.execute_reply":"2022-07-24T16:24:38.954548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p ./mmseg_data/images\n!mkdir -p ./mmseg_data/labels\n!mkdir -p ./mmseg_data/splits","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:24:38.959199Z","iopub.execute_input":"2022-07-24T16:24:38.959748Z","iopub.status.idle":"2022-07-24T16:24:41.683502Z","shell.execute_reply.started":"2022-07-24T16:24:38.959693Z","shell.execute_reply":"2022-07-24T16:24:41.681753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thanks for [THE DEVASTATOR](https://www.kaggle.com/thedevastator) 's great work of data preprocessing!\n* [Dataset (512 x 512)](https://www.kaggle.com/datasets/thedevastator/hubmap-2022-512x512/)\n\n* [Dataset (256 x 256)](https://www.kaggle.com/datasets/thedevastator/hubmap-2022-256x256/)\n\n* [Dataset (128 x 128)](https://www.kaggle.com/datasets/thedevastator/hubmap-2022-128x128/settings)\n\nHere I used 256X256 data.","metadata":{}},{"cell_type":"code","source":"!cp -r /kaggle/input/hubmap-2022-256x256/train/* /kaggle/working/mmseg_data/images/\n!cp -r /kaggle/input/hubmap-2022-256x256/masks/* /kaggle/working/mmseg_data/labels/","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:24:41.689698Z","iopub.execute_input":"2022-07-24T16:24:41.693133Z","iopub.status.idle":"2022-07-24T16:25:06.133333Z","shell.execute_reply.started":"2022-07-24T16:24:41.693081Z","shell.execute_reply":"2022-07-24T16:25:06.131748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nimport numpy as np\nimport cv2\nimport os\nfrom sklearn.model_selection import StratifiedKFold\nFold = 10\nall_mask_files = glob(\"./mmseg_data/labels/*\")\nmasks = []\nnum_wo_mask = np.zeros(Fold)\nnum_w_mask = np.zeros(Fold)\nfor i in range(len(all_mask_files)):\n    mask = cv2.imread(all_mask_files[i])\n    if mask.sum()==0:\n        masks.append(0)\n    else:\n        masks.append(1)\nsplit = list(StratifiedKFold(n_splits=Fold, shuffle=True, random_state=2022).split(all_mask_files, masks))\nfor fold, (train_idx, valid_idx) in enumerate(split):\n    for i in valid_idx:\n        if masks[i]==0:\n            num_wo_mask[fold]+=1\n        else:\n            num_w_mask[fold]+=1\n    with open(f\"./mmseg_data/splits/fold_{fold}.txt\", \"w\") as f:\n        for idx in train_idx:\n            f.write(os.path.basename(all_mask_files[idx])[:-4] + \"\\n\")\n    with open(f\"./mmseg_data/splits/valid_{fold}.txt\", \"w\") as f:\n        for idx in valid_idx:\n            f.write(os.path.basename(all_mask_files[idx])[:-4] + \"\\n\")\nprint(num_wo_mask)\nprint(num_w_mask)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:25:06.137942Z","iopub.execute_input":"2022-07-24T16:25:06.138332Z","iopub.status.idle":"2022-07-24T16:25:09.476187Z","shell.execute_reply.started":"2022-07-24T16:25:06.138297Z","shell.execute_reply":"2022-07-24T16:25:09.474719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Newest version of mmsegmentation.","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/mmsegm/mmsegmentation-master /kaggle/working/ && cd /kaggle/working/mmsegmentation-master && pip install -e . && cd ..","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:25:09.478180Z","iopub.execute_input":"2022-07-24T16:25:09.478890Z","iopub.status.idle":"2022-07-24T16:25:29.619258Z","shell.execute_reply.started":"2022-07-24T16:25:09.478844Z","shell.execute_reply":"2022-07-24T16:25:29.617580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Larger backbone and more iters may lead to better score!","metadata":{}},{"cell_type":"code","source":"%%bash\n\ncat <<EOT >> /kaggle/working/config.py\n\nnorm_cfg = dict(type='SyncBN', requires_grad=True)\nmodel = dict(\n    type='EncoderDecoder',\n    pretrained=None,\n    backbone=dict(\n        type='UNet',\n        in_channels=3,\n        base_channels=64,\n        num_stages=5,\n        strides=(1, 1, 1, 1, 1),\n        enc_num_convs=(2, 2, 2, 2, 2),\n        dec_num_convs=(2, 2, 2, 2),\n        downsamples=(True, True, True, True),\n        enc_dilations=(1, 1, 1, 1, 1),\n        dec_dilations=(1, 1, 1, 1),\n        with_cp=False,\n        conv_cfg=None,\n        norm_cfg=norm_cfg,\n        act_cfg=dict(type='ReLU'),\n        upsample_cfg=dict(type='InterpConv'),\n        norm_eval=False),\n    decode_head=dict(\n        type='FCNHead',\n        in_channels=64,\n        in_index=4,\n        channels=64,\n        num_convs=1,\n        concat_input=False,\n        dropout_ratio=0.1,\n        num_classes=2,\n        norm_cfg=norm_cfg,\n        align_corners=False,\n        loss_decode=dict(\n            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0)),\n    auxiliary_head=dict(\n        type='FCNHead',\n        in_channels=128,\n        in_index=3,\n        channels=64,\n        num_convs=1,\n        concat_input=False,\n        dropout_ratio=0.1,\n        num_classes=2,\n        norm_cfg=norm_cfg,\n        align_corners=False,\n        loss_decode=dict(\n            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=0.4)),\n    train_cfg=dict(),\n    test_cfg=dict(mode='whole'))\n\n# dataset settings\ndataset_type = 'CustomDataset'\ndata_root = '/kaggle/working/mmseg_data/'\nclasses = ['BG', 'FTU']\npalette = [[0,0,0], [255,0,0]]\nimg_norm_cfg = dict(mean=[196.869, 190.186, 194.802], std=[63.010, 66.765, 65.745], to_rgb=True)\nsize = 256\n\ntrain_pipeline = [\n    dict(type='LoadImageFromFile'),\n    dict(type='LoadAnnotations'),\n    dict(type='Resize', img_scale=(size, size), keep_ratio=True),\n    dict(type='RandomFlip', prob=0.5, direction='horizontal'),\n    dict(type='Normalize', **img_norm_cfg),\n    dict(type='Pad', size=(size, size), pad_val=0, seg_pad_val=255),\n    dict(type='DefaultFormatBundle'),\n    dict(type='Collect', keys=['img', 'gt_semantic_seg']),\n]\ntest_pipeline = [\n    dict(type='LoadImageFromFile'),\n    dict(\n        type='MultiScaleFlipAug',\n        img_scale=(size, size),\n        flip=False,\n        transforms=[\n            dict(type='Resize', keep_ratio=True),\n            dict(type='RandomFlip'),\n            dict(type='Normalize', **img_norm_cfg),\n            dict(type='Pad', size=(size, size), pad_val=0, seg_pad_val=255),\n            dict(type='ImageToTensor', keys=['img']),\n            dict(type='Collect', keys=['img']),\n        ])\n]\ndata = dict(\n    samples_per_gpu=8,\n    workers_per_gpu=4,\n    train=dict(\n        type=dataset_type,\n        data_root=data_root,\n        img_dir='images',\n        ann_dir='labels',\n        img_suffix=\".png\",\n        seg_map_suffix='.png',\n        split=\"splits/fold_0.txt\",\n        classes=classes,\n        palette=palette,\n        pipeline=train_pipeline),\n    val=dict(\n        type=dataset_type,\n        data_root=data_root,\n        img_dir='images',\n        ann_dir='labels',\n        img_suffix=\".png\",\n        seg_map_suffix='.png',\n        split=\"splits/valid_0.txt\",\n        classes=classes,\n        palette=palette,\n        pipeline=test_pipeline),\n    test=dict(\n        type=dataset_type,\n        data_root=data_root,\n        test_mode=True,\n        img_dir='test/images',\n        ann_dir='test/labels',\n        img_suffix=\".png\",\n        seg_map_suffix='.png',\n        classes=classes,\n        palette=palette,\n        pipeline=test_pipeline))\n\n# yapf:disable\nlog_config = dict(\n    interval=50,\n    hooks=[\n        dict(type='TextLoggerHook', by_epoch=False),\n    ])\n# yapf:enable\ndist_params = dict(backend='nccl')\nlog_level = 'INFO'\nload_from = None\nresume_from = None\nworkflow = [('train', 1)]\ncudnn_benchmark = True\n\ntotal_iters = 5000\n# optimizer\noptimizer = dict(type='AdamW', lr=1e-3, betas=(0.9, 0.999), weight_decay=0.05)\noptimizer_config = dict(type='Fp16OptimizerHook', loss_scale='dynamic')\n# learning policy\nlr_config = dict(policy='poly',\n                 warmup='linear',\n                 warmup_iters=500,\n                 warmup_ratio=1e-6,\n                 power=1.0, min_lr=0.0, by_epoch=False)\n# runtime settings\nfind_unused_parameters = True\nrunner = dict(type = 'IterBasedRunner', max_iters = total_iters)\ncheckpoint_config = dict(by_epoch=False, interval=-1, save_optimizer=False)\nevaluation = dict(by_epoch=False, interval=500, metric='mDice', pre_eval=True)\nfp16 = dict()\nwork_dir = './baseline'\nEOT","metadata":{"execution":{"iopub.status.busy":"2022-07-24T17:47:59.779202Z","iopub.execute_input":"2022-07-24T17:47:59.779702Z","iopub.status.idle":"2022-07-24T17:47:59.813400Z","shell.execute_reply.started":"2022-07-24T17:47:59.779604Z","shell.execute_reply":"2022-07-24T17:47:59.811968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train with api","metadata":{}},{"cell_type":"code","source":"!python /kaggle/working/mmsegmentation-master/tools/train.py /kaggle/working/config.py","metadata":{"execution":{"iopub.status.busy":"2022-07-24T17:48:00.360952Z","iopub.execute_input":"2022-07-24T17:48:00.361494Z","iopub.status.idle":"2022-07-24T18:23:28.509965Z","shell.execute_reply.started":"2022-07-24T17:48:00.361458Z","shell.execute_reply":"2022-07-24T18:23:28.508353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}