{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Inference Notebook, follow up [Training Notebook](https://www.kaggle.com/mlneo07/swin-transformer-fasterrcnn-training) ","metadata":{}},{"cell_type":"markdown","source":"# **Install Offline MMDetection**","metadata":{"papermill":{"duration":0.016492,"end_time":"2021-12-21T15:54:28.099845","exception":false,"start_time":"2021-12-21T15:54:28.083353","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install '../input/pytorch-190/torch-1.9.0+cu111-cp37-cp37m-linux_x86_64.whl'","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":111.357617,"end_time":"2021-12-21T15:56:19.471509","exception":false,"start_time":"2021-12-21T15:54:28.113892","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T04:51:51.53575Z","iopub.execute_input":"2022-01-10T04:51:51.536071Z","iopub.status.idle":"2022-01-10T04:53:32.925302Z","shell.execute_reply.started":"2022-01-10T04:51:51.535977Z","shell.execute_reply":"2022-01-10T04:53:32.924393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install '/kaggle/input/mmdetectionv2140/addict-2.4.0-py3-none-any.whl' \n!pip install '/kaggle/input/mmdetectionv2140/yapf-0.31.0-py2.py3-none-any.whl' \n!pip install '/kaggle/input/mmdetectionv2140/terminal-0.4.0-py3-none-any.whl'\n!pip install '/kaggle/input/mmdetectionv2140/terminaltables-3.1.0-py3-none-any.whl'\n!pip install '/kaggle/input/mmdetection-v217/mmdetection/mmcv_full-1.3.x-py2.py3-none-any/mmcv_full-1.3.16-cp37-cp37m-manylinux1_x86_64.whl'\n!pip install '/kaggle/input/mmdetectionv2140/pycocotools-2.0.2/pycocotools-2.0.2'\n!pip install '/kaggle/input/mmdetectionv2140/mmpycocotools-12.0.3/mmpycocotools-12.0.3'\n\n!rm -rf mmdetection\n\n!cp -r /kaggle/input/mmdetection-v217/mmdetection/mmdetection-2.18.0 /kaggle/working/\n!mv /kaggle/working/mmdetection-2.18.0 /kaggle/working/mmdetection\n%cd /kaggle/working/mmdetection\n!pip install -e .\n\n%cd ..\n\n# !rm -rf mmdetection\n# !git clone https://github.com/open-mmlab/mmdetection.git /kaggle/working/mmdetection","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":250.419371,"end_time":"2021-12-21T16:00:29.906187","exception":false,"start_time":"2021-12-21T15:56:19.486816","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T04:53:32.927517Z","iopub.execute_input":"2022-01-10T04:53:32.927742Z","iopub.status.idle":"2022-01-10T04:57:44.24719Z","shell.execute_reply.started":"2022-01-10T04:53:32.927715Z","shell.execute_reply":"2022-01-10T04:57:44.246322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install openmim\n# !mim install mmdet","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":0.03379,"end_time":"2021-12-21T16:00:29.968409","exception":false,"start_time":"2021-12-21T16:00:29.934619","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T04:57:44.249254Z","iopub.execute_input":"2022-01-10T04:57:44.249468Z","iopub.status.idle":"2022-01-10T04:57:44.256831Z","shell.execute_reply.started":"2022-01-10T04:57:44.249439Z","shell.execute_reply":"2022-01-10T04:57:44.25607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Import Libraries**","metadata":{"papermill":{"duration":0.02739,"end_time":"2021-12-21T16:00:30.023448","exception":false,"start_time":"2021-12-21T16:00:29.996058","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import sys, os\nsys.path.append('./mmdetection')\nsys.path.append('../input/tensorflow-great-barrier-reef/greatbarrierreef')\nimport numpy as np\nimport mmdet\nimport mmcv\nfrom mmdet.datasets import build_dataset\nfrom mmdet.models import build_detector\nfrom mmdet.apis import train_detector\nfrom mmdet.apis import inference_detector, init_detector, show_result_pyplot, set_random_seed","metadata":{"papermill":{"duration":23.665185,"end_time":"2021-12-21T16:00:53.716392","exception":false,"start_time":"2021-12-21T16:00:30.051207","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T04:57:44.261137Z","iopub.execute_input":"2022-01-10T04:57:44.261452Z","iopub.status.idle":"2022-01-10T04:58:03.18352Z","shell.execute_reply.started":"2022-01-10T04:57:44.26142Z","shell.execute_reply":"2022-01-10T04:58:03.182736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed = 42\nset_random_seed(seed, deterministic=False)","metadata":{"papermill":{"duration":0.036796,"end_time":"2021-12-21T16:00:53.781249","exception":false,"start_time":"2021-12-21T16:00:53.744453","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T04:58:03.185029Z","iopub.execute_input":"2022-01-10T04:58:03.185298Z","iopub.status.idle":"2022-01-10T04:58:03.192053Z","shell.execute_reply.started":"2022-01-10T04:58:03.185263Z","shell.execute_reply":"2022-01-10T04:58:03.191122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Model**","metadata":{"papermill":{"duration":0.027024,"end_time":"2021-12-21T16:00:53.890916","exception":false,"start_time":"2021-12-21T16:00:53.863892","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\n\n%%writefile ./mmdetection/configs/swin/TFGBR_swin_base_faster_rcnn_fp16.py\n\n_base_ = [\n    '../_base_/models/cascade_rcnn_r50_fpn.py',\n    '../_base_/datasets/coco_detection.py',\n    '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py'\n]\npretrained = 'https://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_small_patch4_window7_224.pth'\nmodel = dict(\n    backbone=dict(\n        _delete_=True,\n        type='SwinTransformer',\n        embed_dims=96,\n        depths=[2, 2, 18, 2],\n        num_heads=[3, 6, 12, 24],\n        window_size=7,\n        mlp_ratio=4,\n        qkv_bias=True,\n        qk_scale=None,\n        drop_rate=0.,\n        attn_drop_rate=0.,\n        drop_path_rate=0.3,\n        patch_norm=True,\n        out_indices=(0, 1, 2, 3),\n        with_cp=False,\n        convert_weights=True,\n        init_cfg=dict(type='Pretrained', checkpoint=pretrained)),\n    neck=dict(in_channels=[96, 192, 384, 768]),\n    roi_head=dict(\n        type='CascadeRoIHead',\n        num_stages=3,\n        stage_loss_weights=[1, 0.5, 0.25],\n        bbox_roi_extractor=dict(\n            type='SingleRoIExtractor',\n            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),\n            out_channels=256,\n            featmap_strides=[4, 8, 16, 32]),\n        bbox_head=[\n            dict(\n                type='Shared2FCBBoxHead',\n                in_channels=256,\n                fc_out_channels=1024,\n                roi_feat_size=7,\n                num_classes=1,\n                bbox_coder=dict(\n                    type='DeltaXYWHBBoxCoder',\n                    target_means=[0., 0., 0., 0.],\n                    target_stds=[0.1, 0.1, 0.2, 0.2]),\n                reg_class_agnostic=True,\n                loss_cls=dict(\n                    type='CrossEntropyLoss',\n                    use_sigmoid=False,\n                    loss_weight=1.0),\n                reg_decoded_bbox=True,\n                loss_bbox=dict(type='GIoULoss', loss_weight=10.0)),\n            dict(\n                type='Shared2FCBBoxHead',\n                in_channels=256,\n                fc_out_channels=1024,\n                roi_feat_size=7,\n                num_classes=1,\n                bbox_coder=dict(\n                    type='DeltaXYWHBBoxCoder',\n                    target_means=[0., 0., 0., 0.],\n                    target_stds=[0.05, 0.05, 0.1, 0.1]),\n                reg_class_agnostic=True,\n                loss_cls=dict(\n                    type='CrossEntropyLoss',\n                    use_sigmoid=False,\n                    loss_weight=1.0),\n                reg_decoded_bbox=True,\n                loss_bbox=dict(type='GIoULoss', loss_weight=10.0)),\n            dict(\n                type='Shared2FCBBoxHead',\n                in_channels=256,\n                fc_out_channels=1024,\n                roi_feat_size=7,\n                num_classes=1,\n                bbox_coder=dict(\n                    type='DeltaXYWHBBoxCoder',\n                    target_means=[0., 0., 0., 0.],\n                    target_stds=[0.033, 0.033, 0.067, 0.067]),\n                reg_class_agnostic=True,\n                loss_cls=dict(\n                    type='CrossEntropyLoss',\n                    use_sigmoid=False,\n                    loss_weight=1.0),\n                reg_decoded_bbox=True,\n                loss_bbox=dict(type='GIoULoss', loss_weight=10.0))\n        ]))\n\noptimizer = dict(\n    _delete_=True,\n    type='AdamW',\n    lr=0.0004,\n    betas=(0.9, 0.999),\n    weight_decay=0.05,\n    paramwise_cfg=dict(\n        custom_keys={\n            'absolute_pos_embed': dict(decay_mult=0.),\n            'relative_position_bias_table': dict(decay_mult=0.),\n            'norm': dict(decay_mult=0.)\n        }))\nlr_config = dict(warmup_iters=500, step=[8, 11])\nrunner = dict(max_epochs=14)","metadata":{"execution":{"iopub.status.busy":"2022-01-10T04:58:03.193595Z","iopub.execute_input":"2022-01-10T04:58:03.194051Z","iopub.status.idle":"2022-01-10T04:58:03.20379Z","shell.execute_reply.started":"2022-01-10T04:58:03.193999Z","shell.execute_reply":"2022-01-10T04:58:03.203094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mmcv import Config\ncfg = Config.fromfile('./mmdetection/configs/swin/TFGBR_swin_base_faster_rcnn_fp16.py')","metadata":{"papermill":{"duration":0.050086,"end_time":"2021-12-21T16:00:53.968304","exception":false,"start_time":"2021-12-21T16:00:53.918218","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T04:58:03.20511Z","iopub.execute_input":"2022-01-10T04:58:03.205539Z","iopub.status.idle":"2022-01-10T04:58:03.235848Z","shell.execute_reply.started":"2022-01-10T04:58:03.205503Z","shell.execute_reply":"2022-01-10T04:58:03.235218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile labels.txt \ncots","metadata":{"papermill":{"duration":0.03324,"end_time":"2021-12-21T16:00:54.029361","exception":false,"start_time":"2021-12-21T16:00:53.996121","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T04:58:03.237063Z","iopub.execute_input":"2022-01-10T04:58:03.23748Z","iopub.status.idle":"2022-01-10T04:58:03.242066Z","shell.execute_reply.started":"2022-01-10T04:58:03.237444Z","shell.execute_reply":"2022-01-10T04:58:03.241399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_norm_cfg = dict(\n    mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)\ntrain_pipeline = [\n    dict(type='LoadImageFromFile',to_float32=True),\n    dict(type='LoadAnnotations', with_bbox=True),\n    dict(\n        type='AutoAugment',\n        policies=[[\n            dict(\n                type='Resize',\n                img_scale=[(480, 1333), (512, 1333), (544, 1333), (576, 1333),\n                           (608, 1333), (640, 1333), (672, 1333), (704, 1333),\n                           (736, 1333), (768, 1333), (800, 1333)],\n                multiscale_mode='value',\n                keep_ratio=True)\n        ],\n                  [\n                      dict(\n                          type='Resize',\n                          img_scale=[(400, 1333), (500, 1333), (600, 1333)],\n                          multiscale_mode='value',\n                          keep_ratio=True),\n                      dict(\n                          type='RandomCrop',\n                          crop_type='absolute_range',\n                          crop_size=(384, 600),\n                          allow_negative_crop=True),\n                      dict(\n                          type='Resize',\n                          img_scale=[(480, 1333), (512, 1333), (544, 1333),\n                                     (576, 1333), (608, 1333), (640, 1333),\n                                     (672, 1333), (704, 1333), (736, 1333),\n                                     (768, 1333), (800, 1333)],\n                          multiscale_mode='value',\n                          override=True,\n                          keep_ratio=True),\n                      dict(\n                            type='PhotoMetricDistortion',\n                            brightness_delta=32,\n                            contrast_range=(0.5, 1.5),\n                            saturation_range=(0.5, 1.5),\n                            hue_delta=18),\n                    dict(\n                            type='MinIoURandomCrop',\n                            min_ious=(0.4, 0.5, 0.6, 0.7, 0.8, 0.9),\n                            min_crop_size=0.3),\n                    dict(\n                            type='CutOut',\n                            n_holes=(5, 10),\n                            cutout_shape=[(4, 4), (4, 8), (8, 4), (8, 8),\n                                          (16, 32), (32, 16), (32, 32),\n                                          (32, 48), (48, 32), (48, 48)]\n                            )\n                  ]]),\n    dict(type='RandomFlip', flip_ratio=0.5),\n    dict(type='Normalize', **img_norm_cfg),\n    dict(type='Pad', size_divisor=32),\n    dict(type='DefaultFormatBundle'),\n    dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']),\n]\n\ntest_pipeline = [\n    dict(type='LoadImageFromFile'),\n    dict(\n        type='MultiScaleFlipAug',\n        img_scale=(1333, 800),\n        flip=False,\n        transforms=[\n            dict(type='Resize', keep_ratio=True),\n            dict(type='RandomFlip'),\n            dict(type='Normalize', **img_norm_cfg),\n            dict(type='Pad', size_divisor=32),\n            dict(type='DefaultFormatBundle'),\n            dict(type='Collect', keys=['img'])\n        ])\n]\n","metadata":{"papermill":{"duration":0.046222,"end_time":"2021-12-21T16:00:54.103069","exception":false,"start_time":"2021-12-21T16:00:54.056847","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T04:58:03.243319Z","iopub.execute_input":"2022-01-10T04:58:03.243707Z","iopub.status.idle":"2022-01-10T04:58:03.261815Z","shell.execute_reply.started":"2022-01-10T04:58:03.243673Z","shell.execute_reply":"2022-01-10T04:58:03.261059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg.classes = '/kaggle/working/labels.txt'\ncfg.work_dir = '/kaggle/working/model_output'\ncfg.data_root = '/kaggle/working'\n\n# cfg.model.roi_head.bbox_head.num_classes = 1\n\ncfg.data.test.type = 'CocoDataset'\ncfg.data.test.classes = 'labels.txt'\ncfg.data.test.data_root = '/kaggle/working'\ncfg.data.test.ann_file = '../input/tfreef-coco-dataset-notebook/val_dataset.json'\ncfg.data.test.img_prefix = ''\n\ncfg.data.train.type = 'CocoDataset'\ncfg.data.train.data_root = '/kaggle/working'\ncfg.data.train.ann_file = '../input/tfreef-coco-dataset-notebook/train_dataset.json'\ncfg.data.train.img_prefix = ''\ncfg.data.train.classes = 'labels.txt'\n\ncfg.data.val.type = 'CocoDataset'\ncfg.data.val.data_root = '/kaggle/working'\ncfg.data.val.ann_file = '../input/tfreef-coco-dataset-notebook/val_dataset.json'\ncfg.data.val.img_prefix = ''\ncfg.data.val.classes = 'labels.txt'\n\n\ncfg.train_pipeline = train_pipeline\ncfg.val_pipeline = test_pipeline\ncfg.test_pipeline = test_pipeline\n\ncfg.data.train.pipeline = cfg.train_pipeline\ncfg.data.val.pipeline = cfg.val_pipeline\n\n# cfg.optimizer.lr = 0.02 / 8\n# cfg.lr_config = dict(\n#     policy='CosineAnnealing', \n#     by_epoch=False,\n#     warmup='linear', \n#     warmup_iters= 125, \n#     warmup_ratio= 1/12,\n#     min_lr=1e-07)\n\ncfg.runner.max_epochs = 12\ncfg.evaluation.save_best='auto'\n\ncfg.seed = seed\ncfg.gpu_ids = range(1)\ncfg.fp16 = dict(loss_scale=512.)\nmeta = dict()\nmeta['config'] = cfg.pretty_text\n","metadata":{"papermill":{"duration":5.033107,"end_time":"2021-12-21T16:00:59.164453","exception":false,"start_time":"2021-12-21T16:00:54.131346","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T05:01:11.899059Z","iopub.execute_input":"2022-01-10T05:01:11.899535Z","iopub.status.idle":"2022-01-10T05:01:17.600768Z","shell.execute_reply.started":"2022-01-10T05:01:11.899493Z","shell.execute_reply":"2022-01-10T05:01:17.600071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Inference**","metadata":{"papermill":{"duration":0.02723,"end_time":"2021-12-21T16:00:59.219889","exception":false,"start_time":"2021-12-21T16:00:59.192659","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n\n    return \" \".join(pred_strings)","metadata":{"papermill":{"duration":0.034545,"end_time":"2021-12-21T16:00:59.281946","exception":false,"start_time":"2021-12-21T16:00:59.247401","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T05:01:17.602393Z","iopub.execute_input":"2022-01-10T05:01:17.602637Z","iopub.status.idle":"2022-01-10T05:01:17.608Z","shell.execute_reply.started":"2022-01-10T05:01:17.602603Z","shell.execute_reply":"2022-01-10T05:01:17.607304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import greatbarrierreef\n\nenv = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()  ","metadata":{"papermill":{"duration":0.053024,"end_time":"2021-12-21T16:00:59.362173","exception":false,"start_time":"2021-12-21T16:00:59.309149","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T05:01:17.60926Z","iopub.execute_input":"2022-01-10T05:01:17.609679Z","iopub.status.idle":"2022-01-10T05:01:17.639178Z","shell.execute_reply.started":"2022-01-10T05:01:17.609642Z","shell.execute_reply":"2022-01-10T05:01:17.638537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = init_detector(cfg, '../input/mmdetection-swin-transformer-fasterrcnn-training/model_output/epoch_14.pth')","metadata":{"papermill":{"duration":10.386356,"end_time":"2021-12-21T16:01:09.776809","exception":false,"start_time":"2021-12-21T16:00:59.390453","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T05:01:17.640803Z","iopub.execute_input":"2022-01-10T05:01:17.641108Z","iopub.status.idle":"2022-01-10T05:01:36.267666Z","shell.execute_reply.started":"2022-01-10T05:01:17.641074Z","shell.execute_reply":"2022-01-10T05:01:36.266924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#################################### faster rcnn ############################################\n# device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\n# model = build_detector(cfg.model, train_cfg=None, test_cfg=cfg.test_cfg)\n# checkpoint = load_checkpoint(model, WEIGHTS_FILE, map_location='cpu')\n\n# model.CLASSES = dataset.CLASSES\n\n# model = MMDataParallel(model, device_ids=[0])\n# outputs = single_gpu_test(model, data_loader, False, None, 0.5)\n\nresults = []\n\nfor (pixel_array, sample_prediction_df) in iter_test:\n    result = inference_detector(model, pixel_array[:, :, ::-1])\n#     show_result_pyplot(model, pixel_array[:, :, ::-1], result)    \n    boxes = result[0][:, :4]\n    scores = result[0][:, 4]\n\n    boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n    boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n\n    \n    sample_prediction_df['annotations'] = format_prediction_string(boxes, scores)\n    \n    env.predict(sample_prediction_df)","metadata":{"papermill":{"duration":0.948072,"end_time":"2021-12-21T16:01:10.754265","exception":false,"start_time":"2021-12-21T16:01:09.806193","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T05:02:31.590748Z","iopub.execute_input":"2022-01-10T05:02:31.591023Z","iopub.status.idle":"2022-01-10T05:02:31.598631Z","shell.execute_reply.started":"2022-01-10T05:02:31.59099Z","shell.execute_reply":"2022-01-10T05:02:31.597579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.rmtree('/kaggle/working/mmdetection')","metadata":{"papermill":{"duration":0.086674,"end_time":"2021-12-21T16:01:10.979422","exception":false,"start_time":"2021-12-21T16:01:10.892748","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T04:58:03.316552Z","iopub.status.idle":"2022-01-10T04:58:03.317188Z","shell.execute_reply.started":"2022-01-10T04:58:03.316931Z","shell.execute_reply":"2022-01-10T04:58:03.316955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}