{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Prepare envirment","metadata":{}},{"cell_type":"code","source":"import greatbarrierreef\n\nenv = greatbarrierreef.make_env()   # initialize the environment","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:30.435145Z","iopub.execute_input":"2022-01-27T12:21:30.435897Z","iopub.status.idle":"2022-01-27T12:21:30.498578Z","shell.execute_reply.started":"2022-01-27T12:21:30.435758Z","shell.execute_reply":"2022-01-27T12:21:30.497521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()  ","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:30.50172Z","iopub.execute_input":"2022-01-27T12:21:30.502054Z","iopub.status.idle":"2022-01-27T12:21:30.509056Z","shell.execute_reply.started":"2022-01-27T12:21:30.502013Z","shell.execute_reply":"2022-01-27T12:21:30.507749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Slices data","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nimport re\nimport random\nimport numpy as np\nimport pandas as pd\nimport copy\n\nimport cv2\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torchvision","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:30.512043Z","iopub.execute_input":"2022-01-27T12:21:30.512825Z","iopub.status.idle":"2022-01-27T12:21:32.778612Z","shell.execute_reply.started":"2022-01-27T12:21:30.512785Z","shell.execute_reply":"2022-01-27T12:21:32.777248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# just for visual test\ndef image_tiler(img, s_h=180, s_w=320):\n    stride_h = int(s_h/2)\n    stride_w = int(s_w/2)\n    tiles = np.array([img[x:x+s_h,y:y+s_w] for x in range(0,img.shape[0]-stride_h,stride_h) for y in range(0,img.shape[1]-stride_w,stride_w)])    \n    return tiles","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:32.784933Z","iopub.execute_input":"2022-01-27T12:21:32.787418Z","iopub.status.idle":"2022-01-27T12:21:32.80127Z","shell.execute_reply.started":"2022-01-27T12:21:32.787376Z","shell.execute_reply":"2022-01-27T12:21:32.800139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EfficientNet classification","metadata":{}},{"cell_type":"code","source":"sys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:32.802802Z","iopub.execute_input":"2022-01-27T12:21:32.803156Z","iopub.status.idle":"2022-01-27T12:21:32.821685Z","shell.execute_reply.started":"2022-01-27T12:21:32.803112Z","shell.execute_reply":"2022-01-27T12:21:32.819916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport timm\nfrom timm.models.efficientnet import *","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:32.823871Z","iopub.execute_input":"2022-01-27T12:21:32.82424Z","iopub.status.idle":"2022-01-27T12:21:34.708739Z","shell.execute_reply.started":"2022-01-27T12:21:32.824192Z","shell.execute_reply":"2022-01-27T12:21:34.70746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Net(nn.Module):    \n    def __init__(self):\n        super(Net, self).__init__()\n\n        e = tf_efficientnetv2_s(pretrained=False, drop_path_rate=0.2)\n        \n        self.b0 = nn.Sequential(\n            e.conv_stem,\n            e.bn1,\n            e.act1,\n        )\n        self.b1 = e.blocks[0]\n        self.b2 = e.blocks[1]\n        self.b3 = e.blocks[2]\n        self.b4 = e.blocks[3]\n        self.b5 = e.blocks[4]\n        self.b6 = e.blocks[5]\n        \n        self.b7 = nn.Sequential(\n            e.conv_head, #384, 1536\n            e.bn2,\n            e.act2,\n        )\n\n        self.logit = nn.Linear(1280, 2)\n        \n        self.mask = nn.Sequential(\n            nn.Conv2d(64, 64, kernel_size=3, padding=1),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(64, 64, kernel_size=3, padding=1),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(64, 1, kernel_size=1, padding=0),\n        )\n\n    # @torch.cuda.amp.autocast()\n    def forward(self, image):        \n        batch_size = len(image)\n        x = 2*image-1\n        x = self.b0(x) ; #print (x.shape)  # torch.Size([2, 32, 256, 256])\n        x = self.b1(x) ; #print (x.shape)  # torch.Size([2, 32, 256, 256])\n        x = self.b2(x) ; #print (x.shape)  # torch.Size([2, 56, 128, 128])\n        x = self.b3(x) ; #print (x.shape)  # torch.Size([2, 64, 23, 40])\n        \n        mask = self.mask(x)\n        \n        x = self.b4(x) ; #print (x.shape)  # torch.Size([2, 128, 12, 20])        \n        x = self.b5(x) ; #print (x.shape)  # torch.Size([2, 192, 32, 32])\n        x = self.b6(x) ; #print (x.shape)  # torch.Size([2, 328, 16, 16])        \n        x = self.b7(x) ; #print (x.shape)  # torch.Size([2, 2152, 16, 16])\n        x = F.adaptive_avg_pool2d(x,1).reshape(batch_size,-1)\n        x = F.dropout(x, 0.5, training=self.training)\n        logit = self.logit(x)\n        return logit, mask\n\nclass AmpNet(Net):\n    @torch.cuda.amp.autocast()\n    def forward(self,*args):\n        return super(AmpNet, self).forward(*args)","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:34.711152Z","iopub.execute_input":"2022-01-27T12:21:34.711454Z","iopub.status.idle":"2022-01-27T12:21:34.775991Z","shell.execute_reply.started":"2022-01-27T12:21:34.711409Z","shell.execute_reply":"2022-01-27T12:21:34.774911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.cuda.amp as amp\n    \nscaler = amp.GradScaler()\nnet = AmpNet().cuda()\n\n# todo check point\ninitial_checkpoint = '../input/yolov5data/00037020_model.pth'\nnet.load_state_dict(torch.load(initial_checkpoint)['state_dict'], strict=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:34.778293Z","iopub.execute_input":"2022-01-27T12:21:34.778764Z","iopub.status.idle":"2022-01-27T12:21:41.210996Z","shell.execute_reply.started":"2022-01-27T12:21:34.778705Z","shell.execute_reply":"2022-01-27T12:21:41.209954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def efficinetPredict(batch):\n    # predict on batch \n    net.eval()\n    with torch.no_grad():\n        logit, mask = net(batch)    \n        logit = F.softmax(logit, -1)\n        res = logit[:, 1] > 0.7\n    return res","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:27:07.829894Z","iopub.execute_input":"2022-01-27T12:27:07.830741Z","iopub.status.idle":"2022-01-27T12:27:07.837772Z","shell.execute_reply.started":"2022-01-27T12:27:07.830704Z","shell.execute_reply":"2022-01-27T12:27:07.836085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Yolo Detection","metadata":{}},{"cell_type":"code","source":"def load_model(ckpt_path, conf=0.25, iou=0.40):\n    model = torch.hub.load('/kaggle/input/yolov5-code',\n                           'custom',\n                           path=ckpt_path,\n                           source='local',\n                           force_reload=False)  # local repo\n    model.conf = conf  # NMS confidence threshold\n    model.iou  = iou  # NMS IoU threshold\n    #model.classes = None   # (optional list) filter by class, i.e. = [0, 15, 16] for persons, cats and dogs\n    #model.multi_label = False  # NMS multiple labels per box\n    #model.max_det = 20  # maximum number of detections per image\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:41.22468Z","iopub.execute_input":"2022-01-27T12:21:41.225071Z","iopub.status.idle":"2022-01-27T12:21:41.232904Z","shell.execute_reply.started":"2022-01-27T12:21:41.224976Z","shell.execute_reply":"2022-01-27T12:21:41.231583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, img, size=640, augment=False):\n    #height, width = img.shape[:2]\n    results = model(img, size=size, augment=augment)  # custom inference size\n    preds   = results.pandas().xyxy[0]\n    bboxes  = preds[['xmin','ymin','xmax','ymax']].values\n    if len(bboxes):\n        #bboxes  = voc2coco(bboxes,height,width).astype(int)\n        confs   = preds.confidence.values\n        return bboxes, confs\n    else:\n        return [],[]","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:41.234757Z","iopub.execute_input":"2022-01-27T12:21:41.235406Z","iopub.status.idle":"2022-01-27T12:21:41.247977Z","shell.execute_reply.started":"2022-01-27T12:21:41.235333Z","shell.execute_reply":"2022-01-27T12:21:41.246792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:41.2496Z","iopub.execute_input":"2022-01-27T12:21:41.250021Z","iopub.status.idle":"2022-01-27T12:21:43.042897Z","shell.execute_reply.started":"2022-01-27T12:21:41.249956Z","shell.execute_reply":"2022-01-27T12:21:43.041431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CKPT_PATH = '../input/yolov5data/best.pt'\nCONF = 0.25\nIOU = 0.40\nmodel = load_model(CKPT_PATH, conf=CONF, iou=IOU)\n\ndef yolov5(slice):\n    \n    # predict on single slices\n    bboxes, confs  = predict(model, slice)\n\n    # return p x y w h \n    return bboxes, confs","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:43.048954Z","iopub.execute_input":"2022-01-27T12:21:43.051531Z","iopub.status.idle":"2022-01-27T12:21:47.147509Z","shell.execute_reply.started":"2022-01-27T12:21:43.051484Z","shell.execute_reply":"2022-01-27T12:21:47.146465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def yoloPredict(slicesBatch, indexBatch):    \n    x = indexBatch%7\n    y = torch.div(indexBatch, 7, rounding_mode='floor')\n    _offsetW = 160 * x\n    _offsetH = 90 * y\n    #print(x, y, _offsetH, _offsetW)\n    predictions_list = torch.empty((0,4), dtype=torch.float32)\n    scores_list = torch.empty((0), dtype=torch.float32)\n    for i in range(len(slicesBatch)):\n        bboxs, scores = yolov5(slicesBatch[i])\n        #bboxs, scores = yolov5(slicesBatch[i][:, :, ::-1])\n        \n        #img_show = cv2.cvtColor(slicesBatch[i], cv2.COLOR_BGR2RGB)\n        #plt.figure(figsize=(20, 16))\n        #plt.imshow(img_show)\n        \n        for bbox in bboxs:\n            # p x y w h \n            bbox[0] += int(_offsetW[i])\n            bbox[1] += int(_offsetH[i])\n            bbox[2] += int(_offsetW[i])\n            bbox[3] += int(_offsetH[i])\n            predictions_list = torch.cat((predictions_list, torch.tensor([bbox], dtype=torch.float32)), 0)\n        scores_list = torch.cat((scores_list, torch.tensor(scores, dtype=torch.float32)), 0)\n    # post process\n    post_predictions = postProcess(predictions_list, scores_list)  \n    return post_predictions","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:47.149476Z","iopub.execute_input":"2022-01-27T12:21:47.149786Z","iopub.status.idle":"2022-01-27T12:21:47.165402Z","shell.execute_reply.started":"2022-01-27T12:21:47.149754Z","shell.execute_reply":"2022-01-27T12:21:47.164256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def postProcess(bboxes, scores):\n    # nms \n    _index = torchvision.ops.nms(bboxes, scores, 0.4) # NMS    \n    predictions = []    \n    for i in _index:\n        _bbox = bboxes[i]\n        _score = scores[i]\n        #predictions.append('{:.2f} {} {} {} {}'.format(_score, int(_bbox[0]), int(_bbox[1]), int(_bbox[2]-_bbox[0]), int(_bbox[3]-_bbox[1])))\n        predictions.append([float(_score), int(_bbox[0]), int(_bbox[1]), int(_bbox[2]-_bbox[0]), int(_bbox[3]-_bbox[1])])\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:47.167342Z","iopub.execute_input":"2022-01-27T12:21:47.167955Z","iopub.status.idle":"2022-01-27T12:21:47.178547Z","shell.execute_reply.started":"2022-01-27T12:21:47.167881Z","shell.execute_reply":"2022-01-27T12:21:47.177494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:21:47.180558Z","iopub.execute_input":"2022-01-27T12:21:47.181049Z","iopub.status.idle":"2022-01-27T12:21:47.190475Z","shell.execute_reply.started":"2022-01-27T12:21:47.180969Z","shell.execute_reply":"2022-01-27T12:21:47.189471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# quick test\n# detect = yoloPredict(_activeSlice, (batch_active == True).nonzero())\n\nDEBUG = False\n\nif DEBUG:\n    #test_img = \"../input/tensorflow-great-barrier-reef/train_images/video_2/5742.jpg\"\n    #img = cv2.imread(test_img)\n    img = copy.deepcopy(pixel_array)\n    img_show = img\n    #img_show = img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    #img_show = img[:, :, ::-1]\n    plt.figure(figsize=(20, 16))\n   \n\n    tiles = image_tiler(img)\n    \n    batch = np.stack(copy.deepcopy(tiles)[:, :, :, ::-1])\n    batch = batch.transpose((0, 3, 1, 2))\n    batch = np.ascontiguousarray(batch)\n    batch = batch.astype(np.float16) / 255\n    \n    # predict batch tile \n    batch = torch.from_numpy(batch)\n    batch = batch.cuda()\n    \n    batch_active = efficinetPredict(batch)\n    #print((batch_active == True).nonzero())\n    # [TODO] need rewrite to \n    _activeSlice = tiles[batch_active.cpu()]\n    _list = (batch_active == True).nonzero().cpu().numpy()\n    #print(_list)\n    for s in _list:\n        x = s%7\n        y = s// 7\n        _offsetW = 160 * x\n        _offsetH = 90 * y\n        \n        #print(_offsetW, _offsetH)\n        _offsetW = _offsetW[0]\n        _offsetH = _offsetH[0]\n        cv2.rectangle(img_show, (_offsetW, _offsetH), (_offsetW+320, _offsetH+180), (0, 255, 0), 2)\n            \n    \n    if len(_activeSlice) > 0:\n        detect = yoloPredict(_activeSlice, (batch_active == True).nonzero())\n        print(detect)\n        for d in detect:\n            cv2.rectangle(img_show, (d[1], d[2]), (d[1]+d[3], d[2] + d[4]), (255, 0, 0), 2)\n        prediction_str = ' '.join(['{:.2f} {} {} {} {}'.format(*i) for i in detect])\n        print(prediction_str)\n    \n    plt.imshow(img_show)","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:27:11.616172Z","iopub.execute_input":"2022-01-27T12:27:11.616514Z","iopub.status.idle":"2022-01-27T12:27:13.459095Z","shell.execute_reply.started":"2022-01-27T12:27:11.616484Z","shell.execute_reply":"2022-01-27T12:27:13.446738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"# efficientnet is training use bgr but yolo is useing rgb\n# pixel_array is rgb \nfor (pixel_array, sample_prediction_df) in iter_test:\n    tiles = image_tiler(pixel_array)\n    \n    # convert to bgr [TODO] keep efficientnet useing same rgb rather than bgr\n    batch = np.stack(copy.deepcopy(tiles)[:, :, :, ::-1])\n    \n    batch = batch.transpose((0, 3, 1, 2))\n    batch = np.ascontiguousarray(batch)\n    batch = batch.astype(np.float16) / 255\n    \n    # predict batch tile \n    batch = torch.from_numpy(batch)\n    batch = batch.cuda()\n    \n    batch_active = efficinetPredict(batch)\n    \n    # [TODO] need rewrite to use tensor\n    _activeSlice = tiles[batch_active.cpu()]\n\n    if len(_activeSlice) > 0:\n        detect = yoloPredict(_activeSlice, (batch_active == True).nonzero())\n    else:\n        detect = []\n        \n    prediction_str = ' '.join(['{:.2f} {} {} {} {}'.format(*i) for i in detect])\n    sample_prediction_df['annotations'] = prediction_str #'0.3 0 0 50 50 0.5 10 10 30 30'  #  p x y w h \n    env.predict(sample_prediction_df)\n\nsub_df = pd.read_csv('submission.csv')\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-27T12:24:12.583231Z","iopub.execute_input":"2022-01-27T12:24:12.584199Z","iopub.status.idle":"2022-01-27T12:24:18.96387Z","shell.execute_reply.started":"2022-01-27T12:24:12.584149Z","shell.execute_reply":"2022-01-27T12:24:18.962884Z"},"trusted":true},"execution_count":null,"outputs":[]}]}