{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# YOLOX detections submission made on COTS dataset (PART 2 - DETECTION)\n\nThis notebook shows how to detect starfish objects (COTS dataset) using YOLOX ON  Kaggle. First part - Building Cutom Model on Kaggle using YOLOX I implmeneted in notebook called [YoloX full training pipeline for COTS dataset](https://www.kaggle.com/remekkinas/yolox-full-training-pipeline-for-cots-dataset). It could be good starting point for build own custom model based on YOLOX detector. Full github repository you can find here - [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX)\n\n<div align = 'center'><img src='https://github.com/Megvii-BaseDetection/YOLOX/raw/main/assets/logo.png'/></div>\n\n<div class=\"alert alert-success\" role=\"alert\">\nThis work consists of two parts:     \n    <ul>\n        <li> <a href=\"https://www.kaggle.com/remekkinas/yolox-full-training-pipeline-for-cots-dataset\">YoloX full training pipeline for COTS dataset</a></li>\n        <li> YOLOX detections submission made on COTS dataset</li>\n    </ul>\n    \n</div>\n\n<div class=\"alert alert-warning\" role=\"alert\"><strong><ul><li>This is DEOMO only! What does it mean? Inference is made so far on weak model - trained only on 20 epochs.</li><li>I really appreciate if you <u>vote on both of these notebooks</u> - thank you! I just share my work to make competition fun and more interesting.</li></ul> </strong></div>\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport torch\nimport importlib\nimport cv2 \nimport pandas as pd\n\nfrom PIL import Image\nfrom IPython.display import display","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:09:36.585789Z","iopub.execute_input":"2022-05-27T00:09:36.586352Z","iopub.status.idle":"2022-05-27T00:09:38.472815Z","shell.execute_reply.started":"2022-05-27T00:09:36.586243Z","shell.execute_reply":"2022-05-27T00:09:38.472084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### INSTALL YOLOX \n<div class=\"alert alert-warning\" role=\"alert\"><strong>It unfortunately requires a lot of Kaggle enviroment hacking :) due to competition limitation - no internet access during submission.</strong></div>","metadata":{}},{"cell_type":"code","source":"# download required packages - first time when I created database (https://www.kaggle.com/remekkinas/yolox-cots-models) with required moduls for YOLOX\n# don't use this section of code until Kaggle doesn't change something in the environment (!!)\n\n\n#%mkdir /kaggle/working/yolox-dep\n#!pip download pip -d \"/kaggle/working/yolox-dep\"\n#!pip download loguru -d \"/kaggle/working/yolox-dep\"\n#!pip download ninja -d \"/kaggle/working/yolox-dep\"\n#!pip download onnx==\"1.8.1\" -d \"/kaggle/working/yolox-dep\"\n#!pip download onnxruntime==\"1.8.0\" -d \"/kaggle/working/yolox-dep\"\n#!pip download onnxoptimizer>=\"0.2.5\" -d \"/kaggle/working/yolox-dep\"\n#!pip download thop -d \"/kaggle/working/yolox-dep\"\n#!pip download tabulate -d \"/kaggle/working/yolox-dep\"\n#!pip download onnx-simplifier==0.3.5 -d \"/kaggle/working/yolox-dep\"","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-27T00:09:38.474686Z","iopub.execute_input":"2022-05-27T00:09:38.475145Z","iopub.status.idle":"2022-05-27T00:09:38.479485Z","shell.execute_reply.started":"2022-05-27T00:09:38.475110Z","shell.execute_reply":"2022-05-27T00:09:38.478177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Copy YOLOX and required modules from local repository (Kaggle dataset -> https://www.kaggle.com/remekkinas/yolox-cots-models)\n%cp -r /kaggle/input/yolox-cots-models /kaggle/working/\n%cd /kaggle/working/yolox-cots-models/yolox-dep","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:09:38.480783Z","iopub.execute_input":"2022-05-27T00:09:38.481225Z","iopub.status.idle":"2022-05-27T00:10:00.578821Z","shell.execute_reply.started":"2022-05-27T00:09:38.481191Z","shell.execute_reply":"2022-05-27T00:10:00.577991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install YOLOX required modules\n\n!pip install pip-21.3.1-py3-none-any.whl -f ./ --no-index\n!pip install loguru-0.5.3-py3-none-any.whl -f ./ --no-index\n!pip install ninja-1.10.2.3-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl -f ./ --no-index\n!pip install onnx-1.8.1-cp37-cp37m-manylinux2010_x86_64.whl -f ./ --no-index\n!pip install onnxruntime-1.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -f ./ --no-index\n!pip install onnxoptimizer-0.2.6-cp37-cp37m-manylinux2014_x86_64.whl -f ./ --no-index\n!pip install thop-0.0.31.post2005241907-py3-none-any.whl -f ./ --no-index\n!pip install tabulate-0.8.9-py3-none-any.whl -f ./ --no-index\n#!pip install onnx-simplifier-0.3.6.tar.gz -f ./ --no-index","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-27T00:10:00.581518Z","iopub.execute_input":"2022-05-27T00:10:00.581788Z","iopub.status.idle":"2022-05-27T00:11:17.065750Z","shell.execute_reply.started":"2022-05-27T00:10:00.581753Z","shell.execute_reply":"2022-05-27T00:11:17.064918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install YOLOX\n%cd /kaggle/working/yolox-cots-models/YOLOX\n!pip install -r requirements.txt\n!pip install -v -e . ","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-27T00:11:17.067215Z","iopub.execute_input":"2022-05-27T00:11:17.067503Z","iopub.status.idle":"2022-05-27T00:12:31.025619Z","shell.execute_reply.started":"2022-05-27T00:11:17.067466Z","shell.execute_reply":"2022-05-27T00:12:31.024787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install CocoAPI tool\n%cd /kaggle/working/yolox-cots-models/yolox-dep/cocoapi/PythonAPI\n\n!make\n!make install\n!python setup.py install","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-27T00:12:31.027177Z","iopub.execute_input":"2022-05-27T00:12:31.027465Z","iopub.status.idle":"2022-05-27T00:12:50.073760Z","shell.execute_reply.started":"2022-05-27T00:12:31.027423Z","shell.execute_reply":"2022-05-27T00:12:50.072931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pycocotools","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:12:50.075641Z","iopub.execute_input":"2022-05-27T00:12:50.075923Z","iopub.status.idle":"2022-05-27T00:12:50.082055Z","shell.execute_reply.started":"2022-05-27T00:12:50.075886Z","shell.execute_reply":"2022-05-27T00:12:50.081402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### TEST MODEL - MAKE INFERENCE ON SAMPLE DATA\n","metadata":{}},{"cell_type":"code","source":"CHECKPOINT_FILE = '/kaggle/input/mymodel1/best_ckpt.pth'","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:12:50.083320Z","iopub.execute_input":"2022-05-27T00:12:50.083628Z","iopub.status.idle":"2022-05-27T00:12:50.100351Z","shell.execute_reply.started":"2022-05-27T00:12:50.083592Z","shell.execute_reply":"2022-05-27T00:12:50.099636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/yolox-cots-models/YOLOX","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:14:07.157740Z","iopub.execute_input":"2022-05-27T00:14:07.158073Z","iopub.status.idle":"2022-05-27T00:14:07.165161Z","shell.execute_reply.started":"2022-05-27T00:14:07.158040Z","shell.execute_reply":"2022-05-27T00:14:07.164321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config_file_template = '''\n\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n    def __init__(self):\n        super(Exp, self).__init__()\n        self.depth = 1\n        self.width = 1\n        self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n        self.num_classes = 1\n\n'''\n\nwith open('cots_config.py', 'w') as f:\n    f.write(config_file_template)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:14:12.861205Z","iopub.execute_input":"2022-05-27T00:14:12.861898Z","iopub.status.idle":"2022-05-27T00:14:12.866470Z","shell.execute_reply.started":"2022-05-27T00:14:12.861863Z","shell.execute_reply":"2022-05-27T00:14:12.865450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from yolox.utils import postprocess\nfrom yolox.data.data_augment import ValTransform\n\nCOCO_CLASSES = (\n  \"starfish\",\n)\n\n# get YOLOX experiment\ncurrent_exp = importlib.import_module('cots_config')\nexp = current_exp.Exp()\n\n# set inference parameters\ntest_size = (800, 1280)\nnum_classes = 1\nconfthre = 0.1\nnmsthre = 0.4\n\n\n# get YOLOX model\nmodel = exp.get_model()\nmodel.cuda()\nmodel.eval()\n\n# get custom trained checkpoint\nckpt_file = CHECKPOINT_FILE\nckpt = torch.load(ckpt_file, map_location=\"cpu\")\nmodel.load_state_dict(ckpt[\"model\"])","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:14:14.898995Z","iopub.execute_input":"2022-05-27T00:14:14.899823Z","iopub.status.idle":"2022-05-27T00:14:23.288782Z","shell.execute_reply.started":"2022-05-27T00:14:14.899774Z","shell.execute_reply":"2022-05-27T00:14:23.288090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def yolox_inference(img, model, test_size): \n    bboxes = []\n    bbclasses = []\n    scores = []\n    \n    preproc = ValTransform(legacy = False)\n\n    tensor_img, _ = preproc(img, None, test_size)\n    tensor_img = torch.from_numpy(tensor_img).unsqueeze(0)\n    tensor_img = tensor_img.float()\n    tensor_img = tensor_img.cuda()\n\n    with torch.no_grad():\n        outputs = model(tensor_img)\n        outputs = postprocess(\n                    outputs, num_classes, confthre,\n                    nmsthre, class_agnostic=True\n                )\n\n    if outputs[0] is None:\n        return [], [], []\n    \n    outputs = outputs[0].cpu()\n    bboxes = outputs[:, 0:4]\n\n    bboxes /= min(test_size[0] / img.shape[0], test_size[1] / img.shape[1])\n    bbclasses = outputs[:, 6]\n    scores = outputs[:, 4] * outputs[:, 5]\n    \n    return bboxes, bbclasses, scores","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:14:25.440663Z","iopub.execute_input":"2022-05-27T00:14:25.441439Z","iopub.status.idle":"2022-05-27T00:14:25.450193Z","shell.execute_reply.started":"2022-05-27T00:14:25.441402Z","shell.execute_reply":"2022-05-27T00:14:25.449311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_yolox_predictions(img, bboxes, scores, bbclasses, confthre, classes_dict):\n    for i in range(len(bboxes)):\n            box = bboxes[i]\n            cls_id = int(bbclasses[i])\n            score = scores[i]\n            if score < confthre:\n                continue\n            x0 = int(box[0])\n            y0 = int(box[1])\n            x1 = int(box[2])\n            y1 = int(box[3])\n\n            cv2.rectangle(img, (x0, y0), (x1, y1), (0, 255, 0), 2)\n            cv2.putText(img, '{}:{:.1f}%'.format(classes_dict[cls_id], score * 100), (x0, y0 - 3), cv2.FONT_HERSHEY_PLAIN, 0.8, (0,255,0), thickness = 1)\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:14:30.411136Z","iopub.execute_input":"2022-05-27T00:14:30.411894Z","iopub.status.idle":"2022-05-27T00:14:30.420166Z","shell.execute_reply.started":"2022-05-27T00:14:30.411858Z","shell.execute_reply":"2022-05-27T00:14:30.419447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_IMAGE_PATH = \"/kaggle/input/tensorflow-great-barrier-reef/train_images/video_0/9674.jpg\"\nimg = cv2.imread(TEST_IMAGE_PATH)\n\n# Get predictions\nbboxes, bbclasses, scores = yolox_inference(img, model, test_size)\n\n# Draw predictions\nout_image = draw_yolox_predictions(img, bboxes, scores, bbclasses, confthre, COCO_CLASSES)\n\n# Since we load image using OpenCV we have to convert it \nout_image = cv2.cvtColor(out_image, cv2.COLOR_BGR2RGB)\ndisplay(Image.fromarray(out_image))","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:14:32.525385Z","iopub.execute_input":"2022-05-27T00:14:32.525956Z","iopub.status.idle":"2022-05-27T00:14:38.111024Z","shell.execute_reply.started":"2022-05-27T00:14:32.525912Z","shell.execute_reply":"2022-05-27T00:14:38.110355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## SUBMIT PREDICTION TO COMPETITION","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:14:43.524942Z","iopub.execute_input":"2022-05-27T00:14:43.525275Z","iopub.status.idle":"2022-05-27T00:14:43.531959Z","shell.execute_reply.started":"2022-05-27T00:14:43.525225Z","shell.execute_reply":"2022-05-27T00:14:43.531258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import greatbarrierreef\n\nenv = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()  ","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:14:45.275704Z","iopub.execute_input":"2022-05-27T00:14:45.276521Z","iopub.status.idle":"2022-05-27T00:14:45.322040Z","shell.execute_reply.started":"2022-05-27T00:14:45.276483Z","shell.execute_reply":"2022-05-27T00:14:45.321305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_dict = {\n    'id': [],\n    'prediction_string': [],\n}\n\nfor (image_np, sample_prediction_df) in iter_test:\n \n    bboxes, bbclasses, scores = yolox_inference(image_np[:,:,::-1], model, test_size)\n    \n    predictions = []\n    for i in range(len(bboxes)):\n        box = bboxes[i]\n        cls_id = int(bbclasses[i])\n        score = scores[i]\n        if score < confthre:\n            continue\n        x_min = int(box[0])\n        y_min = int(box[1])\n        x_max = int(box[2])\n        y_max = int(box[3])\n        \n        bbox_width = x_max - x_min\n        bbox_height = y_max - y_min\n        \n        predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n    \n    prediction_str = ' '.join(predictions)\n    sample_prediction_df['annotations'] = prediction_str\n    env.predict(sample_prediction_df)\n\n    print('Prediction:', prediction_str)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:14:47.330796Z","iopub.execute_input":"2022-05-27T00:14:47.331064Z","iopub.status.idle":"2022-05-27T00:14:47.943770Z","shell.execute_reply.started":"2022-05-27T00:14:47.331034Z","shell.execute_reply":"2022-05-27T00:14:47.942165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('submission.csv')\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T00:14:50.783014Z","iopub.execute_input":"2022-05-27T00:14:50.783308Z","iopub.status.idle":"2022-05-27T00:14:50.799298Z","shell.execute_reply.started":"2022-05-27T00:14:50.783273Z","shell.execute_reply":"2022-05-27T00:14:50.798306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div class=\"alert alert-success\" role=\"alert\">\n    Find this notebook helpful? :) Please give me a vote ;) Thank you\n </div>","metadata":{}}]}