{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Train YOLOX on COTS dataset (PART 1 - TRAINING)\n\nThis notebook shows how to train custom object detection model (COTS dataset) on Kaggle. It could be good starting point for build own custom model based on YOLOX detector. Full github repository you can find here - [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX)\n\n
\n\n**Steps covered in this notebook:**\n* Install YOLOX \n* Prepare COTS dataset for YOLOX object detection training\n* Download Pre-Trained Weights for YOLOX\n* Prepare configuration files\n* YOLOX training\n* Run YOLOX inference on test images\n* Export YOLOX weights for Tensorflow inference (soon)\n\nNow I created notebook for learning and prototyping in YOLOX. Next step is too create better model (play with YOLOX experimentation parameters).","metadata":{"execution":{"iopub.status.busy":"2021-11-29T13:34:31.033138Z","iopub.execute_input":"2021-11-29T13:34:31.033449Z","iopub.status.idle":"2021-11-29T13:34:33.455468Z","shell.execute_reply.started":"2021-11-29T13:34:31.033368Z","shell.execute_reply":"2021-11-29T13:34:33.454141Z"}}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport ast\nimport os\nimport json\nimport pandas as pd\nimport torch\nimport importlib\nimport cv2 \n\nfrom shutil import copyfile\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nfrom sklearn.model_selection import GroupKFold\nfrom PIL import Image\nfrom string import Template\nfrom IPython.display import display\n\nTRAIN_PATH = '/kaggle/input/tensorflow-great-barrier-reef'","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:44:28.364584Z","iopub.execute_input":"2021-12-19T16:44:28.365328Z","iopub.status.idle":"2021-12-19T16:44:28.37355Z","shell.execute_reply.started":"2021-12-19T16:44:28.365259Z","shell.execute_reply":"2021-12-19T16:44:28.37287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check Torch and CUDA version\nprint(f\"Torch: {torch.__version__}\")\n!nvcc --version","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:15:12.738144Z","iopub.execute_input":"2021-12-19T13:15:12.738393Z","iopub.status.idle":"2021-12-19T13:15:13.426925Z","shell.execute_reply.started":"2021-12-19T13:15:12.738358Z","shell.execute_reply":"2021-12-19T13:15:13.426056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. INSTALL YOLOX","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/Megvii-BaseDetection/YOLOX -q\n\n%cd YOLOX\n!pip install -U pip && pip install -r requirements.txt\n!pip install -v -e . ","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-19T13:15:13.428231Z","iopub.execute_input":"2021-12-19T13:15:13.428448Z","iopub.status.idle":"2021-12-19T13:16:14.998031Z","shell.execute_reply.started":"2021-12-19T13:15:13.42842Z","shell.execute_reply":"2021-12-19T13:16:14.99719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-19T13:16:15.00098Z","iopub.execute_input":"2021-12-19T13:16:15.00155Z","iopub.status.idle":"2021-12-19T13:16:31.477008Z","shell.execute_reply.started":"2021-12-19T13:16:15.001479Z","shell.execute_reply":"2021-12-19T13:16:31.476077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. PREPARE COTS DATASET FOR YOLOX\nThis section is taken from notebook created by Awsaf [Great-Barrier-Reef: YOLOv5 train](https://www.kaggle.com/awsaf49/great-barrier-reef-yolov5-train)\n\n## A. PREPARE DATASET AND ANNOTATIONS","metadata":{}},{"cell_type":"code","source":"def get_bbox(annots):\n bboxes = [list(annot.values()) for annot in annots]\n return bboxes\n\ndef get_path(row):\n row['image_path'] = f'{TRAIN_PATH}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n return row","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:16:31.47895Z","iopub.execute_input":"2021-12-19T13:16:31.47924Z","iopub.status.idle":"2021-12-19T13:16:31.485213Z","shell.execute_reply.started":"2021-12-19T13:16:31.479198Z","shell.execute_reply":"2021-12-19T13:16:31.484109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\ndf.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:16:31.486556Z","iopub.execute_input":"2021-12-19T13:16:31.487007Z","iopub.status.idle":"2021-12-19T13:16:31.567568Z","shell.execute_reply.started":"2021-12-19T13:16:31.486961Z","shell.execute_reply":"2021-12-19T13:16:31.566773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taken only annotated photos\ndf[\"num_bbox\"] = df['annotations'].apply(lambda x: str.count(x, 'x'))\ndf_train = df[df[\"num_bbox\"]>0]\n\n#Annotations \ndf_train['annotations'] = df_train['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndf_train['bboxes'] = df_train.annotations.progress_apply(get_bbox)\n\n#Images resolution\ndf_train[\"width\"] = 1280\ndf_train[\"height\"] = 720\n\n#Path of images\ndf_train = df_train.progress_apply(get_path, axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:16:31.568818Z","iopub.execute_input":"2021-12-19T13:16:31.569076Z","iopub.status.idle":"2021-12-19T13:16:35.600401Z","shell.execute_reply.started":"2021-12-19T13:16:31.569048Z","shell.execute_reply":"2021-12-19T13:16:35.599648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kf = GroupKFold(n_splits = 5) \ndf_train = df_train.reset_index(drop=True)\ndf_train['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(kf.split(df_train, y = df_train.video_id.tolist(), groups=df_train.sequence)):\n df_train.loc[val_idx, 'fold'] = fold\n\ndf_train.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:16:35.601774Z","iopub.execute_input":"2021-12-19T13:16:35.602236Z","iopub.status.idle":"2021-12-19T13:16:35.63335Z","shell.execute_reply.started":"2021-12-19T13:16:35.602193Z","shell.execute_reply":"2021-12-19T13:16:35.632457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HOME_DIR = '/kaggle/working/' \nDATASET_PATH = 'dataset/images'\n\n!mkdir {HOME_DIR}dataset\n!mkdir {HOME_DIR}{DATASET_PATH}\n!mkdir {HOME_DIR}{DATASET_PATH}/train2017\n!mkdir {HOME_DIR}{DATASET_PATH}/val2017\n!mkdir {HOME_DIR}{DATASET_PATH}/annotations","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:44:59.903242Z","iopub.execute_input":"2021-12-19T16:44:59.903989Z","iopub.status.idle":"2021-12-19T16:45:00.795424Z","shell.execute_reply.started":"2021-12-19T16:44:59.903953Z","shell.execute_reply":"2021-12-19T16:45:00.794589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SELECTED_FOLD = 4\n\nfor i in tqdm(range(len(df_train))):\n row = df_train.loc[i]\n if row.fold != SELECTED_FOLD:\n copyfile(f'{row.image_path}', f'{HOME_DIR}{DATASET_PATH}/train2017/{row.image_id}.jpg')\n else:\n copyfile(f'{row.image_path}', f'{HOME_DIR}{DATASET_PATH}/val2017/{row.image_id}.jpg') ","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:16:39.00523Z","iopub.execute_input":"2021-12-19T13:16:39.005463Z","iopub.status.idle":"2021-12-19T13:17:39.08952Z","shell.execute_reply.started":"2021-12-19T13:16:39.005433Z","shell.execute_reply":"2021-12-19T13:17:39.088837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Number of training files: {len(os.listdir(f\"{HOME_DIR}{DATASET_PATH}/train2017/\"))}')\nprint(f'Number of validation files: {len(os.listdir(f\"{HOME_DIR}{DATASET_PATH}/val2017/\"))}')","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:39.090861Z","iopub.execute_input":"2021-12-19T13:17:39.0914Z","iopub.status.idle":"2021-12-19T13:17:39.100961Z","shell.execute_reply.started":"2021-12-19T13:17:39.091359Z","shell.execute_reply":"2021-12-19T13:17:39.100185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## B. CREATE COCO ANNOTATION FILES","metadata":{}},{"cell_type":"code","source":"def save_annot_json(json_annotation, filename):\n with open(filename, 'w') as f:\n output_json = json.dumps(json_annotation)\n f.write(output_json)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:39.102252Z","iopub.execute_input":"2021-12-19T13:17:39.10244Z","iopub.status.idle":"2021-12-19T13:17:39.108Z","shell.execute_reply.started":"2021-12-19T13:17:39.102417Z","shell.execute_reply":"2021-12-19T13:17:39.106951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annotion_id = 0","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:39.109337Z","iopub.execute_input":"2021-12-19T13:17:39.109602Z","iopub.status.idle":"2021-12-19T13:17:39.11663Z","shell.execute_reply.started":"2021-12-19T13:17:39.109567Z","shell.execute_reply":"2021-12-19T13:17:39.11583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dataset2coco(df, dest_path):\n \n global annotion_id\n \n annotations_json = {\n \"info\": [],\n \"licenses\": [],\n \"categories\": [],\n \"images\": [],\n \"annotations\": []\n }\n \n info = {\n \"year\": \"2021\",\n \"version\": \"1\",\n \"description\": \"COTS dataset - COCO format\",\n \"contributor\": \"\",\n \"url\": \"https://kaggle.com\",\n \"date_created\": \"2021-11-30T15:01:26+00:00\"\n }\n annotations_json[\"info\"].append(info)\n \n lic = {\n \"id\": 1,\n \"url\": \"\",\n \"name\": \"Unknown\"\n }\n annotations_json[\"licenses\"].append(lic)\n\n classes = {\"id\": 0, \"name\": \"starfish\", \"supercategory\": \"none\"}\n\n annotations_json[\"categories\"].append(classes)\n\n \n for ann_row in df.itertuples():\n \n images = {\n \"id\": ann_row[0],\n \"license\": 1,\n \"file_name\": ann_row.image_id + '.jpg',\n \"height\": ann_row.height,\n \"width\": ann_row.width,\n \"date_captured\": \"2021-11-30T15:01:26+00:00\"\n }\n \n annotations_json[\"images\"].append(images)\n \n bbox_list = ann_row.bboxes\n \n for bbox in bbox_list:\n b_width = bbox[2]\n b_height = bbox[3]\n \n # some boxes in COTS are outside the image height and width\n if (bbox[0] + bbox[2] > 1280):\n b_width = bbox[0] - 1280 \n if (bbox[1] + bbox[3] > 720):\n b_height = bbox[1] - 720 \n \n image_annotations = {\n \"id\": annotion_id,\n \"image_id\": ann_row[0],\n \"category_id\": 0,\n \"bbox\": [bbox[0], bbox[1], b_width, b_height],\n \"area\": bbox[2] * bbox[3],\n \"segmentation\": [],\n \"iscrowd\": 0\n }\n \n annotion_id += 1\n annotations_json[\"annotations\"].append(image_annotations)\n \n \n print(f\"Dataset COTS annotation to COCO json format completed! Files: {len(df)}\")\n return annotations_json","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:39.118189Z","iopub.execute_input":"2021-12-19T13:17:39.11856Z","iopub.status.idle":"2021-12-19T13:17:39.130869Z","shell.execute_reply.started":"2021-12-19T13:17:39.118527Z","shell.execute_reply":"2021-12-19T13:17:39.130022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert COTS dataset to JSON COCO\ntrain_annot_json = dataset2coco(df_train[df_train.fold != SELECTED_FOLD], f\"{HOME_DIR}{DATASET_PATH}/train2017/\")\nval_annot_json = dataset2coco(df_train[df_train.fold == SELECTED_FOLD], f\"{HOME_DIR}{DATASET_PATH}/val2017/\")\n\n# Save converted annotations\nsave_annot_json(train_annot_json, f\"{HOME_DIR}{DATASET_PATH}/annotations/train.json\")\nsave_annot_json(val_annot_json, f\"{HOME_DIR}{DATASET_PATH}/annotations/valid.json\")","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:39.13242Z","iopub.execute_input":"2021-12-19T13:17:39.132674Z","iopub.status.idle":"2021-12-19T13:17:39.364269Z","shell.execute_reply.started":"2021-12-19T13:17:39.13264Z","shell.execute_reply":"2021-12-19T13:17:39.362803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. PREPARE CONFIGURATION FILE\n\nConfiguration files for Yolox:\n- [YOLOX-nano](https://github.com/Megvii-BaseDetection/YOLOX/blob/main/exps/default/nano.py)\n- [YOLOX-s](https://github.com/Megvii-BaseDetection/YOLOX/blob/main/exps/default/yolox_s.py)\n- [YOLOX-m](https://github.com/Megvii-BaseDetection/YOLOX/blob/main/exps/default/yolox_m.py)\n\nBelow you can find two (yolox-s and yolox-nano) configuration files for our COTS dataset training.\n","metadata":{}},{"cell_type":"code","source":"# Choose model for your experiments NANO or YOLOX-S (you can adapt for other model type)\n\nNANO = False","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:39.365716Z","iopub.execute_input":"2021-12-19T13:17:39.365986Z","iopub.status.idle":"2021-12-19T13:17:39.370592Z","shell.execute_reply.started":"2021-12-19T13:17:39.365951Z","shell.execute_reply":"2021-12-19T13:17:39.369732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3A. YOLOX-S EXPERIMENT CONFIGURATION FILE\nTraining parameters could be set up in experiment config files. I created custom files for YOLOX-s and nano. You can create your own using files from oryginal github repo.","metadata":{}},{"cell_type":"code","source":"config_file_template = '''\n\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n def __init__(self):\n super(Exp, self).__init__()\n self.depth = 0.33\n self.width = 0.50\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n \n # Define yourself dataset path\n self.data_dir = \"/kaggle/working/dataset/images\"\n self.train_ann = \"train.json\"\n self.val_ann = \"valid.json\"\n\n self.num_classes = 1\n\n self.max_epoch = $max_epoch\n self.data_num_workers = 2\n self.eval_interval = 1\n \n self.mosaic_prob = 1.0\n self.mixup_prob = 1.0\n self.hsv_prob = 1.0\n self.flip_prob = 0.5\n self.no_aug_epochs = 2\n \n self.input_size = (960, 960)\n self.mosaic_scale = (0.5, 1.5)\n self.random_size = (10, 20)\n self.test_size = (960, 960)\n'''","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:39.372365Z","iopub.execute_input":"2021-12-19T13:17:39.372936Z","iopub.status.idle":"2021-12-19T13:17:39.380191Z","shell.execute_reply.started":"2021-12-19T13:17:39.372899Z","shell.execute_reply":"2021-12-19T13:17:39.379378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if NANO:\n config_file_template = '''\n\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nimport torch.nn as nn\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n def __init__(self):\n super(Exp, self).__init__()\n self.depth = 0.33\n self.width = 0.25\n self.input_size = (416, 416)\n self.mosaic_scale = (0.5, 1.5)\n self.random_size = (10, 20)\n self.test_size = (416, 416)\n self.exp_name = os.path.split(\n os.path.realpath(__file__))[1].split(\".\")[0]\n self.enable_mixup = False\n\n # Define yourself dataset path\n self.data_dir = \"/kaggle/working/dataset/images\"\n self.train_ann = \"train.json\"\n self.val_ann = \"valid.json\"\n\n self.num_classes = 1\n\n self.max_epoch = $max_epoch\n self.data_num_workers = 2\n self.eval_interval = 1\n\n def get_model(self, sublinear=False):\n def init_yolo(M):\n for m in M.modules():\n if isinstance(m, nn.BatchNorm2d):\n m.eps = 1e-3\n m.momentum = 0.03\n\n if \"model\" not in self.__dict__:\n from yolox.models import YOLOX, YOLOPAFPN, YOLOXHead\n in_channels = [256, 512, 1024]\n # NANO model use depthwise = True, which is main difference.\n backbone = YOLOPAFPN(self.depth,\n self.width,\n in_channels=in_channels,\n depthwise=True)\n head = YOLOXHead(self.num_classes,\n self.width,\n in_channels=in_channels,\n depthwise=True)\n self.model = YOLOX(backbone, head)\n\n self.model.apply(init_yolo)\n self.model.head.initialize_biases(1e-2)\n return self.model\n\n'''","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:39.381698Z","iopub.execute_input":"2021-12-19T13:17:39.382029Z","iopub.status.idle":"2021-12-19T13:17:39.392737Z","shell.execute_reply.started":"2021-12-19T13:17:39.381988Z","shell.execute_reply":"2021-12-19T13:17:39.391945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PIPELINE_CONFIG_PATH='cots_config.py'\n\npipeline = Template(config_file_template).substitute(max_epoch = 20)\n\nwith open(PIPELINE_CONFIG_PATH, 'w') as f:\n f.write(pipeline)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:39.394265Z","iopub.execute_input":"2021-12-19T13:17:39.394533Z","iopub.status.idle":"2021-12-19T13:17:39.40518Z","shell.execute_reply.started":"2021-12-19T13:17:39.394499Z","shell.execute_reply":"2021-12-19T13:17:39.404454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ./yolox/data/datasets/voc_classes.py\n\nvoc_cls = '''\nVOC_CLASSES = (\n \"starfish\",\n)\n'''\nwith open('./yolox/data/datasets/voc_classes.py', 'w') as f:\n f.write(voc_cls)\n\n# ./yolox/data/datasets/coco_classes.py\n\ncoco_cls = '''\nCOCO_CLASSES = (\n \"starfish\",\n)\n'''\nwith open('./yolox/data/datasets/coco_classes.py', 'w') as f:\n f.write(coco_cls)\n\n# check if everything is ok \n!more ./yolox/data/datasets/coco_classes.py","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:39.406697Z","iopub.execute_input":"2021-12-19T13:17:39.406986Z","iopub.status.idle":"2021-12-19T13:17:40.08652Z","shell.execute_reply.started":"2021-12-19T13:17:39.406935Z","shell.execute_reply":"2021-12-19T13:17:40.085717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. DOWNLOAD PRETRAINED WEIGHTS","metadata":{}},{"cell_type":"markdown","source":"List of pretrained models:\n* YOLOX-s\n* YOLOX-m\n* YOLOX-nano for inference speed (!)\n* etc.","metadata":{}},{"cell_type":"code","source":"sh = 'wget https://github.com/Megvii-BaseDetection/storage/releases/download/0.0.1/yolox_s.pth'\nMODEL_FILE = 'yolox_s.pth'\n\nif NANO:\n sh = '''\n wget https://github.com/Megvii-BaseDetection/storage/releases/download/0.0.1/yolox_nano.pth\n '''\n MODEL_FILE = 'yolox_nano.pth'\n\nwith open('script.sh', 'w') as file:\n file.write(sh)\n\n!bash script.sh","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:40.088055Z","iopub.execute_input":"2021-12-19T13:17:40.088312Z","iopub.status.idle":"2021-12-19T13:17:43.797517Z","shell.execute_reply.started":"2021-12-19T13:17:40.088284Z","shell.execute_reply":"2021-12-19T13:17:43.796661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. TRAIN MODEL","metadata":{}},{"cell_type":"code","source":"!cp ./tools/train.py ./","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:43.799499Z","iopub.execute_input":"2021-12-19T13:17:43.801251Z","iopub.status.idle":"2021-12-19T13:17:44.477128Z","shell.execute_reply.started":"2021-12-19T13:17:43.801213Z","shell.execute_reply":"2021-12-19T13:17:44.476184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python train.py \\\n -f cots_config.py \\\n -d 1 \\\n -b 32 \\\n --fp16 \\\n -o \\\n -c {MODEL_FILE} # Remember to chenge this line if you take different model eg. yolo_nano.pth, yolox_s.pth or yolox_m.pth","metadata":{"execution":{"iopub.status.busy":"2021-12-19T13:17:44.479351Z","iopub.execute_input":"2021-12-19T13:17:44.479816Z","iopub.status.idle":"2021-12-19T16:29:55.301896Z","shell.execute_reply.started":"2021-12-19T13:17:44.479771Z","shell.execute_reply":"2021-12-19T16:29:55.301077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. RUN INFERENCE\n\n## 6A. INFERENCE USING YOLOX TOOL","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:44:11.208241Z","iopub.execute_input":"2021-12-19T16:44:11.208527Z","iopub.status.idle":"2021-12-19T16:44:11.214263Z","shell.execute_reply.started":"2021-12-19T16:44:11.208487Z","shell.execute_reply":"2021-12-19T16:44:11.213367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I have to fix demo.py file because it:\n# - raises error in Kaggle (cvWaitKey does not work) \n# - saves result files in time named directory eg. /2021_11_29_22_51_08/ which is difficult then to automatically show results\n\n%cp ./Kaggle/input/yolox-kaggle-fix-for-demo-inference/demo.py tools/demo.py","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:45:35.073518Z","iopub.execute_input":"2021-12-19T16:45:35.073798Z","iopub.status.idle":"2021-12-19T16:45:35.262578Z","shell.execute_reply.started":"2021-12-19T16:45:35.073767Z","shell.execute_reply":"2021-12-19T16:45:35.261604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_IMAGE_PATH = \"/kaggle/working/dataset/images/val2017/0-4614.jpg\"\nMODEL_PATH = \"/kaggle/working/YOLOX_outputs/cots_config/best_ckpt.pth\"\n\n!python tools/demo.py image \\\n -f cots_config.py \\\n -c {MODEL_PATH} \\\n --path {TEST_IMAGE_PATH} \\\n --conf 0.1 \\\n --nms 0.45 \\\n --tsize 960 \\\n --save_result \\\n --device gpu","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:39:57.10501Z","iopub.execute_input":"2021-12-19T16:39:57.105616Z","iopub.status.idle":"2021-12-19T16:39:57.380583Z","shell.execute_reply.started":"2021-12-19T16:39:57.10558Z","shell.execute_reply":"2021-12-19T16:39:57.379788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUTPUT_IMAGE_PATH = \"./YOLOX_outputs/cots_config/vis_res/0-4614.jpg\" \nImage.open(OUTPUT_IMAGE_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:37:35.865802Z","iopub.execute_input":"2021-12-19T16:37:35.866531Z","iopub.status.idle":"2021-12-19T16:37:35.897351Z","shell.execute_reply.started":"2021-12-19T16:37:35.866486Z","shell.execute_reply":"2021-12-19T16:37:35.895973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6B. INFERENCE USING CUSTOM SCRIPT (IT WOULD BE USED FOR COTS INFERENCE PART)\n\n### 6B.1 SETUP MODEL","metadata":{}},{"cell_type":"code","source":"from yolox.utils import postprocess\nfrom yolox.data.data_augment import ValTransform\n\nCOCO_CLASSES = (\n \"starfish\",\n)\n\n# get YOLOX experiment\ncurrent_exp = importlib.import_module('cots_config')\nexp = current_exp.Exp()\n\n# set inference parameters\ntest_size = (960, 960)\nnum_classes = 1\nconfthre = 0.1\nnmsthre = 0.45\n\n\n# get YOLOX model\nmodel = exp.get_model()\nmodel.cuda()\nmodel.eval()\n\n# get custom trained checkpoint\nckpt_file = \"./YOLOX_outputs/cots_config/best_ckpt.pth\"\nckpt = torch.load(ckpt_file, map_location=\"cpu\")\nmodel.load_state_dict(ckpt[\"model\"])","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:30:04.114472Z","iopub.execute_input":"2021-12-19T16:30:04.114711Z","iopub.status.idle":"2021-12-19T16:30:06.458637Z","shell.execute_reply.started":"2021-12-19T16:30:04.114679Z","shell.execute_reply":"2021-12-19T16:30:06.457808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6B.2 INFERENCE BBOXES","metadata":{}},{"cell_type":"code","source":"def yolox_inference(img, model, test_size): \n bboxes = []\n bbclasses = []\n scores = []\n \n preproc = ValTransform(legacy = False)\n\n tensor_img, _ = preproc(img, None, test_size)\n tensor_img = torch.from_numpy(tensor_img).unsqueeze(0)\n tensor_img = tensor_img.float()\n tensor_img = tensor_img.cuda()\n\n with torch.no_grad():\n outputs = model(tensor_img)\n outputs = postprocess(\n outputs, num_classes, confthre,\n nmsthre, class_agnostic=True\n )\n\n if outputs[0] is None:\n return [], [], []\n \n outputs = outputs[0].cpu()\n bboxes = outputs[:, 0:4]\n\n bboxes /= min(test_size[0] / img.shape[0], test_size[1] / img.shape[1])\n bbclasses = outputs[:, 6]\n scores = outputs[:, 4] * outputs[:, 5]\n \n return bboxes, bbclasses, scores","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:30:06.463672Z","iopub.execute_input":"2021-12-19T16:30:06.464305Z","iopub.status.idle":"2021-12-19T16:30:06.473464Z","shell.execute_reply.started":"2021-12-19T16:30:06.464261Z","shell.execute_reply":"2021-12-19T16:30:06.472792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6B.3 DRAW RESULT","metadata":{}},{"cell_type":"code","source":"def draw_yolox_predictions(img, bboxes, scores, bbclasses, confthre, classes_dict):\n for i in range(len(bboxes)):\n box = bboxes[i]\n cls_id = int(bbclasses[i])\n score = scores[i]\n if score < confthre:\n continue\n x0 = int(box[0])\n y0 = int(box[1])\n x1 = int(box[2])\n y1 = int(box[3])\n\n cv2.rectangle(img, (x0, y0), (x1, y1), (0, 255, 0), 2)\n cv2.putText(img, '{}:{:.1f}%'.format(classes_dict[cls_id], score * 100), (x0, y0 - 3), cv2.FONT_HERSHEY_PLAIN, 0.8, (0,255,0), thickness = 1)\n return img","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:30:06.474703Z","iopub.execute_input":"2021-12-19T16:30:06.474973Z","iopub.status.idle":"2021-12-19T16:30:06.485044Z","shell.execute_reply.started":"2021-12-19T16:30:06.474938Z","shell.execute_reply":"2021-12-19T16:30:06.484273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6B.4 ALL PUZZLES TOGETHER","metadata":{}},{"cell_type":"code","source":"TEST_IMAGE_PATH = \"/kaggle/working/dataset/images/val2017/0-4614.jpg\"\nimg = cv2.imread(TEST_IMAGE_PATH)\n\n# Get predictions\nbboxes, bbclasses, scores = yolox_inference(img, model, test_size)\n\n# Draw predictions\nout_image = draw_yolox_predictions(img, bboxes, scores, bbclasses, confthre, COCO_CLASSES)\n\n# Since we load image using OpenCV we have to convert it \nout_image = cv2.cvtColor(out_image, cv2.COLOR_BGR2RGB)\ndisplay(Image.fromarray(out_image))","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:37:21.68211Z","iopub.execute_input":"2021-12-19T16:37:21.682407Z","iopub.status.idle":"2021-12-19T16:37:22.027035Z","shell.execute_reply.started":"2021-12-19T16:37:21.682379Z","shell.execute_reply":"2021-12-19T16:37:22.026393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. SUBMIT TO COTS COMPETITION AND EVALUATE","metadata":{}},{"cell_type":"code","source":"import greatbarrierreef\n\nenv = greatbarrierreef.make_env() # initialize the environment\niter_test = env.iter_test() ","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:36:38.938684Z","iopub.execute_input":"2021-12-19T16:36:38.939005Z","iopub.status.idle":"2021-12-19T16:36:38.965215Z","shell.execute_reply.started":"2021-12-19T16:36:38.938973Z","shell.execute_reply":"2021-12-19T16:36:38.96423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_dict = {\n 'id': [],\n 'prediction_string': [],\n}\n\nfor (image_np, sample_prediction_df) in iter_test:\n \n bboxes, bbclasses, scores = yolox_inference(image_np, model, test_size)\n \n predictions = []\n for i in range(len(bboxes)):\n box = bboxes[i]\n cls_id = int(bbclasses[i])\n score = scores[i]\n if score < confthre:\n continue\n x_min = int(box[0])\n y_min = int(box[1])\n x_max = int(box[2])\n y_max = int(box[3])\n \n bbox_width = x_max - x_min\n bbox_height = y_max - y_min\n \n predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n \n prediction_str = ' '.join(predictions)\n sample_prediction_df['annotations'] = prediction_str\n env.predict(sample_prediction_df)\n\n print('Prediction:', prediction_str)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:36:48.769327Z","iopub.execute_input":"2021-12-19T16:36:48.769612Z","iopub.status.idle":"2021-12-19T16:36:48.776991Z","shell.execute_reply.started":"2021-12-19T16:36:48.769579Z","shell.execute_reply":"2021-12-19T16:36:48.776302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('submission.csv')\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:37:08.932096Z","iopub.execute_input":"2021-12-19T16:37:08.932404Z","iopub.status.idle":"2021-12-19T16:37:08.961566Z","shell.execute_reply.started":"2021-12-19T16:37:08.932369Z","shell.execute_reply":"2021-12-19T16:37:08.960403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:35:25.253261Z","iopub.execute_input":"2021-12-19T16:35:25.253986Z","iopub.status.idle":"2021-12-19T16:35:25.26164Z","shell.execute_reply.started":"2021-12-19T16:35:25.253952Z","shell.execute_reply":"2021-12-19T16:35:25.26062Z"},"trusted":true},"execution_count":null,"outputs":[]}]}