{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Installing YOLOX Libraries","metadata":{}},{"cell_type":"markdown","source":"Credit over to Remek Kinas for the following code: \n\nCode: https://www.kaggle.com/remekkinas/yolox-inference-on-kaggle-for-cots-lb-0-507?scriptVersionId=81625924","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport ast\nfrom sklearn.model_selection import GroupKFold\nfrom string import Template\nimport json\nimport torch\nfrom shutil import copyfile\nimport greatbarrierreef\nimport importlib\nimport random\nimport cv2\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom IPython.display import display","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:40:17.692627Z","iopub.execute_input":"2022-01-31T20:40:17.692959Z","iopub.status.idle":"2022-01-31T20:40:19.949683Z","shell.execute_reply.started":"2022-01-31T20:40:17.692855Z","shell.execute_reply":"2022-01-31T20:40:19.948969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/Megvii-BaseDetection/YOLOX -q\n\n%cd YOLOX\n!pip install -U pip && pip install -r requirements.txt\n!pip install -v -e . ","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:40:19.951444Z","iopub.execute_input":"2022-01-31T20:40:19.951827Z","iopub.status.idle":"2022-01-31T20:41:50.561630Z","shell.execute_reply.started":"2022-01-31T20:40:19.951797Z","shell.execute_reply":"2022-01-31T20:41:50.560808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:41:50.564982Z","iopub.execute_input":"2022-01-31T20:41:50.565247Z","iopub.status.idle":"2022-01-31T20:42:08.970605Z","shell.execute_reply.started":"2022-01-31T20:41:50.565216Z","shell.execute_reply":"2022-01-31T20:42:08.969781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_SPLITS = 5\nFOLD = random.randint(0, N_SPLITS - 1)\nprint(FOLD)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:42:08.973066Z","iopub.execute_input":"2022-01-31T20:42:08.973835Z","iopub.status.idle":"2022-01-31T20:42:08.979304Z","shell.execute_reply.started":"2022-01-31T20:42:08.973794Z","shell.execute_reply":"2022-01-31T20:42:08.978603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Splitting Data Into Training/Validation","metadata":{}},{"cell_type":"markdown","source":"**EDIT Version 12**: Because we will be cutting out any entries that do not contain any starfish, we will use GroupKFolds to create our folds. ","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:42:08.981964Z","iopub.execute_input":"2022-01-31T20:42:08.982467Z","iopub.status.idle":"2022-01-31T20:42:09.087513Z","shell.execute_reply.started":"2022-01-31T20:42:08.982428Z","shell.execute_reply":"2022-01-31T20:42:09.086826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['annotations'] = train['annotations'].apply(lambda x: ast.literal_eval(x))\ntrain['box'] = train['annotations'].apply(lambda x: [list(y.values()) for y in x])\ntrain['image_path'] = \"video_\" + train['video_id'].astype(str) + \"/\" + train['video_frame'].astype(str) + \".jpg\"\ntrain['fold'] = -1\ntrain = train[train['annotations'].str.len() > 0].reset_index(drop = True)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:42:09.088785Z","iopub.execute_input":"2022-01-31T20:42:09.089023Z","iopub.status.idle":"2022-01-31T20:42:09.641364Z","shell.execute_reply.started":"2022-01-31T20:42:09.088991Z","shell.execute_reply":"2022-01-31T20:42:09.640649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kf = GroupKFold(n_splits=N_SPLITS)\nfor fold, (train_idx, val_idx) in enumerate(kf.split(train, y = train.video_id.tolist(), groups = train.sequence)):\n    train.loc[val_idx, 'fold'] = fold\ntrain['fold'] = train['fold'].astype(int)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:42:09.642492Z","iopub.execute_input":"2022-01-31T20:42:09.642729Z","iopub.status.idle":"2022-01-31T20:42:09.669295Z","shell.execute_reply.started":"2022-01-31T20:42:09.642697Z","shell.execute_reply":"2022-01-31T20:42:09.668485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/dataset\n!mkdir /kaggle/working/dataset/images\n!mkdir /kaggle/working/dataset/images/train2017\n!mkdir /kaggle/working/dataset/images/val2017\n!mkdir /kaggle/working/dataset/images/annotations","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:42:09.670699Z","iopub.execute_input":"2022-01-31T20:42:09.670963Z","iopub.status.idle":"2022-01-31T20:42:12.978366Z","shell.execute_reply.started":"2022-01-31T20:42:09.670929Z","shell.execute_reply":"2022-01-31T20:42:12.977475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(train)):\n    row = train.loc[i]\n    if(row.fold != FOLD):\n        copyfile(f'/kaggle/input/tensorflow-great-barrier-reef/train_images/{row.image_path}', f'/kaggle/working/dataset/images/train2017/{row.image_id}.jpg')\n    else:\n        copyfile(f'/kaggle/input/tensorflow-great-barrier-reef/train_images/{row.image_path}', f'/kaggle/working/dataset/images/val2017/{row.image_id}.jpg')","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:42:12.979964Z","iopub.execute_input":"2022-01-31T20:42:12.980240Z","iopub.status.idle":"2022-01-31T20:43:12.658140Z","shell.execute_reply.started":"2022-01-31T20:42:12.980200Z","shell.execute_reply":"2022-01-31T20:43:12.657338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Convert Dataset to YOLOX Format","metadata":{}},{"cell_type":"markdown","source":"There are two kinds of evaluators that can be used: the COCO evaluator and the VOC evaluator, both requiring specific formats. Only the COCO evaluator will be made for the time being. ","metadata":{}},{"cell_type":"code","source":"#COCO:\ndef datasetToCOCO(dataset):\n    annotation = 0\n    json = {\n        \"info\": [],\n        \"licenses\": [],\n        \"categories\": [],\n        \"images\": [],\n        \"annotations\": []\n    }\n    info = {\n        \"year\": \"2021\",\n        \"version\": \"1\",\n        \"description\": \"COTS dataset - COCO format\",\n        \"contributor\": \"\",\n        \"url\": \"https://kaggle.com\",\n        \"date_created\": \"2021-11-30T15:01:26+00:00\"\n    }\n    json['info'].append(info)\n    licenses = {\n        \"id\": 1,\n        \"url\": \"\",\n        \"name\": \"Unknown\"\n    }\n    json['licenses'].append(licenses)\n    categories = {\n        \"id\": 0, \n        \"name\": \"starfish\", \n        \"supercategory\": \"none\"\n    }\n    json[\"categories\"].append(categories)\n    for row in dataset.itertuples():\n        images = {\n            \"id\": row[0],\n            \"license\": 1,\n            \"file_name\": row.image_id + '.jpg',\n            \"height\": 720,\n            \"width\": 1280,\n            \"date_captured\": \"2021-11-30T15:01:26+00:00\"\n        }\n        json['images'].append(images)\n        boxes = row.box\n        for box in boxes:\n            width = box[2]\n            height = box[3]\n            if (box[0] + box[2] > 1280):\n                width = 1280 - box[0] \n            if (box[1] + box[3] > 720):\n                height = 720 - box[1] \n            annotations = {\n                \"id\": annotation,\n                \"image_id\": row[0],\n                \"category_id\": 0,\n                \"bbox\": [box[0], box[1], width, height],\n                \"area\": box[2] * box[3],\n                \"segmentation\": [],\n                \"iscrowd\": 0\n            }\n            annotation += 1\n            json['annotations'].append(annotations)\n    return json","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:43:12.659498Z","iopub.execute_input":"2022-01-31T20:43:12.659753Z","iopub.status.idle":"2022-01-31T20:43:12.671997Z","shell.execute_reply.started":"2022-01-31T20:43:12.659719Z","shell.execute_reply":"2022-01-31T20:43:12.671324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_coco = datasetToCOCO(train[train['fold'] != FOLD])\nwith open(f\"/kaggle/working/dataset/images/annotations/train.json\", 'w') as f:\n    output_json = json.dumps(train_coco)\n    f.write(output_json)\nval_coco = datasetToCOCO(train[train['fold'] == FOLD])\nwith open(f\"/kaggle/working/dataset/images/annotations/val.json\", 'w') as f:\n    output_json = json.dumps(val_coco)\n    f.write(output_json)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:46:15.195296Z","iopub.execute_input":"2022-01-31T20:46:15.196091Z","iopub.status.idle":"2022-01-31T20:46:15.294103Z","shell.execute_reply.started":"2022-01-31T20:46:15.196039Z","shell.execute_reply":"2022-01-31T20:46:15.293329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Apply Evaluator","metadata":{}},{"cell_type":"markdown","source":"There are multiple kinds of YOLOX models, but for the time being, we will just focus on YOLOX-s.","metadata":{}},{"cell_type":"code","source":"template = '''\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n    def __init__(self):\n        super(Exp, self).__init__()\n        self.num_classes = 1\n        self.depth = 0.33\n        self.width = 0.50\n        self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n        self.data_dir = \"/kaggle/working/dataset/images/\"\n        self.train_ann = 'train.json'\n        self.val_ann = 'val.json'\n        self.max_epoch = $max_epoch\n        #self.eval_interval = 1\n        #self.data_num_workers = 2\n        #self.input_size = (960, 960)\n        #self.test_size = (960, 960)\n        #self.no_aug_epochs = 2\n        #self.mosaic_scale = (0.5, 1.5)\n        #self.random_size = (10, 20)\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:46:17.504758Z","iopub.execute_input":"2022-01-31T20:46:17.505165Z","iopub.status.idle":"2022-01-31T20:46:17.511941Z","shell.execute_reply.started":"2022-01-31T20:46:17.505128Z","shell.execute_reply":"2022-01-31T20:46:17.510861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline = Template(template).substitute(max_epoch = 20)\nwith open('cots_config.py', 'w') as f:\n    f.write(pipeline)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:46:19.189587Z","iopub.execute_input":"2022-01-31T20:46:19.189832Z","iopub.status.idle":"2022-01-31T20:46:19.195936Z","shell.execute_reply.started":"2022-01-31T20:46:19.189804Z","shell.execute_reply":"2022-01-31T20:46:19.193943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#VERSION 14: Added voc_cls to see if any differences arise.\nvoc_cls = '''\nVOC_CLASSES = (\n  \"starfish\",\n)\n'''\nwith open('/kaggle/working/YOLOX/yolox/data/datasets/voc_classes.py', 'w') as f:\n    f.write(voc_cls)\n\ncoco_cls = '''\nCOCO_CLASSES = (\n  \"starfish\",\n)\n'''\nwith open('/kaggle/working/YOLOX/yolox/data/datasets/coco_classes.py', 'w') as f:\n    f.write(coco_cls)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:46:21.615166Z","iopub.execute_input":"2022-01-31T20:46:21.615828Z","iopub.status.idle":"2022-01-31T20:46:21.622584Z","shell.execute_reply.started":"2022-01-31T20:46:21.615788Z","shell.execute_reply":"2022-01-31T20:46:21.621592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Weights","metadata":{}},{"cell_type":"markdown","source":"And now to import the weights file for YOLOX-s.","metadata":{}},{"cell_type":"code","source":"sh = 'wget https://github.com/Megvii-BaseDetection/storage/releases/download/0.0.1/yolox_s.pth'\nMODEL = 'yolox_s.pth'\nwith open('script.sh', 'w') as file:\n    file.write(sh)\n!bash script.sh","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:47:18.814246Z","iopub.execute_input":"2022-01-31T20:47:18.814544Z","iopub.status.idle":"2022-01-31T20:47:20.327252Z","shell.execute_reply.started":"2022-01-31T20:47:18.814509Z","shell.execute_reply":"2022-01-31T20:47:20.326471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Training","metadata":{}},{"cell_type":"code","source":"!cp /kaggle/working/YOLOX/tools/train.py ./","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:47:22.604166Z","iopub.execute_input":"2022-01-31T20:47:22.604813Z","iopub.status.idle":"2022-01-31T20:47:23.272884Z","shell.execute_reply.started":"2022-01-31T20:47:22.604771Z","shell.execute_reply":"2022-01-31T20:47:23.271976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python train.py -f cots_config.py -d 1 -b 32 --fp16 -o -c {MODEL}","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:47:24.609787Z","iopub.execute_input":"2022-01-31T20:47:24.610150Z","iopub.status.idle":"2022-01-31T20:50:51.811164Z","shell.execute_reply.started":"2022-01-31T20:47:24.610108Z","shell.execute_reply":"2022-01-31T20:50:51.810331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Evaluation","metadata":{}},{"cell_type":"markdown","source":"The following was taken from Remek Kina's excellent training pipeline notebook: https://www.kaggle.com/remekkinas/yolox-training-pipeline-cots-dataset-lb-0-507","metadata":{}},{"cell_type":"code","source":"from yolox.utils import postprocess\nfrom yolox.data.data_augment import ValTransform","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:50:51.814898Z","iopub.execute_input":"2022-01-31T20:50:51.815131Z","iopub.status.idle":"2022-01-31T20:50:51.929979Z","shell.execute_reply.started":"2022-01-31T20:50:51.815103Z","shell.execute_reply":"2022-01-31T20:50:51.928136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"current_exp = importlib.import_module('cots_config')\nexp = current_exp.Exp()\ntest_size = (640, 640)\nnum_classes = 1\nconfthre = 0.01\nnmsthre = 0.65\nmodel = exp.get_model()\nmodel.cuda()\nmodel.eval()\nckpt = torch.load(\"/kaggle/working/YOLOX/YOLOX_outputs/cots_config/best_ckpt.pth\", map_location=\"cpu\")\nmodel.load_state_dict(ckpt[\"model\"])","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:50:51.935210Z","iopub.execute_input":"2022-01-31T20:50:51.935698Z","iopub.status.idle":"2022-01-31T20:50:54.571743Z","shell.execute_reply.started":"2022-01-31T20:50:51.935645Z","shell.execute_reply":"2022-01-31T20:50:54.571043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/\ndef yolox_inference(img, model, test_size): \n    bboxes = []\n    bbclasses = []\n    scores = []\n    \n    preproc = ValTransform(legacy = False)\n\n    tensor_img, _ = preproc(img, None, test_size)\n    tensor_img = torch.from_numpy(tensor_img).unsqueeze(0)\n    tensor_img = tensor_img.float()\n    tensor_img = tensor_img.cuda()\n\n    with torch.no_grad():\n        outputs = model(tensor_img)\n        outputs = postprocess(\n                    outputs, num_classes, confthre,\n                    nmsthre, class_agnostic=True\n                )\n\n    if outputs[0] is None:\n        return [], [], []\n    \n    outputs = outputs[0].cpu()\n    bboxes = outputs[:, 0:4]\n\n    bboxes /= min(test_size[0] / img.shape[0], test_size[1] / img.shape[1])\n    bbclasses = outputs[:, 6]\n    scores = outputs[:, 4] * outputs[:, 5]\n    \n    return bboxes, bbclasses, scores","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:50:54.573780Z","iopub.execute_input":"2022-01-31T20:50:54.574220Z","iopub.status.idle":"2022-01-31T20:50:54.588296Z","shell.execute_reply.started":"2022-01-31T20:50:54.574183Z","shell.execute_reply":"2022-01-31T20:50:54.587457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:50:54.589643Z","iopub.execute_input":"2022-01-31T20:50:54.589886Z","iopub.status.idle":"2022-01-31T20:50:54.713171Z","shell.execute_reply.started":"2022-01-31T20:50:54.589853Z","shell.execute_reply":"2022-01-31T20:50:54.712397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_dict = {\n    'id': [],\n    'prediction_string': [],\n}\n\nfor (image_np, sample_prediction_df) in iter_test:\n \n    bboxes, bbclasses, scores = yolox_inference(image_np, model, test_size)\n    \n    predictions = []\n    for i in range(len(bboxes)):\n        box = bboxes[i]\n        cls_id = int(bbclasses[i])\n        score = scores[i]\n        if score < confthre:\n            continue\n        x_min = int(box[0])\n        y_min = int(box[1])\n        x_max = int(box[2])\n        y_max = int(box[3])\n        \n        bbox_width = x_max - x_min\n        bbox_height = y_max - y_min\n        \n        predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n    \n    prediction_str = ' '.join(predictions)\n    sample_prediction_df['annotations'] = prediction_str\n    env.predict(sample_prediction_df)\n\n    print('Prediction:', prediction_str)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:50:54.714630Z","iopub.execute_input":"2022-01-31T20:50:54.714965Z","iopub.status.idle":"2022-01-31T20:50:55.838883Z","shell.execute_reply.started":"2022-01-31T20:50:54.714873Z","shell.execute_reply":"2022-01-31T20:50:55.838157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('submission.csv')\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T20:50:55.840072Z","iopub.execute_input":"2022-01-31T20:50:55.841113Z","iopub.status.idle":"2022-01-31T20:50:55.852373Z","shell.execute_reply.started":"2022-01-31T20:50:55.841070Z","shell.execute_reply":"2022-01-31T20:50:55.851486Z"},"trusted":true},"execution_count":null,"outputs":[]}]}