{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Notebook on how to train a YOLOX_l Model for the Starfish Detection Competition. 🐠\n\n## Inspired on the notebook: [YoloX training pipeline COTS dataset [LB 0.507] !!](https://www.kaggle.com/remekkinas/yolox-training-pipeline-cots-dataset-lb-0-507)\n## Submission is perfomed using the notebook: [YoloX inference + Tracking on COTS [LB 0.539]](https://www.kaggle.com/parapapapam/yolox-inference-tracking-on-cots-lb-0-539)\n\n### Feel free to adjust the parameters and achieve a better LB. 🤛","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport ast\nimport os\nimport json\nimport pandas as pd\nimport torch\nimport importlib\nimport cv2 \n\nfrom shutil import copyfile\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nfrom sklearn.model_selection import GroupKFold\nfrom PIL import Image\nfrom string import Template\nfrom IPython.display import display\n\nTRAIN_PATH = '/kaggle/input/tensorflow-great-barrier-reef'","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:36:03.608416Z","iopub.execute_input":"2021-12-27T19:36:03.610546Z","iopub.status.idle":"2021-12-27T19:36:05.785791Z","shell.execute_reply.started":"2021-12-27T19:36:03.610489Z","shell.execute_reply":"2021-12-27T19:36:05.785034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set a GPU as an Accelerator","metadata":{}},{"cell_type":"code","source":"# check Torch and CUDA version\nprint(f\"Torch: {torch.__version__}\")\n!nvcc --version","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:36:05.787617Z","iopub.execute_input":"2021-12-27T19:36:05.787864Z","iopub.status.idle":"2021-12-27T19:36:06.461628Z","shell.execute_reply.started":"2021-12-27T19:36:05.787831Z","shell.execute_reply":"2021-12-27T19:36:06.460875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. INSTALL YOLOX","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/Megvii-BaseDetection/YOLOX -q\n#!git clone https://github.com/digantamisra98/YOLOX -q \n\n%cd YOLOX\n!pip install -U pip && pip install -r requirements.txt\n!pip install -v -e .","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:36:06.462798Z","iopub.execute_input":"2021-12-27T19:36:06.463014Z","iopub.status.idle":"2021-12-27T19:36:58.449645Z","shell.execute_reply.started":"2021-12-27T19:36:06.462987Z","shell.execute_reply":"2021-12-27T19:36:58.44879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:36:58.453743Z","iopub.execute_input":"2021-12-27T19:36:58.453968Z","iopub.status.idle":"2021-12-27T19:37:14.985546Z","shell.execute_reply.started":"2021-12-27T19:36:58.453937Z","shell.execute_reply":"2021-12-27T19:37:14.984626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. PREPARE COTS DATASET FOR YOLOX\nThis section is taken from notebook created by Awsaf\n[Great-Barrier-Reef: YOLOv5 [train] 🌊\n](https://www.kaggle.com/awsaf49/great-barrier-reef-yolov5-train)","metadata":{}},{"cell_type":"markdown","source":"## A. PREPARE DATASET AND ANNOTATIONS¶\n","metadata":{}},{"cell_type":"code","source":"def get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_path(row):\n    row['image_path'] = f'{TRAIN_PATH}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:37:14.987516Z","iopub.execute_input":"2021-12-27T19:37:14.987788Z","iopub.status.idle":"2021-12-27T19:37:14.993031Z","shell.execute_reply.started":"2021-12-27T19:37:14.987752Z","shell.execute_reply":"2021-12-27T19:37:14.992252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\ndf.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:37:14.994553Z","iopub.execute_input":"2021-12-27T19:37:14.995138Z","iopub.status.idle":"2021-12-27T19:37:15.06996Z","shell.execute_reply.started":"2021-12-27T19:37:14.995099Z","shell.execute_reply":"2021-12-27T19:37:15.069237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taken only annotated photos\ndf[\"num_bbox\"] = df['annotations'].apply(lambda x: str.count(x, 'x'))\ndf_train = df[df[\"num_bbox\"]>0]\n\n#Annotations \ndf_train['annotations'] = df_train['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndf_train['bboxes'] = df_train.annotations.progress_apply(get_bbox)\n\n#Images resolution\ndf_train[\"width\"] = 1280\ndf_train[\"height\"] = 720\n\n#Path of images\ndf_train = df_train.progress_apply(get_path, axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:37:15.071278Z","iopub.execute_input":"2021-12-27T19:37:15.071543Z","iopub.status.idle":"2021-12-27T19:37:18.682774Z","shell.execute_reply.started":"2021-12-27T19:37:15.07151Z","shell.execute_reply":"2021-12-27T19:37:18.682074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kf = GroupKFold(n_splits = 5) \ndf_train = df_train.reset_index(drop=True)\ndf_train['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(kf.split(df_train, y = df_train.video_id.tolist(), groups=df_train.sequence)):\n    df_train.loc[val_idx, 'fold'] = fold\n\ndf_train.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:37:18.683963Z","iopub.execute_input":"2021-12-27T19:37:18.684507Z","iopub.status.idle":"2021-12-27T19:37:18.714938Z","shell.execute_reply.started":"2021-12-27T19:37:18.684462Z","shell.execute_reply":"2021-12-27T19:37:18.714209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HOME_DIR = '/kaggle/working/' \nDATASET_PATH = 'dataset/images'\n\n!mkdir {HOME_DIR}dataset\n!mkdir {HOME_DIR}{DATASET_PATH}\n!mkdir {HOME_DIR}{DATASET_PATH}/train2017\n!mkdir {HOME_DIR}{DATASET_PATH}/val2017\n!mkdir {HOME_DIR}{DATASET_PATH}/annotations","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:37:18.716225Z","iopub.execute_input":"2021-12-27T19:37:18.716473Z","iopub.status.idle":"2021-12-27T19:37:22.245747Z","shell.execute_reply.started":"2021-12-27T19:37:18.716439Z","shell.execute_reply":"2021-12-27T19:37:22.244619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SELECTED_FOLD = 4\n\nfor i in tqdm(range(len(df_train))):\n    row = df_train.loc[i]\n    if row.fold != SELECTED_FOLD:\n        copyfile(f'{row.image_path}', f'{HOME_DIR}{DATASET_PATH}/train2017/{row.image_id}.jpg')\n    else:\n        copyfile(f'{row.image_path}', f'{HOME_DIR}{DATASET_PATH}/val2017/{row.image_id}.jpg') ","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:37:22.249511Z","iopub.execute_input":"2021-12-27T19:37:22.250008Z","iopub.status.idle":"2021-12-27T19:38:15.264032Z","shell.execute_reply.started":"2021-12-27T19:37:22.24996Z","shell.execute_reply":"2021-12-27T19:38:15.263339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Number of training files: {len(os.listdir(f\"{HOME_DIR}{DATASET_PATH}/train2017/\"))}')\nprint(f'Number of validation files: {len(os.listdir(f\"{HOME_DIR}{DATASET_PATH}/val2017/\"))}')","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:38:15.265288Z","iopub.execute_input":"2021-12-27T19:38:15.266047Z","iopub.status.idle":"2021-12-27T19:38:15.275125Z","shell.execute_reply.started":"2021-12-27T19:38:15.266007Z","shell.execute_reply":"2021-12-27T19:38:15.274447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_annot_json(json_annotation, filename):\n    with open(filename, 'w') as f:\n        output_json = json.dumps(json_annotation)\n        f.write(output_json)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:38:15.276415Z","iopub.execute_input":"2021-12-27T19:38:15.276836Z","iopub.status.idle":"2021-12-27T19:38:16.19237Z","shell.execute_reply.started":"2021-12-27T19:38:15.2768Z","shell.execute_reply":"2021-12-27T19:38:16.191447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annotion_id = 0","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:38:16.193816Z","iopub.execute_input":"2021-12-27T19:38:16.194097Z","iopub.status.idle":"2021-12-27T19:38:16.201827Z","shell.execute_reply.started":"2021-12-27T19:38:16.194059Z","shell.execute_reply":"2021-12-27T19:38:16.201044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dataset2coco(df, dest_path):\n    \n    global annotion_id\n    \n    annotations_json = {\n        \"info\": [],\n        \"licenses\": [],\n        \"categories\": [],\n        \"images\": [],\n        \"annotations\": []\n    }\n    \n    info = {\n        \"year\": \"2021\",\n        \"version\": \"1\",\n        \"description\": \"COTS dataset - COCO format\",\n        \"contributor\": \"\",\n        \"url\": \"https://kaggle.com\",\n        \"date_created\": \"2021-11-30T15:01:26+00:00\"\n    }\n    annotations_json[\"info\"].append(info)\n    \n    lic = {\n            \"id\": 1,\n            \"url\": \"\",\n            \"name\": \"Unknown\"\n        }\n    annotations_json[\"licenses\"].append(lic)\n\n    classes = {\"id\": 0, \"name\": \"starfish\", \"supercategory\": \"none\"}\n\n    annotations_json[\"categories\"].append(classes)\n\n    \n    for ann_row in df.itertuples():\n            \n        images = {\n            \"id\": ann_row[0],\n            \"license\": 1,\n            \"file_name\": ann_row.image_id + '.jpg',\n            \"height\": ann_row.height,\n            \"width\": ann_row.width,\n            \"date_captured\": \"2021-11-30T15:01:26+00:00\"\n        }\n        \n        annotations_json[\"images\"].append(images)\n        \n        bbox_list = ann_row.bboxes\n        \n        for bbox in bbox_list:\n            b_width = bbox[2]\n            b_height = bbox[3]\n            \n            # some boxes in COTS are outside the image height and width\n            if (bbox[0] + bbox[2] > 1280):\n                b_width = bbox[0] - 1280 \n            if (bbox[1] + bbox[3] > 720):\n                b_height = bbox[1] - 720 \n                \n            image_annotations = {\n                \"id\": annotion_id,\n                \"image_id\": ann_row[0],\n                \"category_id\": 0,\n                \"bbox\": [bbox[0], bbox[1], b_width, b_height],\n                \"area\": bbox[2] * bbox[3],\n                \"segmentation\": [],\n                \"iscrowd\": 0\n            }\n            \n            annotion_id += 1\n            annotations_json[\"annotations\"].append(image_annotations)\n        \n        \n    print(f\"Dataset COTS annotation to COCO json format completed! Files: {len(df)}\")\n    return annotations_json","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:38:16.203529Z","iopub.execute_input":"2021-12-27T19:38:16.203829Z","iopub.status.idle":"2021-12-27T19:38:16.217328Z","shell.execute_reply.started":"2021-12-27T19:38:16.20379Z","shell.execute_reply":"2021-12-27T19:38:16.216557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert COTS dataset to JSON COCO\ntrain_annot_json = dataset2coco(df_train[df_train.fold != SELECTED_FOLD], f\"{HOME_DIR}{DATASET_PATH}/train2017/\")\nval_annot_json = dataset2coco(df_train[df_train.fold == SELECTED_FOLD], f\"{HOME_DIR}{DATASET_PATH}/val2017/\")\n\n# Save converted annotations\nsave_annot_json(train_annot_json, f\"{HOME_DIR}{DATASET_PATH}/annotations/train.json\")\nsave_annot_json(val_annot_json, f\"{HOME_DIR}{DATASET_PATH}/annotations/valid.json\")","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:38:16.219925Z","iopub.execute_input":"2021-12-27T19:38:16.220119Z","iopub.status.idle":"2021-12-27T19:38:16.325538Z","shell.execute_reply.started":"2021-12-27T19:38:16.220087Z","shell.execute_reply":"2021-12-27T19:38:16.324227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. PREPARE CONFIGURATION FILE¶\n### Change the parameters to achieve better results.","metadata":{}},{"cell_type":"code","source":"config_file_template = \"\"\"\n\nimport os\n\nimport torch\nimport torch.distributed as dist\nimport torch.nn as nn\nimport random\n\nfrom yolox.exp import Exp as MyExp\n\nclass Exp(MyExp):\n    def __init__(self):\n        super().__init__()\n\n        # ---------------- model config ---------------- #\n        self.num_classes = 1\n        self.depth = 1 \n        self.width = 1 \n        self.act = 'silu'\n\n        # ---------------- dataloader config ---------------- #\n        # set worker to 4 for shorter dataloader init time\n        self.data_num_workers = 2\n        self.input_size = (960, 960)  # (height, width) \n        # Actual multiscale ranges: [640-5*32, 640+5*32].\n        # To disable multiscale training, set the\n        # self.multiscale_range to 0.\n        self.multiscale_range = 5\n        # You can uncomment this line to specify a multiscale range\n        # self.random_size = (14, 26)\n        self.data_dir = \"/kaggle/working/dataset/images\"\n        #self.train_ann = \"instances_train2017.json\"\n        self.train_ann = 'train.json'\n        self.val_ann = 'valid.json'\n\n        # --------------- transform config ----------------- #\n        self.mosaic_prob = 0.5\n        self.mixup_prob = 1\n        self.hsv_prob = 0.7\n        self.flip_prob = 0.7\n        self.degrees = 10.0\n        self.translate = 0.1\n        self.mosaic_scale = (0.5, 1.5)\n        self.mixup_scale = (0.5, 1.5)\n        self.shear = 2.0\n        self.enable_mixup = True\n\n        # --------------  training config --------------------- #\n        self.warmup_epochs = 5\n        self.max_epoch = $max_epoch\n        self.warmup_lr = 0\n        self.basic_lr_per_img = 0.01/64\n        self.scheduler = \"yoloxwarmcos\"\n        self.no_aug_epochs = 2\n        self.min_lr_ratio = 0.05\n        self.ema = True\n\n        self.weight_decay = 5e-4\n        self.momentum = 0.937\n        self.print_interval = 10\n        self.eval_interval = 1\n        self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n\n        # -----------------  testing config ------------------ #\n        self.test_size = (960,960)\n        self.test_conf = 0.01\n        self.nmsthre = 0.65\n\n    def get_model(self):\n        from yolox.models import YOLOX, YOLOPAFPN, YOLOXHead\n\n        def init_yolo(M):\n            for m in M.modules():\n                if isinstance(m, nn.BatchNorm2d):\n                    m.eps = 1e-3\n                    m.momentum = 0.03\n\n        if getattr(self, \"model\", None) is None:\n            in_channels = [256, 512, 1024]\n            backbone = YOLOPAFPN(self.depth, self.width, in_channels=in_channels, act=self.act)\n            head = YOLOXHead(self.num_classes, self.width, in_channels=in_channels, act=self.act)\n            self.model = YOLOX(backbone, head)\n\n        self.model.apply(init_yolo)\n        self.model.head.initialize_biases(1e-2)\n        return self.model\n\n    def get_data_loader(\n        self, batch_size, is_distributed, no_aug=False, cache_img=False\n    ):\n        from yolox.data import (\n            COCODataset,\n            TrainTransform,\n            YoloBatchSampler,\n            DataLoader,\n            InfiniteSampler,\n            MosaicDetection,\n            worker_init_reset_seed,\n        )\n        from yolox.utils import (\n            wait_for_the_master,\n            get_local_rank,\n        )\n\n        local_rank = get_local_rank()\n\n        with wait_for_the_master(local_rank):\n            dataset = COCODataset(\n                data_dir=self.data_dir,\n                json_file=self.train_ann,\n                img_size=self.input_size,\n                preproc=TrainTransform(\n                    max_labels=50,\n                    flip_prob=self.flip_prob,\n                    hsv_prob=self.hsv_prob),\n                cache=cache_img,\n            )\n\n        dataset = MosaicDetection(\n            dataset,\n            mosaic=not no_aug,\n            img_size=self.input_size,\n            preproc=TrainTransform(\n                max_labels=120,\n                flip_prob=self.flip_prob,\n                hsv_prob=self.hsv_prob),\n            degrees=self.degrees,\n            translate=self.translate,\n            mosaic_scale=self.mosaic_scale,\n            mixup_scale=self.mixup_scale,\n            shear=self.shear,\n            enable_mixup=self.enable_mixup,\n            mosaic_prob=self.mosaic_prob,\n            mixup_prob=self.mixup_prob,\n        )\n\n        self.dataset = dataset\n\n        if is_distributed:\n            batch_size = batch_size // dist.get_world_size()\n\n        sampler = InfiniteSampler(len(self.dataset), seed=self.seed if self.seed else 0)\n\n        batch_sampler = YoloBatchSampler(\n            sampler=sampler,\n            batch_size=batch_size,\n            drop_last=False,\n            mosaic=not no_aug,\n        )\n\n        dataloader_kwargs = {\"num_workers\": self.data_num_workers, \"pin_memory\": True}\n        dataloader_kwargs[\"batch_sampler\"] = batch_sampler\n\n        # Make sure each process has different random seed, especially for 'fork' method.\n        # Check https://github.com/pytorch/pytorch/issues/63311 for more details.\n        dataloader_kwargs[\"worker_init_fn\"] = worker_init_reset_seed\n\n        train_loader = DataLoader(self.dataset, **dataloader_kwargs)\n\n        return train_loader\n\n    def random_resize(self, data_loader, epoch, rank, is_distributed):\n        tensor = torch.LongTensor(2).cuda()\n\n        if rank == 0:\n            size_factor = self.input_size[1] * 1.0 / self.input_size[0]\n            if not hasattr(self, 'random_size'):\n                min_size = int(self.input_size[0] / 32) - self.multiscale_range\n                max_size = int(self.input_size[0] / 32) + self.multiscale_range\n                self.random_size = (min_size, max_size)\n            size = random.randint(*self.random_size)\n            size = (int(32 * size), 32 * int(size * size_factor))\n            tensor[0] = size[0]\n            tensor[1] = size[1]\n\n        if is_distributed:\n            dist.barrier()\n            dist.broadcast(tensor, 0)\n\n        input_size = (tensor[0].item(), tensor[1].item())\n        return input_size\n\n    def preprocess(self, inputs, targets, tsize):\n        scale_y = tsize[0] / self.input_size[0]\n        scale_x = tsize[1] / self.input_size[1]\n        if scale_x != 1 or scale_y != 1:\n            inputs = nn.functional.interpolate(\n                inputs, size=tsize, mode=\"bilinear\", align_corners=False\n            )\n            targets[..., 1::2] = targets[..., 1::2] * scale_x\n            targets[..., 2::2] = targets[..., 2::2] * scale_y\n        return inputs, targets\n\n    def get_optimizer(self, batch_size):\n        if \"optimizer\" not in self.__dict__:\n            if self.warmup_epochs > 0:\n                lr = self.warmup_lr\n            else:\n                lr = self.basic_lr_per_img * batch_size\n\n            pg0, pg1, pg2 = [], [], []  # optimizer parameter groups\n\n            for k, v in self.model.named_modules():\n                if hasattr(v, \"bias\") and isinstance(v.bias, nn.Parameter):\n                    pg2.append(v.bias)  # biases\n                if isinstance(v, nn.BatchNorm2d) or \"bn\" in k:\n                    pg0.append(v.weight)  # no decay\n                elif hasattr(v, \"weight\") and isinstance(v.weight, nn.Parameter):\n                    pg1.append(v.weight)  # apply decay\n\n            optimizer = torch.optim.SGD(\n                pg0, lr=lr, momentum=self.momentum, nesterov=True\n            )\n            optimizer.add_param_group(\n                {\"params\": pg1, \"weight_decay\": self.weight_decay}\n            )  # add pg1 with weight_decay\n            optimizer.add_param_group({\"params\": pg2})\n            self.optimizer = optimizer\n\n        return self.optimizer\n\n    def get_lr_scheduler(self, lr, iters_per_epoch):\n        from yolox.utils import LRScheduler\n\n        scheduler = LRScheduler(\n            self.scheduler,\n            lr,\n            #iters_per_epoch,\n            50,\n            self.max_epoch,\n            warmup_epochs=self.warmup_epochs,\n            warmup_lr_start=self.warmup_lr,\n            no_aug_epochs=self.no_aug_epochs,\n            min_lr_ratio=self.min_lr_ratio,\n        )\n        return scheduler\n\n    def get_eval_loader(self, batch_size, is_distributed, testdev=False, legacy=False):\n        from yolox.data import COCODataset, ValTransform\n\n        valdataset = COCODataset(\n            data_dir=self.data_dir,\n            json_file=self.val_ann if not testdev else \"image_info_test-dev2017.json\",\n            name=\"val2017\" if not testdev else \"test2017\",\n            img_size=self.test_size,\n            preproc=ValTransform(legacy=legacy),\n        )\n\n        if is_distributed:\n            batch_size = batch_size // dist.get_world_size()\n            sampler = torch.utils.data.distributed.DistributedSampler(\n                valdataset, shuffle=False\n            )\n        else:\n            sampler = torch.utils.data.SequentialSampler(valdataset)\n\n        dataloader_kwargs = {\n            \"num_workers\": self.data_num_workers,\n            \"pin_memory\": True,\n            \"sampler\": sampler,\n        }\n        dataloader_kwargs[\"batch_size\"] = batch_size\n        val_loader = torch.utils.data.DataLoader(valdataset, **dataloader_kwargs)\n\n        return val_loader\n\n    def get_evaluator(self, batch_size, is_distributed, testdev=False, legacy=False):\n        from yolox.evaluators import COCOEvaluator\n\n        val_loader = self.get_eval_loader(batch_size, is_distributed, testdev, legacy)\n        evaluator = COCOEvaluator(\n            dataloader=val_loader,\n            img_size=self.test_size,\n            confthre=self.test_conf,\n            nmsthre=self.nmsthre,\n            num_classes=self.num_classes,\n            testdev=testdev,\n        )\n        return evaluator\n\n    def eval(self, model, evaluator, is_distributed, half=False):\n        return evaluator.evaluate(model, is_distributed, half)\n\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define number of Epochs on **max_epoch**","metadata":{}},{"cell_type":"code","source":"PIPELINE_CONFIG_PATH='cots_config.py'\n\npipeline = Template(config_file_template).substitute(max_epoch = 20)\n\nwith open(PIPELINE_CONFIG_PATH, 'w') as f:\n    f.write(pipeline)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ./yolox/data/datasets/voc_classes.py\n\nvoc_cls = '''\nVOC_CLASSES = (\n  \"starfish\",\n)\n'''\nwith open('./yolox/data/datasets/voc_classes.py', 'w') as f:\n    f.write(voc_cls)\n\n# ./yolox/data/datasets/coco_classes.py\n\ncoco_cls = '''\nCOCO_CLASSES = (\n  \"starfish\",\n)\n'''\nwith open('./yolox/data/datasets/coco_classes.py', 'w') as f:\n    f.write(coco_cls)\n\n# check if everything is ok    \n!more ./yolox/data/datasets/coco_classes.py","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:38:16.353508Z","iopub.execute_input":"2021-12-27T19:38:16.353769Z","iopub.status.idle":"2021-12-27T19:38:17.016351Z","shell.execute_reply.started":"2021-12-27T19:38:16.353734Z","shell.execute_reply":"2021-12-27T19:38:17.015502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. DOWNLOAD PRETRAINED WEIGHTS¶\n","metadata":{}},{"cell_type":"code","source":"sh = 'wget https://github.com/Megvii-BaseDetection/storage/releases/download/0.0.1/yolox_l.pth'\nMODEL_FILE = 'yolox_l.pth'\n\nwith open('script.sh', 'w') as file:\n  file.write(sh)\n\n!bash script.sh","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:38:17.018239Z","iopub.execute_input":"2021-12-27T19:38:17.018518Z","iopub.status.idle":"2021-12-27T19:38:28.160704Z","shell.execute_reply.started":"2021-12-27T19:38:17.018479Z","shell.execute_reply":"2021-12-27T19:38:28.15991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. TRAIN MODEL¶","metadata":{}},{"cell_type":"code","source":"!cp ./tools/train.py ./","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:38:28.177648Z","iopub.execute_input":"2021-12-27T19:38:28.17795Z","iopub.status.idle":"2021-12-27T19:38:28.841876Z","shell.execute_reply.started":"2021-12-27T19:38:28.177915Z","shell.execute_reply":"2021-12-27T19:38:28.840957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python train.py \\\n    -f cots_config.py \\\n    -d 1 \\\n    -b 8 \\\n    --fp16 \\\n    -o \\\n    -c {MODEL_FILE}   # Remember to chenge this line if you take different model eg. yolo_nano.pth, yolox_s.pth or yolox_m.pth\\","metadata":{"execution":{"iopub.status.busy":"2021-12-27T19:49:10.897015Z","iopub.execute_input":"2021-12-27T19:49:10.897298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Go to [YoloX inference + Tracking on COTS [LB 0.539]](https://www.kaggle.com/parapapapam/yolox-inference-tracking-on-cots-lb-0-539) to perform the inference, using the output file. 😁","metadata":{}}]}