{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# YOLOXの準備","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/Megvii-BaseDetection/YOLOX -q\n\n%cd YOLOX\n!pip install -U pip && pip install -r requirements.txt\n!pip install -v -e . \n!pip install 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'\n!pip install Cython","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:29:13.950376Z","iopub.execute_input":"2022-01-17T08:29:13.950690Z","iopub.status.idle":"2022-01-17T08:30:35.767429Z","shell.execute_reply.started":"2022-01-17T08:29:13.950606Z","shell.execute_reply":"2022-01-17T08:30:35.766536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# データセットの準備","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working\n\nfrom tqdm import tqdm\ntqdm.pandas()\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport ast\nimport os\nimport json\nimport pandas as pd\nimport torch\nimport importlib\nimport cv2 \n\nfrom shutil import copyfile\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nfrom sklearn.model_selection import GroupKFold\nfrom PIL import Image\nfrom string import Template\nfrom IPython.display import display","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:33:09.931714Z","iopub.execute_input":"2022-01-17T08:33:09.931995Z","iopub.status.idle":"2022-01-17T08:33:09.942400Z","shell.execute_reply.started":"2022-01-17T08:33:09.931963Z","shell.execute_reply":"2022-01-17T08:33:09.941467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_path(row):\n    row['image_path'] = f'{TRAIN_PATH}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:45:04.226628Z","iopub.execute_input":"2022-01-17T08:45:04.226902Z","iopub.status.idle":"2022-01-17T08:45:04.233524Z","shell.execute_reply.started":"2022-01-17T08:45:04.226864Z","shell.execute_reply":"2022-01-17T08:45:04.232773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train.csvの読み出し","metadata":{}},{"cell_type":"code","source":"import pandas as pd\ndf = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:45:04.706343Z","iopub.execute_input":"2022-01-17T08:45:04.706786Z","iopub.status.idle":"2022-01-17T08:45:04.747138Z","shell.execute_reply.started":"2022-01-17T08:45:04.706751Z","shell.execute_reply":"2022-01-17T08:45:04.746340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## アノテーションがある部分のみ抽出","metadata":{}},{"cell_type":"code","source":"# Taken only annotated photos\ndf[\"num_bbox\"] = df['annotations'].apply(lambda x: str.count(x, 'x'))\ndf_train = df[df[\"num_bbox\"]>0]\ndf_train.head()\n\n\n#Annotations \ndf_train['annotations'] = df_train['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndf_train['bboxes'] = df_train.annotations.progress_apply(get_bbox)\n\n#Images resolution\ndf_train[\"width\"] = 1280\ndf_train[\"height\"] = 720\n\n#Path of images\nTRAIN_PATH = '/kaggle/input/tensorflow-great-barrier-reef'\ndf_train = df_train.progress_apply(get_path, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:45:05.122213Z","iopub.execute_input":"2022-01-17T08:45:05.122456Z","iopub.status.idle":"2022-01-17T08:45:08.655865Z","shell.execute_reply.started":"2022-01-17T08:45:05.122427Z","shell.execute_reply":"2022-01-17T08:45:08.655061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ヒトデのカット&ペースト","metadata":{}},{"cell_type":"code","source":"SELECTED_VIDEO = 2","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:45:09.636676Z","iopub.execute_input":"2022-01-17T08:45:09.636950Z","iopub.status.idle":"2022-01-17T08:45:09.640866Z","shell.execute_reply.started":"2022-01-17T08:45:09.636901Z","shell.execute_reply":"2022-01-17T08:45:09.640031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n!rm -r /kaggle/working/cut_paste/\n!mkdir -p /kaggle/working/cut_paste/video_0/\n!mkdir -p /kaggle/working/cut_paste/video_1/\n!mkdir -p /kaggle/working/cut_paste/video_2/\n\nimport cv2\nimport random\nimport torchvision.ops.boxes as bops\nimport copy\ndf_train\n\nPASTE_NUM = 20\nbboxes_list = []\nimg_path = []\nfor i in tqdm(range(len(df_train))):\n    if df_train.iloc[i]['video_id'] == SELECTED_VIDEO:\n        bboxes_list.append(df_train.iloc[i]['bboxes'])\n        img_path.append(df_train.iloc[i]['image_path'])\n        continue\n    bboxes = copy.deepcopy(df_train.iloc[i]['bboxes'])\n    new_bbox_list = copy.deepcopy(bboxes)\n    path = df_train.iloc[i]['image_path']\n    video_id = df_train.iloc[i]['video_id']\n    video_frame = df_train.iloc[i]['video_frame']\n    img = cv2.imread(path)\n    for i in range(PASTE_NUM):\n        bbox_tensor = torch.tensor([[bbox[0] , bbox[1] , bbox[0]+bbox[2] , bbox[1]+bbox[3]] for bbox in new_bbox_list])\n        bbox = random.choice(bboxes)\n        cut_img = img[bbox[1]:bbox[1]+bbox[3] , bbox[0]:bbox[0]+bbox[2]]\n        point = random.randint(0 , 1280 - bbox[2]) ,random.randint(0 , 720 - bbox[3] ) \n        iou = bops.box_iou(bbox_tensor , torch.tensor([[point[0]  , point[1] ,point[0] +  bbox[2] ,point[1]+  bbox[3]]]))\n        if(torch.sum(iou) > 0):\n            continue\n        try:\n            img[point[1]:point[1]+bbox[3]  , point[0]:point[0] + bbox[2]] = cut_img\n            new_bbox_list.append([point[0], point[1] , bbox[2]  , bbox[3]])\n        except:\n            continue\n    bboxes_list.append(new_bbox_list)\n\n    cv2.imwrite(f'/kaggle/working/cut_paste/video_{video_id}/{video_frame}.jpg' , img) \n    img_path.append(f'/kaggle/working/cut_paste/video_{video_id}/{video_frame}.jpg')","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:45:10.531877Z","iopub.execute_input":"2022-01-17T08:45:10.532342Z","iopub.status.idle":"2022-01-17T08:48:17.670395Z","shell.execute_reply.started":"2022-01-17T08:45:10.532306Z","shell.execute_reply":"2022-01-17T08:48:17.669658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['bboxes'] = bboxes_list\ndf_train['image_path'] = img_path\ndf_train","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:44:24.315498Z","iopub.execute_input":"2022-01-17T08:44:24.316318Z","iopub.status.idle":"2022-01-17T08:44:24.428880Z","shell.execute_reply.started":"2022-01-17T08:44:24.316276Z","shell.execute_reply":"2022-01-17T08:44:24.428124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## COCOフォーマットに変更","metadata":{}},{"cell_type":"code","source":"HOME_DIR = '/kaggle/working/' \nDATASET_PATH = f'kaggle_dataset/images'\n\n!rm -r {HOME_DIR}{DATASET_PATH}\n\n!mkdir {HOME_DIR}kaggle_dataset\n!mkdir {HOME_DIR}{DATASET_PATH}\n!mkdir {HOME_DIR}{DATASET_PATH}/train2017\n!mkdir {HOME_DIR}{DATASET_PATH}/val2017\n!mkdir {HOME_DIR}{DATASET_PATH}/annotations\n\n\nfor i in tqdm(range(len(df_train))):\n    row = df_train.iloc[i]\n    if row.video_id != SELECTED_VIDEO:\n        copyfile(f'{row.image_path}', f'{HOME_DIR}{DATASET_PATH}/train2017/{row.image_id}.jpg')\n    else:\n        copyfile(f'{row.image_path}', f'{HOME_DIR}{DATASET_PATH}/val2017/{row.image_id}.jpg') \nprint(f'FOLD {SELECTED_VIDEO} Number of training files: {len(os.listdir(f\"{HOME_DIR}{DATASET_PATH}/train2017/\"))}')\nprint(f'FOLD {SELECTED_VIDEO} Number of validation files: {len(os.listdir(f\"{HOME_DIR}{DATASET_PATH}/val2017/\"))}')","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:51:14.668206Z","iopub.execute_input":"2022-01-17T08:51:14.668774Z","iopub.status.idle":"2022-01-17T08:51:35.947144Z","shell.execute_reply.started":"2022-01-17T08:51:14.668729Z","shell.execute_reply":"2022-01-17T08:51:35.946258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_annot_json(json_annotation, filename):\n    with open(filename, 'w') as f:\n        output_json = json.dumps(json_annotation)\n        f.write(output_json)\n\nannotion_id = 0","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:51:39.935329Z","iopub.execute_input":"2022-01-17T08:51:39.935591Z","iopub.status.idle":"2022-01-17T08:51:39.941671Z","shell.execute_reply.started":"2022-01-17T08:51:39.935560Z","shell.execute_reply":"2022-01-17T08:51:39.939982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dataset2coco(df, dest_path):\n    \n    global annotion_id\n    \n    annotations_json = {\n        \"info\": [],\n        \"licenses\": [],\n        \"categories\": [],\n        \"images\": [],\n        \"annotations\": []\n    }\n    \n    info = {\n        \"year\": \"2021\",\n        \"version\": \"1\",\n        \"description\": \"COTS dataset - COCO format\",\n        \"contributor\": \"\",\n        \"url\": \"https://kaggle.com\",\n        \"date_created\": \"2021-11-30T15:01:26+00:00\"\n    }\n    annotations_json[\"info\"].append(info)\n    \n    lic = {\n            \"id\": 1,\n            \"url\": \"\",\n            \"name\": \"Unknown\"\n        }\n    annotations_json[\"licenses\"].append(lic)\n\n    classes = {\"id\": 0, \"name\": \"starfish\", \"supercategory\": \"none\"}\n\n    annotations_json[\"categories\"].append(classes)\n\n    \n    for ann_row in df.itertuples():\n            \n        images = {\n            \"id\": ann_row[0],\n            \"license\": 1,\n            \"file_name\": ann_row.image_id + '.jpg',\n            \"height\": ann_row.height,\n            \"width\": ann_row.width,\n            \"date_captured\": \"2021-11-30T15:01:26+00:00\"\n        }\n        \n        annotations_json[\"images\"].append(images)\n        \n        bbox_list = ann_row.bboxes\n        \n        for bbox in bbox_list:\n            b_width = bbox[2]\n            b_height = bbox[3]\n            \n            # some boxes in COTS are outside the image height and width\n            if (bbox[0] + bbox[2] > 1280):\n                b_width = bbox[0] - 1280 \n            if (bbox[1] + bbox[3] > 720):\n                b_height = bbox[1] - 720 \n                \n            image_annotations = {\n                \"id\": annotion_id,\n                \"image_id\": ann_row[0],\n                \"category_id\": 0,\n                \"bbox\": [bbox[0], bbox[1], b_width, b_height],\n                \"area\": bbox[2] * bbox[3],\n                \"segmentation\": [],\n                \"iscrowd\": 0\n            }\n            \n            annotion_id += 1\n            annotations_json[\"annotations\"].append(image_annotations)\n        \n        \n    print(f\"Dataset COTS annotation to COCO json format completed! Files: {len(df)}\")\n    return annotations_json","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:51:41.473900Z","iopub.execute_input":"2022-01-17T08:51:41.474410Z","iopub.status.idle":"2022-01-17T08:51:41.490639Z","shell.execute_reply.started":"2022-01-17T08:51:41.474366Z","shell.execute_reply":"2022-01-17T08:51:41.489754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Convert COTS dataset to JSON COCO\ntrain_annot_json = dataset2coco(df_train[df_train.video_id != SELECTED_VIDEO], f\"{HOME_DIR}{DATASET_PATH}/train2017/\")\nval_annot_json = dataset2coco(df_train[df_train.video_id == SELECTED_VIDEO], f\"{HOME_DIR}{DATASET_PATH}/val2017/\")\n\n# Save converted annotations\nsave_annot_json(train_annot_json, f\"{HOME_DIR}{DATASET_PATH}/annotations/train.json\")\nsave_annot_json(val_annot_json, f\"{HOME_DIR}{DATASET_PATH}/annotations/valid.json\")","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:51:42.852805Z","iopub.execute_input":"2022-01-17T08:51:42.853079Z","iopub.status.idle":"2022-01-17T08:51:42.971384Z","shell.execute_reply.started":"2022-01-17T08:51:42.853050Z","shell.execute_reply":"2022-01-17T08:51:42.970623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# データの分布を確認(Valは重要)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport random\nimport numpy as np\nhist_list = []\nfor k in [SELECTED_VIDEO]:\n    train_area_list = []\n    val_area_list = []\n    #print(f'fold {k}')\n    for i in tqdm(range(len(df_train))):\n        bboxes = df_train.iloc[i].bboxes\n        for bbox in bboxes:\n            area = np.sqrt(bbox[2]*bbox[3])\n            if df_train.iloc[i].video_id == k:\n                val_area_list.append(area)\n            else:\n                train_area_list.append(area)\n    #sample_train = random.sample(train_area_list, len(val_area_list))\n    #train = np.array(sample_train)\n    #val = np.array(val_area_list)\n\n    \n    #plt.hist(sample_train,  range=(0, 5000),bins=100);\n    #plt.hist(val_area_list ,  range=(0, 5000),bins=100);\n\n    print(f'train: mean {np.mean(train_area_list)} std {np.std(train_area_list)}')\n    print(f'val: mean {np.mean(val_area_list)} std {np.std(val_area_list)}')\n    print(np.mean(train_area_list) - np.mean(val_area_list) ,  np.std(train_area_list) - np.std(val_area_list))\n    # ヒストグラムを描画する\n    plt.hist(train_area_list,  range=(0, 200),bins=100);\n    plt.hist(val_area_list ,  range=(0, 200),bins=100);","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:54:17.545325Z","iopub.execute_input":"2022-01-17T08:54:17.545884Z","iopub.status.idle":"2022-01-17T08:54:20.993465Z","shell.execute_reply.started":"2022-01-17T08:54:17.545844Z","shell.execute_reply":"2022-01-17T08:54:20.992766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# YOLOXの学習設定","metadata":{}},{"cell_type":"code","source":"def make_config(fold):\n    config_file_template = f'''\n\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n  def __init__(self):\n    super(Exp, self).__init__()\n    self.depth = 1\n    self.width = 1\n    self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n    self.output_dir = \"/kaggle/working\"\n\n    # Define yourself dataset path\n    self.data_dir = \"/kaggle/working/kaggle_dataset/images\"\n    self.train_ann = \"train.json\"\n    self.val_ann = \"valid.json\"\n    self.basic_lr_per_img = 1e-2 / 64.0 \n    self.num_classes = 1\n\n    self.max_epoch = $max_epoch\n    self.data_num_workers = 2\n    self.eval_interval = 1\n\n    self.mosaic_prob = 0.5\n    self.mixup_prob = 0.5\n    self.hsv_prob = 1.0\n    self.flip_prob = 0.5\n    self.no_aug_epochs = 2\n\n    self.input_size = (1280, 1280)\n    self.mosaic_scale = (1.0, 1.5)\n    self.mixup_scale = (1.0 , 1.5)\n    self.test_size = (1280, 1280)\n'''\n    return config_file_template","metadata":{"execution":{"iopub.status.busy":"2022-01-17T09:00:18.844089Z","iopub.execute_input":"2022-01-17T09:00:18.844663Z","iopub.status.idle":"2022-01-17T09:00:18.849847Z","shell.execute_reply.started":"2022-01-17T09:00:18.844624Z","shell.execute_reply":"2022-01-17T09:00:18.848992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 学習済みモデルのダウンロード","metadata":{}},{"cell_type":"code","source":"!wget https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_l.pth","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:54:37.664863Z","iopub.execute_input":"2022-01-17T08:54:37.665625Z","iopub.status.idle":"2022-01-17T08:55:01.671711Z","shell.execute_reply.started":"2022-01-17T08:54:37.665586Z","shell.execute_reply":"2022-01-17T08:55:01.670967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CV出力用のパッチコードを移植","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working/YOLOX\n!cp ./tools/train.py ./\n!cp  -r /kaggle/input/yolox-cots-cv-patch/yolox /kaggle/working/YOLOX/","metadata":{"execution":{"iopub.status.busy":"2022-01-17T08:55:43.963852Z","iopub.execute_input":"2022-01-17T08:55:43.964609Z","iopub.status.idle":"2022-01-17T08:55:45.300670Z","shell.execute_reply.started":"2022-01-17T08:55:43.964566Z","shell.execute_reply":"2022-01-17T08:55:45.299735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 学習","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working/YOLOX\n\n#!rm -r /content/kaggle_dataset_{SELECTED_FOLD}/images/img_resized_cache_train2017.array\nconfig_file_template = make_config(SELECTED_VIDEO)\nPIPELINE_CONFIG_PATH=f'./cots_config_video_id_{SELECTED_VIDEO}_epoch_20_yolox_l_size_1280_data_mizumashi_0114.py'\n\npipeline = Template(config_file_template).substitute(max_epoch = 0)\n\nwith open(PIPELINE_CONFIG_PATH, 'w') as f:\n    f.write(pipeline)\n    # ./yolox/data/datasets/voc_cl asses.py\n\nvoc_cls = '''VOC_CLASSES = (\"starfish\",)'''\nwith open('/kaggle/working/YOLOX/yolox/data/datasets/voc_classes.py', 'w') as f:\n    f.write(voc_cls)\n\n# ./yolox/data/datasets/coco_classes.py\n\ncoco_cls = '''COCO_CLASSES = (\"starfish\",)'''\n\nwith open('/kaggle/working/YOLOX/yolox/data/datasets/coco_classes.py', 'w') as f:\n    f.write(coco_cls)\n\n# check if everything is ok    \n!more /kaggle/working/YOLOX/yolox/data/datasets/coco_classes.py\n!python train.py \\\n  -f {PIPELINE_CONFIG_PATH} \\\n  -d 1 \\\n  -b 4\\\n  --fp16 \\\n  --cache\\\n  -o \\\n  -c /kaggle/working/yolox_l.pth\\\n  ","metadata":{"execution":{"iopub.status.busy":"2022-01-17T09:08:19.900169Z","iopub.execute_input":"2022-01-17T09:08:19.900451Z","iopub.status.idle":"2022-01-17T09:08:40.193657Z","shell.execute_reply.started":"2022-01-17T09:08:19.900419Z","shell.execute_reply":"2022-01-17T09:08:40.192781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}