{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Yolov5 high resolution training\n\n### Major modification\n* img=3600\n* mixup=0.5\n* fliplr: 0.5\n\n### Hardware to reproduce\n* RTX3090","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom shutil import copyfile\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-13T15:39:58.612891Z","iopub.execute_input":"2022-01-13T15:39:58.613408Z","iopub.status.idle":"2022-01-13T15:39:58.639423Z","shell.execute_reply.started":"2022-01-13T15:39:58.613315Z","shell.execute_reply":"2022-01-13T15:39:58.638739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\ntrain['pos'] = train.annotations != '[]'","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:39:59.357125Z","iopub.execute_input":"2022-01-13T15:39:59.357384Z","iopub.status.idle":"2022-01-13T15:39:59.420212Z","shell.execute_reply.started":"2022-01-13T15:39:59.357357Z","shell.execute_reply":"2022-01-13T15:39:59.419502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p ./yolo_data/fold1/images/val\n!mkdir -p ./yolo_data/fold1/images/train\n\n!mkdir -p ./yolo_data/fold1/labels/val\n!mkdir -p ./yolo_data/fold1/labels/train","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:39:59.929918Z","iopub.execute_input":"2022-01-13T15:39:59.930429Z","iopub.status.idle":"2022-01-13T15:40:02.550509Z","shell.execute_reply.started":"2022-01-13T15:39:59.930389Z","shell.execute_reply":"2022-01-13T15:40:02.549581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 1\n\nannos = []\nfor i, x in train.iterrows():\n    if x.video_id == fold:\n        mode = 'val'\n    else:\n        # train\n        mode = 'train'\n        if not x.pos: continue\n        # val\n    copyfile(f'../input/tensorflow-great-barrier-reef/train_images/video_{x.video_id}/{x.video_frame}.jpg',\n                f'./yolo_data/fold{fold}/images/{mode}/{x.image_id}.jpg')\n    if not x.pos:\n        continue\n    r = ''\n    anno = eval(x.annotations)\n    for an in anno:\n#            annos.append(an)\n        r += '0 {} {} {} {}\\n'.format((an['x'] + an['width'] / 2) / 1280,\n                                        (an['y'] + an['height'] / 2) / 720,\n                                        an['width'] / 1280, an['height'] / 720)\n    with open(f'./yolo_data/fold{fold}/labels/{mode}/{x.image_id}.txt', 'w') as fp:\n        fp.write(r)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:40:02.552544Z","iopub.execute_input":"2022-01-13T15:40:02.552793Z","iopub.status.idle":"2022-01-13T15:42:13.650088Z","shell.execute_reply.started":"2022-01-13T15:40:02.552751Z","shell.execute_reply":"2022-01-13T15:42:13.649343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hyps = '''\n# YOLOv5 by Ultralytics, GPL-3.0 license\n# Hyperparameters for COCO training from scratch\n# python train.py --batch 40 --cfg yolov5m.yaml --weights '' --data coco.yaml --img 640 --epochs 300\n# See tutorials for hyperparameter evolution https://github.com/ultralytics/yolov5#tutorials\n\nlr0: 0.01  # initial learning rate (SGD=1E-2, Adam=1E-3)\nlrf: 0.1  # final OneCycleLR learning rate (lr0 * lrf)\nmomentum: 0.937  # SGD momentum/Adam beta1\nweight_decay: 0.0005  # optimizer weight decay 5e-4\nwarmup_epochs: 3.0  # warmup epochs (fractions ok)\nwarmup_momentum: 0.8  # warmup initial momentum\nwarmup_bias_lr: 0.1  # warmup initial bias lr\nbox: 0.05  # box loss gain\ncls: 0.5  # cls loss gain\ncls_pw: 1.0  # cls BCELoss positive_weight\nobj: 1.0  # obj loss gain (scale with pixels)\nobj_pw: 1.0  # obj BCELoss positive_weight\niou_t: 0.20  # IoU training threshold\nanchor_t: 4.0  # anchor-multiple threshold\n# anchors: 3  # anchors per output layer (0 to ignore)\nfl_gamma: 0.0  # focal loss gamma (efficientDet default gamma=1.5)\nhsv_h: 0.015  # image HSV-Hue augmentation (fraction)\nhsv_s: 0.7  # image HSV-Saturation augmentation (fraction)\nhsv_v: 0.4  # image HSV-Value augmentation (fraction)\ndegrees: 0.0  # image rotation (+/- deg)\ntranslate: 0.1  # image translation (+/- fraction)\nscale: 0.5  # image scale (+/- gain)\nshear: 0.0  # image shear (+/- deg)\nperspective: 0.0  # image perspective (+/- fraction), range 0-0.001\nflipud: 0.5  # image flip up-down (probability)\nfliplr: 0.5  # image flip left-right (probability)\nmosaic: 1.0  # image mosaic (probability)\nmixup: 0.5  # image mixup (probability)\ncopy_paste: 0.0  # segment copy-paste (probability)\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:43:48.970887Z","iopub.execute_input":"2022-01-13T15:43:48.971393Z","iopub.status.idle":"2022-01-13T15:43:48.976183Z","shell.execute_reply.started":"2022-01-13T15:43:48.971357Z","shell.execute_reply":"2022-01-13T15:43:48.975484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = '''\n# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..]\npath: ../yolo_data/fold1/  # dataset root dir\ntrain: images/train  # train images (relative to 'path') 128 images\nval: images/val  # val images (relative to 'path') 128 images\ntest:  # test images (optional)\n\n# Classes\nnc: 1  # number of classes\nnames: ['reef']  # class names\n\n\n# Download script/URL (optional)\n# download: https://ultralytics.com/assets/coco128.zip\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:46:59.716132Z","iopub.execute_input":"2022-01-13T15:46:59.716419Z","iopub.status.idle":"2022-01-13T15:46:59.721499Z","shell.execute_reply.started":"2022-01-13T15:46:59.716378Z","shell.execute_reply":"2022-01-13T15:46:59.720390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/ultralytics/yolov5.git","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:43:50.421543Z","iopub.execute_input":"2022-01-13T15:43:50.421810Z","iopub.status.idle":"2022-01-13T15:43:54.770972Z","shell.execute_reply.started":"2022-01-13T15:43:50.421761Z","shell.execute_reply":"2022-01-13T15:43:54.770116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('./yolov5/data/reef_f1_naive.yaml', 'w') as fp:\n    fp.write(data)\nwith open('./yolov5/data/hyps/hyp.heavy.2.yaml', 'w') as fp:\n    fp.write(hyps)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:47:12.544233Z","iopub.execute_input":"2022-01-13T15:47:12.544490Z","iopub.status.idle":"2022-01-13T15:47:12.550087Z","shell.execute_reply.started":"2022-01-13T15:47:12.544460Z","shell.execute_reply":"2022-01-13T15:47:12.549254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd yolov5","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:47:13.894247Z","iopub.execute_input":"2022-01-13T15:47:13.894948Z","iopub.status.idle":"2022-01-13T15:47:13.900750Z","shell.execute_reply.started":"2022-01-13T15:47:13.894900Z","shell.execute_reply":"2022-01-13T15:47:13.900065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls data/","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:46:27.056616Z","iopub.execute_input":"2022-01-13T15:46:27.057182Z","iopub.status.idle":"2022-01-13T15:46:27.714560Z","shell.execute_reply.started":"2022-01-13T15:46:27.057138Z","shell.execute_reply":"2022-01-13T15:46:27.713662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python train.py --img 3000 --batch 4 --epochs 5 --data reef_f1_naive.yaml --weights yolov5s6.pt --name l6_3600_uflip_vm5_f1 --hyp data/hyps/hyp.heavy.2.yaml","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:47:16.951390Z","iopub.execute_input":"2022-01-13T15:47:16.952339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:38:38.000795Z","iopub.execute_input":"2022-01-13T15:38:38.001119Z","iopub.status.idle":"2022-01-13T15:38:38.274427Z","shell.execute_reply.started":"2022-01-13T15:38:38.001095Z","shell.execute_reply":"2022-01-13T15:38:38.273423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}