{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom shutil import copyfile\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-28T18:20:15.292101Z","iopub.execute_input":"2022-01-28T18:20:15.292813Z","iopub.status.idle":"2022-01-28T18:20:16.318727Z","shell.execute_reply.started":"2022-01-28T18:20:15.292713Z","shell.execute_reply":"2022-01-28T18:20:16.317814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\ntrain['has_anno'] = train.annotations != '[]'\ntrain = train.loc[train['has_anno'] == True]\ntrain.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-28T18:20:16.320996Z","iopub.execute_input":"2022-01-28T18:20:16.321478Z","iopub.status.idle":"2022-01-28T18:20:16.393574Z","shell.execute_reply.started":"2022-01-28T18:20:16.321438Z","shell.execute_reply":"2022-01-28T18:20:16.392637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True)\nfor fold, (train_idx, val_idx) in enumerate(skf.split(train, train[\"video_id\"])):\n    train.loc[val_idx, 'fold'] = fold","metadata":{"execution":{"iopub.status.busy":"2022-01-28T18:20:16.395345Z","iopub.execute_input":"2022-01-28T18:20:16.395996Z","iopub.status.idle":"2022-01-28T18:20:16.413357Z","shell.execute_reply.started":"2022-01-28T18:20:16.395951Z","shell.execute_reply":"2022-01-28T18:20:16.412459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HOME_DIR = '/kaggle/working'\n!mkdir -p ./yolov5_data/fold3/images/val\n!mkdir -p ./yolov5_data/fold3/images/train\n\n!mkdir -p ./yolov5_data/fold3/labels/val\n!mkdir -p ./yolov5_data/fold3/labels/train","metadata":{"execution":{"iopub.status.busy":"2022-01-28T18:20:16.416002Z","iopub.execute_input":"2022-01-28T18:20:16.416407Z","iopub.status.idle":"2022-01-28T18:20:19.661954Z","shell.execute_reply.started":"2022-01-28T18:20:16.416363Z","shell.execute_reply":"2022-01-28T18:20:19.660709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 3\n\nannos = []\nfor i, x in train.iterrows():\n    if x['fold'] == fold:\n        mode = 'val'\n    else:\n        mode = 'train'\n\n    copyfile(f'../input/tensorflow-great-barrier-reef/train_images/video_{x.video_id}/{x.video_frame}.jpg',\n                f'./yolov5_data/fold{fold}/images/{mode}/{x.image_id}.jpg')\n    r = ''\n    anno = eval(x.annotations)\n    for an in anno:\n        w = an['width'] \n        h = an['height']\n    \n        if (an['x'] + an['width'] > 1280):\n            w = 1280 - an['x'] \n        if (an['y'] + an['height'] > 720):\n            h = 720 - an['y'] \n        \n        r += '0 {} {} {} {}\\n'.format((an['x'] + int(np.round(w/2))) / 1280,\n                                        (an['y'] + int(np.round(h/2))) / 720,\n                                        w / 1280, h / 720)\n    with open(f'./yolov5_data/fold{fold}/labels/{mode}/{x.image_id}.txt', 'w') as fp:\n        fp.write(r)","metadata":{"execution":{"iopub.status.busy":"2022-01-28T18:20:19.664104Z","iopub.execute_input":"2022-01-28T18:20:19.664464Z","iopub.status.idle":"2022-01-28T18:21:13.151690Z","shell.execute_reply.started":"2022-01-28T18:20:19.664413Z","shell.execute_reply":"2022-01-28T18:21:13.150801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = '/kaggle/working/yolov5_data/fold3/images/train'\nval_path = '/kaggle/working/yolov5_data/fold3/images/val'\ncwd  = '/kaggle/working'\nimages_train = os.listdir(train_path)\nimages_val = os.listdir(val_path)\nwith open(os.path.join(cwd , 'train.txt'), 'w') as f:\n    for path in images_train:\n        f.write(train_path+'/'+path+'\\n')\n            \nwith open(os.path.join(cwd , 'val.txt'), 'w') as f:\n    for path in images_val:\n        f.write(val_path+'/'+path+'\\n')","metadata":{"execution":{"iopub.status.busy":"2022-01-28T18:21:13.155691Z","iopub.execute_input":"2022-01-28T18:21:13.155920Z","iopub.status.idle":"2022-01-28T18:21:13.171513Z","shell.execute_reply.started":"2022-01-28T18:21:13.155892Z","shell.execute_reply":"2022-01-28T18:21:13.170450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working\n!rm -r /kaggle/working/yolov5\n!git clone https://github.com/ultralytics/yolov5 # clone\n!cp -r /kaggle/input/yolov5-lib-ds /kaggle/working/yolov5\n%cd yolov5\n%pip install -qr requirements.txt  # install","metadata":{"execution":{"iopub.status.busy":"2022-01-28T18:21:13.173132Z","iopub.execute_input":"2022-01-28T18:21:13.173508Z","iopub.status.idle":"2022-01-28T18:21:32.327364Z","shell.execute_reply.started":"2022-01-28T18:21:13.173463Z","shell.execute_reply":"2022-01-28T18:21:32.326420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hyps = '''\nlr0: 0.01  # initial learning rate (SGD=1E-2, Adam=1E-3)\nlrf: 0.1  # final OneCycleLR learning rate (lr0 * lrf)\nmomentum: 0.937  # SGD momentum/Adam beta1\nweight_decay: 0.0005  # optimizer weight decay 5e-4\nwarmup_epochs: 3.0  # warmup epochs (fractions ok)\nwarmup_momentum: 0.8  # warmup initial momentum\nwarmup_bias_lr: 0.1  # warmup initial bias lr\nbox: 0.05  # box loss gain\ncls: 0.5  # cls loss gain\ncls_pw: 1.0  # cls BCELoss positive_weight\nobj: 1.0  # obj loss gain (scale with pixels)\nobj_pw: 1.0  # obj BCELoss positive_weight\niou_t: 0.20  # IoU training threshold\nanchor_t: 4.0  # anchor-multiple threshold\n# anchors: 3  # anchors per output layer (0 to ignore)\nfl_gamma: 0.0  # focal loss gamma (efficientDet default gamma=1.5)\nhsv_h: 0.015  # image HSV-Hue augmentation (fraction)\nhsv_s: 0.7  # image HSV-Saturation augmentation (fraction)\nhsv_v: 0.4  # image HSV-Value augmentation (fraction)\ndegrees: 0.0  # image rotation (+/- deg)\ntranslate: 0.1  # image translation (+/- fraction)\nscale: 0.5  # image scale (+/- gain)\nshear: 0.0  # image shear (+/- deg)\nperspective: 0.0  # image perspective (+/- fraction), range 0-0.001\nflipud: 0.5  # image flip up-down (probability)\nfliplr: 0.5  # image flip left-right (probability)\nmosaic: 1.0  # image mosaic (probability)\nmixup: 0.5  # image mixup (probability)\ncopy_paste: 0.0  # segment copy-paste (probability)\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-28T18:21:32.329573Z","iopub.execute_input":"2022-01-28T18:21:32.329862Z","iopub.status.idle":"2022-01-28T18:21:32.337478Z","shell.execute_reply.started":"2022-01-28T18:21:32.329829Z","shell.execute_reply":"2022-01-28T18:21:32.336245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = '''\ntrain: /kaggle/working/train.txt  # train images (relative to 'path')\nval: /kaggle/working/val.txt  # val images (relative to 'path')\ntest:  # test images (optional)\n\n# Classes\nnc: 1  # number of classes\nnames: ['starfish']  # class names\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-28T18:21:32.339585Z","iopub.execute_input":"2022-01-28T18:21:32.340001Z","iopub.status.idle":"2022-01-28T18:21:32.352511Z","shell.execute_reply.started":"2022-01-28T18:21:32.339947Z","shell.execute_reply":"2022-01-28T18:21:32.351468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(f'{HOME_DIR}/Yolov5-protect-reef.yaml', 'w') as fp:\n    fp.write(data)\nwith open(f'{HOME_DIR}/hyp-yolov5.yaml', 'w') as fp:\n    fp.write(hyps)","metadata":{"execution":{"iopub.status.busy":"2022-01-28T18:21:32.356916Z","iopub.execute_input":"2022-01-28T18:21:32.357179Z","iopub.status.idle":"2022-01-28T18:21:32.365930Z","shell.execute_reply.started":"2022-01-28T18:21:32.357148Z","shell.execute_reply":"2022-01-28T18:21:32.364826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wandb off","metadata":{"execution":{"iopub.status.busy":"2022-01-28T18:21:32.367860Z","iopub.execute_input":"2022-01-28T18:21:32.368304Z","iopub.status.idle":"2022-01-28T18:21:35.169882Z","shell.execute_reply.started":"2022-01-28T18:21:32.368258Z","shell.execute_reply":"2022-01-28T18:21:35.168725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python train.py --img 3500\\\n--batch 4\\\n--epochs 7\\\n--optimizer 'Adam'\\\n--data '{HOME_DIR}/Yolov5-protect-reef.yaml'\\\n--hyp '{HOME_DIR}/hyp-yolov5.yaml'\\\n--weights 'yolov5n6.pt'\\\n--project 'Protect_reef' --name 'yolov5n6'\\\n--exist-ok","metadata":{"execution":{"iopub.status.busy":"2022-01-28T18:21:35.171928Z","iopub.execute_input":"2022-01-28T18:21:35.172270Z","iopub.status.idle":"2022-01-28T18:23:44.157048Z","shell.execute_reply.started":"2022-01-28T18:21:35.172220Z","shell.execute_reply":"2022-01-28T18:23:44.155908Z"},"trusted":true},"execution_count":null,"outputs":[]}]}