{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Train Only\n\n[Prepare Data](https://www.kaggle.com/ihorin/great-barrier-reef-prepare-data)","metadata":{}},{"cell_type":"markdown","source":"# 🛠 Install Libraries","metadata":{"execution":{"iopub.status.busy":"2022-02-05T07:37:31.618952Z","iopub.execute_input":"2022-02-05T07:37:31.619509Z","iopub.status.idle":"2022-02-05T07:37:31.635679Z","shell.execute_reply.started":"2022-02-05T07:37:31.619362Z","shell.execute_reply":"2022-02-05T07:37:31.635078Z"}}},{"cell_type":"code","source":"!pip install -qU wandb\n!pip install -qU bbox-utility # check https://github.com/awsaf49/bbox for source code","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:43.628466Z","iopub.execute_input":"2022-02-12T17:13:43.628979Z","iopub.status.idle":"2022-02-12T17:13:45.725268Z","shell.execute_reply.started":"2022-02-12T17:13:43.628929Z","shell.execute_reply":"2022-02-12T17:13:45.722023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📚 Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nimport glob\n\nimport shutil\nimport sys\nsys.path.append('../input/tensorflow-great-barrier-reef')\n\nfrom joblib import Parallel, delayed\n\nfrom IPython.display import display, HTML\n\nfrom matplotlib import animation, rc\nrc('animation', html='jshtml')\n\n# for DA\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.utils.data as Data\nimport ast #?\nfrom fastprogress.fastprogress import master_bar, progress_bar #?","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-12T17:13:45.728137Z","iopub.status.idle":"2022-02-12T17:13:45.730151Z","shell.execute_reply.started":"2022-02-12T17:13:45.729899Z","shell.execute_reply":"2022-02-12T17:13:45.729928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📌 Key-Points\n* 提供されているpython時系列APIを使用して予測を送信する必要があります。これにより、このコンテストは以前のオブジェクト検出コンテストとは異なります。\n* 各予測行には、画像のすべての境界ボックスを含める必要があります。提出はフォーマットもCOCOのようです。これは`[x_min、y_min、幅、高さ]`を意味します\n* CopmetitionメトリックF2は、ヒトデを見逃すことがほとんどないことを保証するために、いくつかの誤検知（FP）を許容します。つまり、誤検知（FN）は、誤検知（FP）よりも重要です。\n$$F2 = 5 \\cdot \\frac{precision \\cdot recall}{4\\cdot precision + recall}$$","metadata":{}},{"cell_type":"markdown","source":"# ⭐ WandB","metadata":{}},{"cell_type":"code","source":"import wandb\n\ntry:\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    api_key = user_secrets.get_secret(\"wandb_team_iforine\")\n    wandb.login(key=api_key)\n    anonymous = None\nexcept:\n    wandb.login(anonymous='must')\n    print('To use your W&B account,\\nGo to Add-ons -> Secrets and provide your W&B access token. Use the Label name as WANDB. \\nGet your W&B access token from here: https://wandb.ai/authorize')","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:45.740404Z","iopub.status.idle":"2022-02-12T17:13:45.742429Z","shell.execute_reply.started":"2022-02-12T17:13:45.742186Z","shell.execute_reply":"2022-02-12T17:13:45.742213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLD      = 2 # which fold to train\nDIM       = 2016\nMODEL     = 'yolov5s'\nBATCH     = 8\nEPOCHS    = 10\nOPTIM     = 'Adam'\nAUG       = '3_HFlip_HE_CHE_Gamma'\n\nPROJECT   = 'iforine/great-barrier-reef-public' # w&b in yolov5\nNAME      = f'{MODEL}-dim{DIM}-fold{FOLD}-bat{BATCH}-opt{OPTIM}-aug{AUG}-epch{EPOCHS}' # w&b for yolov5\n\nROOT_DIR  = '/kaggle/input/tensorflow-great-barrier-reef/'\nDATA_DIR  = '/kaggle/input/video-great-barrier-reef-prepare-data'\nIMAGE_DIR = '/kaggle/working/images' # directory to save images\nLABEL_DIR = '/kaggle/working/labels' # directory to save labels\n\nWORKER = 4 # よくわかってない。スレッドの数とか？\n\nnp.random.seed(42)","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:45.745150Z","iopub.status.idle":"2022-02-12T17:13:45.745868Z","shell.execute_reply.started":"2022-02-12T17:13:45.745603Z","shell.execute_reply":"2022-02-12T17:13:45.745632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Directories","metadata":{}},{"cell_type":"code","source":"!mkdir -p {IMAGE_DIR}\n!mkdir -p {LABEL_DIR}","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:45.750755Z","iopub.status.idle":"2022-02-12T17:13:45.751361Z","shell.execute_reply.started":"2022-02-12T17:13:45.751139Z","shell.execute_reply":"2022-02-12T17:13:45.751162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ✏️InputからWorkingへデータをコピー\n\n* We need to copy the Images to Current Directory(`/kaggle/working`) as `/kaggle/input` doesn't have **write access** which is needed for **YOLOv5**.\n* We can make this process faster using **Joblib** which uses **Parallel** computing.\n\n* / kaggle / inputにはYOLOv5に必要な書き込みアクセス権がないため、イメージを現在のディレクトリ（/ kaggle / working）にコピーする必要があります。\n* この処理を高速化するには、**並列**計算を利用する**Joblib**を使用します。","metadata":{}},{"cell_type":"markdown","source":"shutil.copyfile(src, dst, *, follow_symlinks=True)\n\nsrc という名前のファイルの内容 (メタデータを含まない) を dst という名前のファイルにコピーし、最も効率的な方法で dst を返します。 src と dst は path-like object または文字列でパス名を指定します。","metadata":{}},{"cell_type":"code","source":"def make_copy(row):\n    shutil.copyfile(row.old_path, row.new_path)\n    return","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:45.752510Z","iopub.status.idle":"2022-02-12T17:13:45.753121Z","shell.execute_reply.started":"2022-02-12T17:13:45.752890Z","shell.execute_reply":"2022-02-12T17:13:45.752913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"並列処理\n\n```\njoblib.Parallel(<Parallelへの引数>)(\n    joblib.delayed(<実行する関数>)(<関数への引数>) for 変数名 in イテラブル\n)\n```","metadata":{}},{"cell_type":"markdown","source":"iterrows()メソッドを使うと、1行ずつ、インデックス名（行名）とその行のデータ（pandas.Series型）のタプル(index, Series)を取得できる。","metadata":{}},{"cell_type":"code","source":"paths = pd.read_csv(f'{DATA_DIR}/copy_path.csv')\n_ = Parallel(n_jobs=-1, backend='threading')(delayed(make_copy)(row) for _, row in tqdm(paths.iterrows(), total=len(paths)))","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:45.754274Z","iopub.status.idle":"2022-02-12T17:13:45.754872Z","shell.execute_reply.started":"2022-02-12T17:13:45.754648Z","shell.execute_reply":"2022-02-12T17:13:45.754672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.txt, val.txtのコピー\nshutil.copyfile(f'{DATA_DIR}/train.txt', \n                '/kaggle/working/train.txt')\nshutil.copyfile(f'{DATA_DIR}/valid.txt', \n                '/kaggle/working/val.txt')","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:45.765444Z","iopub.status.idle":"2022-02-12T17:13:45.766067Z","shell.execute_reply.started":"2022-02-12T17:13:45.765830Z","shell.execute_reply":"2022-02-12T17:13:45.765854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# YAML Setting","metadata":{}},{"cell_type":"code","source":"import yaml\n\ncwd = '/kaggle/working/'\n\ndata = dict(\n    path  = '/kaggle/working',\n    train =  os.path.join( cwd , 'train.txt') ,\n    val   =  os.path.join( cwd , 'val.txt' ),\n    nc    = 1,\n    names = ['cots'],\n    )\n\nwith open(os.path.join( cwd , 'gbr.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(os.path.join( cwd , 'gbr.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:45.767199Z","iopub.status.idle":"2022-02-12T17:13:45.767813Z","shell.execute_reply.started":"2022-02-12T17:13:45.767587Z","shell.execute_reply":"2022-02-12T17:13:45.767612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile /kaggle/working/hyp.yaml\nlr0: 0.01  # initial learning rate (SGD=1E-2, Adam=1E-3)\nlrf: 0.1  # final OneCycleLR learning rate (lr0 * lrf)\nmomentum: 0.937  # SGD momentum/Adam beta1\nweight_decay: 0.0005  # optimizer weight decay 5e-4\nwarmup_epochs: 3.0  # warmup epochs (fractions ok)\nwarmup_momentum: 0.8  # warmup initial momentum\nwarmup_bias_lr: 0.1  # warmup initial bias lr\nbox: 0.05  # box loss gain\ncls: 0.5  # cls loss gain\ncls_pw: 1.0  # cls BCELoss positive_weight\nobj: 1.0  # obj loss gain (scale with pixels)\nobj_pw: 1.0  # obj BCELoss positive_weight\niou_t: 0.20  # IoU training threshold\nanchor_t: 4.0  # anchor-multiple threshold\n# anchors: 3  # anchors per output layer (0 to ignore)\nfl_gamma: 0.0  # focal loss gamma (efficientDet default gamma=1.5)\nhsv_h: 0.015  # image HSV-Hue augmentation (fraction)\nhsv_s: 0.7  # image HSV-Saturation augmentation (fraction)\nhsv_v: 0.4  # image HSV-Value augmentation (fraction)\ndegrees: 0.0  # image rotation (+/- deg)\ntranslate: 0.10  # image translation (+/- fraction)\nscale: 0.5  # image scale (+/- gain)\nshear: 0.0  # image shear (+/- deg)\nperspective: 0.0  # image perspective (+/- fraction), range 0-0.001\nflipud: 0.5  # image flip up-down (probability)\nfliplr: 0.5  # image flip left-right (probability)\nmosaic: 0.5  # image mosaic (probability)\nmixup: 0.5 # image mixup (probability)\ncopy_paste: 0.0  # segment copy-paste (probability)","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:45.769005Z","iopub.status.idle":"2022-02-12T17:13:45.769649Z","shell.execute_reply.started":"2022-02-12T17:13:45.769398Z","shell.execute_reply":"2022-02-12T17:13:45.769421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📦 [YOLOv5](https://github.com/ultralytics/yolov5/)","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working\n!rm -r /kaggle/working/yolov5\n# !git clone https://github.com/ultralytics/yolov5 # clone\n!cp -r /kaggle/input/yolov5-lib-ds /kaggle/working/yolov5\n%cd yolov5\n%pip install -qr requirements.txt  # install\n\nfrom yolov5 import utils\ndisplay = utils.notebook_init()  # check","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:45.770791Z","iopub.status.idle":"2022-02-12T17:13:45.775822Z","shell.execute_reply.started":"2022-02-12T17:13:45.775585Z","shell.execute_reply":"2022-02-12T17:13:45.775611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🚅 Training","metadata":{}},{"cell_type":"code","source":"!python train.py --img {DIM}\\\n--batch {BATCH}\\\n--epochs {EPOCHS}\\\n--data /kaggle/working/gbr.yaml\\\n--hyp /kaggle/working/hyp.yaml\\\n--weights {MODEL}.pt\\\n--optimizer {OPTIM}\\\n--project {PROJECT} --name {NAME}\\\n--exist-ok","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:45.781066Z","iopub.status.idle":"2022-02-12T17:13:45.781681Z","shell.execute_reply.started":"2022-02-12T17:13:45.781435Z","shell.execute_reply":"2022-02-12T17:13:45.781458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ✂️ Remove Files","metadata":{}},{"cell_type":"code","source":"!rm -r {IMAGE_DIR}\n!rm -r {LABEL_DIR}","metadata":{"execution":{"iopub.status.busy":"2022-02-12T17:13:45.782823Z","iopub.status.idle":"2022-02-12T17:13:45.783435Z","shell.execute_reply.started":"2022-02-12T17:13:45.783215Z","shell.execute_reply":"2022-02-12T17:13:45.783239Z"},"trusted":true},"execution_count":null,"outputs":[]}]}