{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvidia-smi","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-19T17:02:39.371795Z","iopub.execute_input":"2021-07-19T17:02:39.37215Z","iopub.status.idle":"2021-07-19T17:02:40.048028Z","shell.execute_reply.started":"2021-07-19T17:02:39.37207Z","shell.execute_reply":"2021-07-19T17:02:40.047166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nos.makedirs('/tmp/yolov5',exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:02:40.049935Z","iopub.execute_input":"2021-07-19T17:02:40.050213Z","iopub.status.idle":"2021-07-19T17:02:40.074421Z","shell.execute_reply.started":"2021-07-19T17:02:40.050186Z","shell.execute_reply":"2021-07-19T17:02:40.073775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/kumar-shubham-ml/yolov5\n!cp -r yolov5/. /tmp/yolov5/.\n!rm -r yolov5","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:02:40.076995Z","iopub.execute_input":"2021-07-19T17:02:40.077239Z","iopub.status.idle":"2021-07-19T17:02:43.890101Z","shell.execute_reply.started":"2021-07-19T17:02:40.077215Z","shell.execute_reply":"2021-07-19T17:02:43.889101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls -lh /tmp/yolov5/","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:02:43.893734Z","iopub.execute_input":"2021-07-19T17:02:43.894005Z","iopub.status.idle":"2021-07-19T17:02:44.575158Z","shell.execute_reply.started":"2021-07-19T17:02:43.893979Z","shell.execute_reply":"2021-07-19T17:02:44.574257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"FOLD = 1\nN_CLASSES = 1\nIMG_SIZE = 512\nBATCH_SIZE = 16\nN_EPOCHS = 20\nMODEL_NAME = 'yolov5x'\nCHECKPOINTS_ROOT = f\"{MODEL_NAME}_fold{FOLD}_img{IMG_SIZE}\"\nEVALUATE = True\nRELEASE = None #One of 1-5 (Refer: https://github.com/ultralytics/yolov5/tags)  #### If None, picks the latest version\nprint(CHECKPOINTS_ROOT)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:02:44.579275Z","iopub.execute_input":"2021-07-19T17:02:44.579545Z","iopub.status.idle":"2021-07-19T17:02:44.587431Z","shell.execute_reply.started":"2021-07-19T17:02:44.579518Z","shell.execute_reply":"2021-07-19T17:02:44.586476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Config","metadata":{}},{"cell_type":"code","source":"%%writefile data.yaml\n\n\ntrain: /tmp/images/train\nval: /tmp/images/val\n\n# number of classes\nnc: 1\n\n# class names\nnames: ['0']","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:02:44.589044Z","iopub.execute_input":"2021-07-19T17:02:44.589404Z","iopub.status.idle":"2021-07-19T17:02:44.598151Z","shell.execute_reply.started":"2021-07-19T17:02:44.589369Z","shell.execute_reply":"2021-07-19T17:02:44.597173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameters\n\n","metadata":{}},{"cell_type":"code","source":"%%writefile hyperparameters.yaml\n\n# Hyperparameters for training from scratch\n\nlr0: 0.01  # initial learning rate (SGD=1E-2, Adam=1E-3)\nlrf: 0.2  # final OneCycleLR learning rate (lr0 * lrf)\nmomentum: 0.937  # SGD momentum/Adam beta1\nweight_decay: 0.0005  # optimizer weight decay 5e-4\nwarmup_epochs: 3.0  # warmup epochs (fractions ok)\nwarmup_momentum: 0.8  # warmup initial momentum\nwarmup_bias_lr: 0.1  # warmup initial bias lr\nbox: 0.05  # box loss gain\ncls: 0.5  # cls loss gain\ncls_pw: 1.0  # cls BCELoss positive_weight\nobj: 1.0  # obj loss gain (scale with pixels)\nobj_pw: 3.0  # obj BCELoss positive_weight\niou_t: 0.20  # IoU training threshold\nanchor_t: 4.0  # anchor-multiple threshold\n# anchors: 3  # anchors per output layer (0 to ignore)\nfl_gamma: 0.0  # focal loss gamma (efficientDet default gamma=1.5)\nhsv_h: 0.015  # image HSV-Hue augmentation (fraction)\nhsv_s: 0.7  # image HSV-Saturation augmentation (fraction)\nhsv_v: 0.4  # image HSV-Value augmentation (fraction)\ndegrees: 10.0  # image rotation (+/- deg)\ntranslate: 0.1  # image translation (+/- fraction)\nscale: 0.5  # image scale (+/- gain)\nshear: 0.0  # image shear (+/- deg)\nperspective: 0.0  # image perspective (+/- fraction), range 0-0.001\nflipud: 0.0  # image flip up-down (probability)\nfliplr: 0.5  # image flip left-right (probability)\nmosaic: 1.0  # image mosaic (probability)\nmixup: 0.0  # image mixup (probability)\ncopy_paste: 0.0","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:02:44.600292Z","iopub.execute_input":"2021-07-19T17:02:44.600706Z","iopub.status.idle":"2021-07-19T17:02:44.609002Z","shell.execute_reply.started":"2021-07-19T17:02:44.600671Z","shell.execute_reply":"2021-07-19T17:02:44.607779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Config","metadata":{}},{"cell_type":"code","source":"# Read in the model config\ntry:\n    with open(f'/tmp/yolov5/models/{MODEL_NAME}.yaml',) as file :\n        filedata = file.read()\nexcept:\n    with open(f'/tmp/yolov5/models/hub/{MODEL_NAME}.yaml',) as file :\n        filedata = file.read()\n\n# Replace the target string\nfiledata = filedata.replace('nc: 80', f'nc: {N_CLASSES}' )\n\n# Write the file out again\nwith open(f'{MODEL_NAME}.yaml','w') as file:\n    file.write(filedata)\n\nprint(filedata)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:02:47.138524Z","iopub.execute_input":"2021-07-19T17:02:47.138886Z","iopub.status.idle":"2021-07-19T17:02:47.147422Z","shell.execute_reply.started":"2021-07-19T17:02:47.138856Z","shell.execute_reply":"2021-07-19T17:02:47.146251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Resolve Requirements Conflicts","metadata":{}},{"cell_type":"code","source":"!pip install -r /tmp/yolov5/requirements.txt >/dev/null 2>&1","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:02:51.011864Z","iopub.execute_input":"2021-07-19T17:02:51.012179Z","iopub.status.idle":"2021-07-19T17:02:58.817569Z","shell.execute_reply.started":"2021-07-19T17:02:51.01215Z","shell.execute_reply":"2021-07-19T17:02:58.816443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Data","metadata":{}},{"cell_type":"code","source":"mask_root = '../input/chest-xray-masks-and-labels/Lung Segmentation/masks/'\nimg_root = '../input/chest-xray-masks-and-labels/Lung Segmentation/CXR_png/'\nall_imgs = os.listdir(img_root)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:07:43.462144Z","iopub.execute_input":"2021-07-19T17:07:43.462506Z","iopub.status.idle":"2021-07-19T17:07:43.469159Z","shell.execute_reply.started":"2021-07-19T17:07:43.462473Z","shell.execute_reply":"2021-07-19T17:07:43.468373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/tmp/images/train/',exist_ok=True)\nos.makedirs('/tmp/images/val/',exist_ok=True)\nos.makedirs('/tmp/labels/train/',exist_ok=True)\nos.makedirs('/tmp/labels/val/',exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:03:43.189321Z","iopub.execute_input":"2021-07-19T17:03:43.189696Z","iopub.status.idle":"2021-07-19T17:03:43.195411Z","shell.execute_reply.started":"2021-07-19T17:03:43.189663Z","shell.execute_reply":"2021-07-19T17:03:43.194403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(open('../input/siim-lung-boxes-1/CHNCXR_0001_0.txt').read())","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:04:07.001137Z","iopub.execute_input":"2021-07-19T17:04:07.001471Z","iopub.status.idle":"2021-07-19T17:04:07.010813Z","shell.execute_reply.started":"2021-07-19T17:04:07.00144Z","shell.execute_reply":"2021-07-19T17:04:07.00961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_file = all_imgs[0]","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:13:24.203769Z","iopub.execute_input":"2021-07-19T17:13:24.204132Z","iopub.status.idle":"2021-07-19T17:13:24.209395Z","shell.execute_reply.started":"2021-07-19T17:13:24.204098Z","shell.execute_reply":"2021-07-19T17:13:24.208367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom tqdm.auto import tqdm\nimport joblib\n# TRAIN_ROOT_IMAGES = '../input/siim-covid-images-jpegs/images/'\nimg_root = '/kaggle/input/chest-xray-masks-and-labels/Lung Segmentation/CXR_png/'\nmask_root = '/kaggle/input/siim-lung-boxes-1/'\ndef arrange_images_and_labels(i,image_file):\n    text_file = image_file.replace('.png','.txt')\n    if i%5==FOLD:\n        os.system(f\"cp '{img_root}{image_file}' /tmp/images/val/{image_file}\")\n        os.system(f\"cp '{mask_root}{text_file}' /tmp/labels/val/{text_file}\")\n    else:\n        os.system(f\"cp '{img_root}{image_file}' /tmp/images/train/{image_file}\")\n        os.system(f\"cp '{mask_root}{text_file}' /tmp/labels/train/{text_file}\")\n_ = joblib.Parallel(n_jobs=16)(\n    joblib.delayed(arrange_images_and_labels)(i,image_file) for i,image_file in tqdm(enumerate(all_imgs))\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:16:07.3253Z","iopub.execute_input":"2021-07-19T17:16:07.325663Z","iopub.status.idle":"2021-07-19T17:16:35.0847Z","shell.execute_reply.started":"2021-07-19T17:16:07.325628Z","shell.execute_reply":"2021-07-19T17:16:35.083711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /tmp/images/train/ | wc -l","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:16:35.088339Z","iopub.execute_input":"2021-07-19T17:16:35.088595Z","iopub.status.idle":"2021-07-19T17:16:36.031751Z","shell.execute_reply.started":"2021-07-19T17:16:35.088569Z","shell.execute_reply":"2021-07-19T17:16:36.030654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /tmp/labels/train/ | wc -l","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:16:38.941871Z","iopub.execute_input":"2021-07-19T17:16:38.942281Z","iopub.status.idle":"2021-07-19T17:16:39.626027Z","shell.execute_reply.started":"2021-07-19T17:16:38.942234Z","shell.execute_reply":"2021-07-19T17:16:39.624945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /tmp/images/val/ | wc -l","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:16:42.220522Z","iopub.execute_input":"2021-07-19T17:16:42.220877Z","iopub.status.idle":"2021-07-19T17:17:04.606234Z","shell.execute_reply.started":"2021-07-19T17:16:42.220847Z","shell.execute_reply":"2021-07-19T17:17:04.605121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /tmp/labels/val/ | wc -l","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:17:04.608179Z","iopub.execute_input":"2021-07-19T17:17:04.60849Z","iopub.status.idle":"2021-07-19T17:17:05.250381Z","shell.execute_reply.started":"2021-07-19T17:17:04.608463Z","shell.execute_reply":"2021-07-19T17:17:05.24945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"import time\nstart = time.time()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:17:05.251939Z","iopub.execute_input":"2021-07-19T17:17:05.252279Z","iopub.status.idle":"2021-07-19T17:17:05.258498Z","shell.execute_reply.started":"2021-07-19T17:17:05.252234Z","shell.execute_reply":"2021-07-19T17:17:05.257548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ## Download Model\nif RELEASE:\n    !wget https://github.com/ultralytics/yolov5/releases/download/v{RELEASE}.0/{MODEL_NAME}.pt","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:17:05.26053Z","iopub.execute_input":"2021-07-19T17:17:05.260962Z","iopub.status.idle":"2021-07-19T17:17:05.266887Z","shell.execute_reply.started":"2021-07-19T17:17:05.260892Z","shell.execute_reply":"2021-07-19T17:17:05.265926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!WANDB_MODE=\"dryrun\" python3 /tmp/yolov5/train.py --img {IMG_SIZE} --batch {BATCH_SIZE} --epochs {N_EPOCHS} --data data.yaml --cfg {MODEL_NAME}.yaml --name {CHECKPOINTS_ROOT} --weights {MODEL_NAME}.pt --device 0 --hyp hyperparameters.yaml --cache","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:17:05.269385Z","iopub.execute_input":"2021-07-19T17:17:05.269912Z","iopub.status.idle":"2021-07-19T17:20:38.693499Z","shell.execute_reply.started":"2021-07-19T17:17:05.269877Z","shell.execute_reply":"2021-07-19T17:20:38.692307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(time.time()-start)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:20:52.611403Z","iopub.execute_input":"2021-07-19T17:20:52.611746Z","iopub.status.idle":"2021-07-19T17:20:52.616776Z","shell.execute_reply.started":"2021-07-19T17:20:52.611715Z","shell.execute_reply":"2021-07-19T17:20:52.615717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualisation","metadata":{}},{"cell_type":"code","source":"## Visualise Results for single batch\n\nimport matplotlib.pyplot as plt\nprint(\"Targets\")\nplt.figure(figsize=(30,30))\nplt.imshow(plt.imread(f'runs/train/{CHECKPOINTS_ROOT}/test_batch0_labels.jpg'))\nplt.show()\nprint(\"Prediction\")\nplt.figure(figsize=(30,30))\nplt.imshow(plt.imread(f'runs/train/{CHECKPOINTS_ROOT}/test_batch0_pred.jpg'))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:20:54.207158Z","iopub.execute_input":"2021-07-19T17:20:54.207515Z","iopub.status.idle":"2021-07-19T17:20:57.208474Z","shell.execute_reply.started":"2021-07-19T17:20:54.207488Z","shell.execute_reply":"2021-07-19T17:20:57.207649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs(CHECKPOINTS_ROOT,exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:22:09.395455Z","iopub.execute_input":"2021-07-19T17:22:09.395791Z","iopub.status.idle":"2021-07-19T17:22:09.400402Z","shell.execute_reply.started":"2021-07-19T17:22:09.395762Z","shell.execute_reply":"2021-07-19T17:22:09.399475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r runs/train/{CHECKPOINTS_ROOT}/weights {CHECKPOINTS_ROOT}/.\n!rm -r runs\n!rm -r wandb","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:22:09.920098Z","iopub.execute_input":"2021-07-19T17:22:09.920449Z","iopub.status.idle":"2021-07-19T17:22:12.41398Z","shell.execute_reply.started":"2021-07-19T17:22:09.920418Z","shell.execute_reply":"2021-07-19T17:22:12.412777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -lh","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:22:12.416064Z","iopub.execute_input":"2021-07-19T17:22:12.416437Z","iopub.status.idle":"2021-07-19T17:22:13.096561Z","shell.execute_reply.started":"2021-07-19T17:22:12.416398Z","shell.execute_reply":"2021-07-19T17:22:13.09553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}