{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%cd ../\n!mkdir /kaggle/tmp\n!mkdir /kaggle/tmp/mask\n!mkdir /kaggle/tmp/train\n!mkdir /kaggle/tmp/test\n%cd tmp","metadata":{"execution":{"iopub.status.busy":"2021-07-31T04:59:46.111331Z","iopub.execute_input":"2021-07-31T04:59:46.111792Z","iopub.status.idle":"2021-07-31T04:59:49.215836Z","shell.execute_reply.started":"2021-07-31T04:59:46.111661Z","shell.execute_reply":"2021-07-31T04:59:49.214556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tar -zxf /kaggle/input/siiim-covid-stratified-k-fold-and-create-mask/mask.tar.gz -C /kaggle/tmp/mask\n!tar -zxf /kaggle/input/siiim-covid-stratified-k-fold-and-create-mask/train.tar.gz -C /kaggle/tmp/train\n!tar -zxf /kaggle/input/siiim-covid-stratified-k-fold-and-create-mask/test.tar.gz -C /kaggle/tmp/test","metadata":{"execution":{"iopub.status.busy":"2021-07-31T04:59:49.220006Z","iopub.execute_input":"2021-07-31T04:59:49.22033Z","iopub.status.idle":"2021-07-31T05:00:05.664591Z","shell.execute_reply.started":"2021-07-31T04:59:49.220293Z","shell.execute_reply":"2021-07-31T05:00:05.663205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Download YOLOv5\n!git clone https://github.com/ultralytics/yolov5  # clone repo\n%cd yolov5\n# Install dependencies\n%pip install -qr requirements.txt  # install dependencies","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:05.669015Z","iopub.execute_input":"2021-07-31T05:00:05.669506Z","iopub.status.idle":"2021-07-31T05:00:19.384552Z","shell.execute_reply.started":"2021-07-31T05:00:05.669455Z","shell.execute_reply":"2021-07-31T05:00:19.38327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd ../","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:19.387077Z","iopub.execute_input":"2021-07-31T05:00:19.387568Z","iopub.status.idle":"2021-07-31T05:00:19.395881Z","shell.execute_reply.started":"2021-07-31T05:00:19.387518Z","shell.execute_reply":"2021-07-31T05:00:19.394665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Install W&B \n#!pip install -q --upgrade wandb\n## Login \n#import wandb\n#wandb.login()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:19.397997Z","iopub.execute_input":"2021-07-31T05:00:19.398505Z","iopub.status.idle":"2021-07-31T05:00:19.408241Z","shell.execute_reply.started":"2021-07-31T05:00:19.398417Z","shell.execute_reply":"2021-07-31T05:00:19.406908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom shutil import copyfile\nimport matplotlib.pyplot as plt\n\nimport csv","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:19.412035Z","iopub.execute_input":"2021-07-31T05:00:19.412406Z","iopub.status.idle":"2021-07-31T05:00:21.146636Z","shell.execute_reply.started":"2021-07-31T05:00:19.412347Z","shell.execute_reply":"2021-07-31T05:00:21.145453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_study_level.csv\")\nimage_df = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_image_level.csv\")\nmeta_df = pd.read_csv(\"/kaggle/input/siiim-covid-stratified-k-fold-and-create-mask/meta.csv\")\nfold_df = pd.read_csv(\"/kaggle/input/siiim-covid-stratified-k-fold-and-create-mask/updated_iamge_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:21.150526Z","iopub.execute_input":"2021-07-31T05:00:21.150889Z","iopub.status.idle":"2021-07-31T05:00:21.281652Z","shell.execute_reply.started":"2021-07-31T05:00:21.150858Z","shell.execute_reply":"2021-07-31T05:00:21.280543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_duplicateList_path = '/kaggle/input/siiim-covid-stratified-k-fold-and-create-mask/dublicate.txt'","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:21.285453Z","iopub.execute_input":"2021-07-31T05:00:21.285875Z","iopub.status.idle":"2021-07-31T05:00:21.29068Z","shell.execute_reply.started":"2021-07-31T05:00:21.285828Z","shell.execute_reply":"2021-07-31T05:00:21.28914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_duplicateList = []\nwith open(_duplicateList_path, newline='') as csvfile:\n    spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|')\n    for row in spamreader:\n        _duplicateList += row\n\n_duplicateList[:5]","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:21.293603Z","iopub.execute_input":"2021-07-31T05:00:21.294452Z","iopub.status.idle":"2021-07-31T05:00:21.316104Z","shell.execute_reply.started":"2021-07-31T05:00:21.294381Z","shell.execute_reply":"2021-07-31T05:00:21.314775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = '/kaggle/tmp/train/'\nIMG_SIZE = 512\nBATCH_SIZE = 16\nEPOCHS = 16","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:21.317848Z","iopub.execute_input":"2021-07-31T05:00:21.318324Z","iopub.status.idle":"2021-07-31T05:00:21.324276Z","shell.execute_reply.started":"2021-07-31T05:00:21.318266Z","shell.execute_reply":"2021-07-31T05:00:21.322595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modify values in the id column\ndf = fold_df.copy()\n\ndf['id'] = df.apply(lambda row: row.id.split('_')[0], axis=1)\n# Add absolute path\ndf['path'] = df.apply(lambda row: TRAIN_PATH+row.id+'.png', axis=1)\n# Get image level labels\ndf['image_level'] = df.apply(lambda row: row.label.split(' ')[0], axis=1)\n\ndf.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:21.32629Z","iopub.execute_input":"2021-07-31T05:00:21.326986Z","iopub.status.idle":"2021-07-31T05:00:21.703213Z","shell.execute_reply.started":"2021-07-31T05:00:21.326942Z","shell.execute_reply":"2021-07-31T05:00:21.701877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:21.705272Z","iopub.execute_input":"2021-07-31T05:00:21.705761Z","iopub.status.idle":"2021-07-31T05:00:21.719776Z","shell.execute_reply.started":"2021-07-31T05:00:21.705673Z","shell.execute_reply":"2021-07-31T05:00:21.718236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.merge(meta_df, left_on='id', right_on=\"image_id\")\ndf.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:21.72165Z","iopub.execute_input":"2021-07-31T05:00:21.722208Z","iopub.status.idle":"2021-07-31T05:00:21.760063Z","shell.execute_reply.started":"2021-07-31T05:00:21.722163Z","shell.execute_reply":"2021-07-31T05:00:21.758708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"before drop duplicate\", len(df))\ndf = df[~df['id'].isin(_duplicateList)]\nprint(\"after drop duplicate\", len(df))","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:21.761875Z","iopub.execute_input":"2021-07-31T05:00:21.76235Z","iopub.status.idle":"2021-07-31T05:00:21.776777Z","shell.execute_reply.started":"2021-07-31T05:00:21.762305Z","shell.execute_reply":"2021-07-31T05:00:21.775502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('tmp/covid/images/train', exist_ok=True)\nos.makedirs('tmp/covid/images/valid', exist_ok=True)\n\nos.makedirs('tmp/covid/labels/train', exist_ok=True)\nos.makedirs('tmp/covid/labels/valid', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:21.778408Z","iopub.execute_input":"2021-07-31T05:00:21.778964Z","iopub.status.idle":"2021-07-31T05:00:21.78855Z","shell.execute_reply.started":"2021-07-31T05:00:21.778913Z","shell.execute_reply":"2021-07-31T05:00:21.787198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:21.790447Z","iopub.execute_input":"2021-07-31T05:00:21.79103Z","iopub.status.idle":"2021-07-31T05:00:21.820847Z","shell.execute_reply.started":"2021-07-31T05:00:21.790981Z","shell.execute_reply":"2021-07-31T05:00:21.819287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Move the images to relevant split folder.\n# 5 fold\nfor _fold in range(5):\n    os.makedirs(f'/kaggle/tmp/covid/images/train/fold{_fold}', exist_ok=True)\n    os.makedirs(f'/kaggle/tmp/covid/images/valid/fold{_fold}', exist_ok=True)\n\n    for i in tqdm(range(len(df))):\n        row = df.iloc[i]\n        if row.fold != _fold:\n            copyfile(row.path, f'/kaggle/tmp/covid/images/train/fold{_fold}/{row.id}.png')\n        else:\n            copyfile(row.path, f'/kaggle/tmp/covid/images/valid/fold{_fold}/{row.id}.png')","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:21.822903Z","iopub.execute_input":"2021-07-31T05:00:21.823477Z","iopub.status.idle":"2021-07-31T05:00:46.41132Z","shell.execute_reply.started":"2021-07-31T05:00:21.823433Z","shell.execute_reply":"2021-07-31T05:00:46.409363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/tmp/covid/images/train/fold0/ | wc -l","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:46.412859Z","iopub.execute_input":"2021-07-31T05:00:46.413215Z","iopub.status.idle":"2021-07-31T05:00:47.228562Z","shell.execute_reply.started":"2021-07-31T05:00:46.413177Z","shell.execute_reply":"2021-07-31T05:00:47.227339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the raw bounding box by parsing the row value of the label column.\n# Ref: https://www.kaggle.com/yujiariyasu/plot-3positive-classes\ndef get_bbox(row):\n    bboxes = []\n    bbox = []\n    for i, l in enumerate(row.label.split(' ')):\n        if (i % 6 == 0) | (i % 6 == 1):\n            continue\n        bbox.append(float(l))\n        if i % 6 == 5:\n            bboxes.append(bbox)\n            bbox = []  \n            \n    return bboxes\n\n# Scale the bounding boxes according to the size of the resized image. \ndef scale_bbox(row, bboxes):\n    # Get scaling factor\n    scale_x = IMG_SIZE/row.dim1\n    scale_y = IMG_SIZE/row.dim0\n    \n    scaled_bboxes = []\n    for bbox in bboxes:\n        x = int(np.round(bbox[0]*scale_x, 4))\n        y = int(np.round(bbox[1]*scale_y, 4))\n        x1 = int(np.round(bbox[2]*(scale_x), 4))\n        y1= int(np.round(bbox[3]*scale_y, 4))\n\n        scaled_bboxes.append([x, y, x1, y1]) # xmin, ymin, xmax, ymax\n        \n    return scaled_bboxes\n\n# Convert the bounding boxes in YOLO format.\ndef get_yolo_format_bbox(img_w, img_h, bboxes):\n    yolo_boxes = []\n    for bbox in bboxes:\n        w = bbox[2] - bbox[0] # xmax - xmin\n        h = bbox[3] - bbox[1] # ymax - ymin\n        xc = bbox[0] + int(np.round(w/2)) # xmin + width/2\n        yc = bbox[1] + int(np.round(h/2)) # ymin + height/2\n        \n        yolo_boxes.append([xc/img_w, yc/img_h, w/img_w, h/img_h]) # x_center y_center width height\n    \n    return yolo_boxes","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:47.230279Z","iopub.execute_input":"2021-07-31T05:00:47.23074Z","iopub.status.idle":"2021-07-31T05:00:57.494693Z","shell.execute_reply.started":"2021-07-31T05:00:47.230692Z","shell.execute_reply":"2021-07-31T05:00:57.493446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare the txt files for bounding box\nfor _fold in range(5):\n    os.makedirs(f'/kaggle/tmp/covid/labels/train/fold{_fold}', exist_ok=True)\n    os.makedirs(f'/kaggle/tmp/covid/labels/valid/fold{_fold}', exist_ok=True)\n    \n    for i in tqdm(range(len(df))):\n        row = df.iloc[i]\n        # Get image id\n        img_id = row.id\n        # Get split\n        split = row.split\n        # Get image-level label\n        label = row.image_level\n\n        if row.fold != _fold:\n            file_name = f'/kaggle/tmp/covid/labels/train/fold{_fold}/{row.id}.txt'\n        else:\n            file_name = f'/kaggle/tmp/covid/labels/valid/fold{_fold}/{row.id}.txt'\n        \n        if label=='opacity':\n            # Get bboxes\n            bboxes = get_bbox(row)\n            # Scale bounding boxes\n            scale_bboxes = scale_bbox(row, bboxes)\n            # Format for YOLOv5\n            yolo_bboxes = get_yolo_format_bbox(IMG_SIZE, IMG_SIZE, scale_bboxes)\n\n            with open(file_name, 'w') as f:\n                for bbox in yolo_bboxes:\n                    bbox = [1]+bbox\n                    bbox = [str(i) for i in bbox]\n                    bbox = ' '.join(bbox)\n                    f.write(bbox)\n                    f.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:00:57.496442Z","iopub.execute_input":"2021-07-31T05:00:57.496918Z","iopub.status.idle":"2021-07-31T05:01:17.321682Z","shell.execute_reply.started":"2021-07-31T05:00:57.49687Z","shell.execute_reply":"2021-07-31T05:01:17.320377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat /kaggle/tmp/covid/labels/valid/fold0/0012ff7358bc.txt","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:01:17.323423Z","iopub.execute_input":"2021-07-31T05:01:17.324138Z","iopub.status.idle":"2021-07-31T05:01:18.088711Z","shell.execute_reply.started":"2021-07-31T05:01:17.324092Z","shell.execute_reply":"2021-07-31T05:01:18.087373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/tmp/covid/labels/valid/fold1 | wc -l","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:01:18.090853Z","iopub.execute_input":"2021-07-31T05:01:18.091392Z","iopub.status.idle":"2021-07-31T05:01:18.852274Z","shell.execute_reply.started":"2021-07-31T05:01:18.09132Z","shell.execute_reply":"2021-07-31T05:01:18.851126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd yolov5","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:01:18.858416Z","iopub.execute_input":"2021-07-31T05:01:18.858765Z","iopub.status.idle":"2021-07-31T05:01:18.866437Z","shell.execute_reply.started":"2021-07-31T05:01:18.858708Z","shell.execute_reply":"2021-07-31T05:01:18.865132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create .yaml file \nimport yaml\nfor _fold in range(5):\n    data_yaml = dict(\n        train = f'../covid/images/train/fold{_fold}',\n        val = f'../covid/images/valid/fold{_fold}',\n        nc = 2,\n        names = ['none', 'opacity']\n    )\n\n    # Note that I am creating the file in the yolov5/data/ directory.\n    with open(f'data/data-fold{_fold}.yaml', 'w') as outfile:\n        yaml.dump(data_yaml, outfile, default_flow_style=True)\n    \n%cat data/data-fold0.yaml\n%cat data/data-fold1.yaml\n%cat data/data-fold2.yaml\n%cat data/data-fold3.yaml\n%cat data/data-fold4.yaml","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:01:18.868355Z","iopub.execute_input":"2021-07-31T05:01:18.86926Z","iopub.status.idle":"2021-07-31T05:01:22.67928Z","shell.execute_reply.started":"2021-07-31T05:01:18.869216Z","shell.execute_reply":"2021-07-31T05:01:22.677599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/tmp/covid/labels/valid/fold0 | wc -l","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:01:22.685958Z","iopub.execute_input":"2021-07-31T05:01:22.689511Z","iopub.status.idle":"2021-07-31T05:01:23.475487Z","shell.execute_reply.started":"2021-07-31T05:01:22.689458Z","shell.execute_reply":"2021-07-31T05:01:23.474241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!WANDB_MODE='dryrun' python train.py --img {IMG_SIZE} \\\n                 --batch {BATCH_SIZE} \\\n                 --epochs {EPOCHS} \\\n                 --data data-fold0.yaml \\\n                 --weights yolov5s.pt \\\n                 --save_period 1\\\n                 --project /kaggle/working/kaggle-siim-covid","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:01:23.479115Z","iopub.execute_input":"2021-07-31T05:01:23.479438Z","iopub.status.idle":"2021-07-31T05:06:34.775519Z","shell.execute_reply.started":"2021-07-31T05:01:23.479407Z","shell.execute_reply":"2021-07-31T05:06:34.774328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working/kaggle-siim-covid","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:06:34.777451Z","iopub.execute_input":"2021-07-31T05:06:34.777929Z","iopub.status.idle":"2021-07-31T05:06:35.544542Z","shell.execute_reply.started":"2021-07-31T05:06:34.777879Z","shell.execute_reply":"2021-07-31T05:06:35.543186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%cp /content/yolov5/weights/","metadata":{"execution":{"iopub.status.busy":"2021-07-31T05:06:35.546816Z","iopub.execute_input":"2021-07-31T05:06:35.547286Z","iopub.status.idle":"2021-07-31T05:06:35.651856Z","shell.execute_reply.started":"2021-07-31T05:06:35.547238Z","shell.execute_reply":"2021-07-31T05:06:35.649661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!WANDB_MODE='dryrun' python train.py --img {IMG_SIZE} \\\n                 --batch {BATCH_SIZE} \\\n                 --epochs {EPOCHS} \\\n                 --data data-fold1.yaml \\\n                 --weights yolov5s.pt \\\n                 --save_period 1\\\n                 --project /kaggle/working/kaggle-siim-covid","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!WANDB_MODE='dryrun' python train.py --img {IMG_SIZE} \\\n                 --batch {BATCH_SIZE} \\\n                 --epochs {EPOCHS} \\\n                 --data data-fold2.yaml \\\n                 --weights yolov5s.pt \\\n                 --save_period 1\\\n                 --project /kaggle/working/kaggle-siim-covid","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!WANDB_MODE='dryrun' python train.py --img {IMG_SIZE} \\\n                 --batch {BATCH_SIZE} \\\n                 --epochs {EPOCHS} \\\n                 --data data-fold3.yaml \\\n                 --weights yolov5s.pt \\\n                 --save_period 1\\\n                 --project /kaggle/working/kaggle-siim-covid","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!WANDB_MODE='dryrun' python train.py --img {IMG_SIZE} \\\n                 --batch {BATCH_SIZE} \\\n                 --epochs {EPOCHS} \\\n                 --data data-fold4.yaml \\\n                 --weights yolov5s.pt \\\n                 --save_period 1\\\n                 --project /kaggle/working/kaggle-siim-covid","metadata":{},"execution_count":null,"outputs":[]}]}