{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom shutil import copyfile\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\n#customize iPython writefile so we can write variables\nfrom IPython.core.magic import register_line_cell_magic\n\n@register_line_cell_magic\ndef writetemplate(line, cell):\n    with open(line, 'w') as f:\n        f.write(cell.format(**globals()))\n","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:34.898888Z","iopub.execute_input":"2021-07-11T07:45:34.899288Z","iopub.status.idle":"2021-07-11T07:45:35.795887Z","shell.execute_reply.started":"2021-07-11T07:45:34.899208Z","shell.execute_reply":"2021-07-11T07:45:35.795086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data_dir='../input/siim-covid19-detection'\ntrain=pd.read_csv(f'{Data_dir}/train_image_level.csv')\ntrain_study=pd.read_csv(f'{Data_dir}/train_study_level.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:35.797338Z","iopub.execute_input":"2021-07-11T07:45:35.797656Z","iopub.status.idle":"2021-07-11T07:45:35.854749Z","shell.execute_reply.started":"2021-07-11T07:45:35.797620Z","shell.execute_reply":"2021-07-11T07:45:35.853990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:35.856491Z","iopub.execute_input":"2021-07-11T07:45:35.856750Z","iopub.status.idle":"2021-07-11T07:45:35.879726Z","shell.execute_reply.started":"2021-07-11T07:45:35.856726Z","shell.execute_reply":"2021-07-11T07:45:35.879041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd ../\n!mkdir tmp\n%cd tmp","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:35.882626Z","iopub.execute_input":"2021-07-11T07:45:35.882865Z","iopub.status.idle":"2021-07-11T07:45:36.533825Z","shell.execute_reply.started":"2021-07-11T07:45:35.882840Z","shell.execute_reply":"2021-07-11T07:45:36.532767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Download YOLOv5\n!git clone https://github.com/ultralytics/yolov5  # clone repo\n%cd yolov5\n# Install dependencies\n%pip install -qr requirements.txt  # install dependencies\n\n%cd ../\nimport torch\nprint(f\"Setup complete. Using torch {torch.__version__} ({torch.cuda.get_device_properties(0).name if torch.cuda.is_available() else 'CPU'})\")","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:36.535328Z","iopub.execute_input":"2021-07-11T07:45:36.535669Z","iopub.status.idle":"2021-07-11T07:45:47.062386Z","shell.execute_reply.started":"2021-07-11T07:45:36.535629Z","shell.execute_reply":"2021-07-11T07:45:47.061424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = 'input/siim-covid19-resized-to-256px-jpg/train/'\nIMG_SIZE = 256\nBATCH_SIZE = 16\nEPOCHS = 10","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:47.063793Z","iopub.execute_input":"2021-07-11T07:45:47.064148Z","iopub.status.idle":"2021-07-11T07:45:47.069856Z","shell.execute_reply.started":"2021-07-11T07:45:47.064110Z","shell.execute_reply":"2021-07-11T07:45:47.067433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Everything is done from /kaggle directory.\n%cd ../\n\n# Load image level csv file\ndf = pd.read_csv('input/siim-covid19-detection/train_image_level.csv')\ndf['id'] = df.apply(lambda row: row.id.split('_')[0], axis=1)\ndf['image_path']=df.apply(lambda row:TRAIN_PATH+row.id+'.jpg', axis=1)\ndf['image_level']=df.apply(lambda row: row.label.split(' ')[0],axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:47.071333Z","iopub.execute_input":"2021-07-11T07:45:47.072091Z","iopub.status.idle":"2021-07-11T07:45:47.362688Z","shell.execute_reply.started":"2021-07-11T07:45:47.072054Z","shell.execute_reply":"2021-07-11T07:45:47.361932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load meta.csv file\n# Original dimensions are required to scale the bounding box coordinates appropriately.\nmeta_df = pd.read_csv('input/siim-covid19-resized-to-256px-jpg/meta.csv')\ntrain_meta_df = meta_df.loc[meta_df.split == 'train']\ntrain_meta_df = train_meta_df.drop('split', axis=1)\ntrain_meta_df.columns = ['id', 'dim0', 'dim1']\n\ntrain_meta_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:47.363828Z","iopub.execute_input":"2021-07-11T07:45:47.364182Z","iopub.status.idle":"2021-07-11T07:45:47.393012Z","shell.execute_reply.started":"2021-07-11T07:45:47.364146Z","shell.execute_reply":"2021-07-11T07:45:47.392047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1=df.copy()","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:47.395530Z","iopub.execute_input":"2021-07-11T07:45:47.395874Z","iopub.status.idle":"2021-07-11T07:45:47.399838Z","shell.execute_reply.started":"2021-07-11T07:45:47.395839Z","shell.execute_reply":"2021-07-11T07:45:47.398753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_df['id']=meta_df['image_id']","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:47.401625Z","iopub.execute_input":"2021-07-11T07:45:47.402343Z","iopub.status.idle":"2021-07-11T07:45:47.410428Z","shell.execute_reply.started":"2021-07-11T07:45:47.402303Z","shell.execute_reply":"2021-07-11T07:45:47.409552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_df=meta_df.drop(['image_id'],axis=1)\nmeta_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:47.413076Z","iopub.execute_input":"2021-07-11T07:45:47.413320Z","iopub.status.idle":"2021-07-11T07:45:47.428195Z","shell.execute_reply.started":"2021-07-11T07:45:47.413297Z","shell.execute_reply":"2021-07-11T07:45:47.427187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = df1.merge(meta_df, on='id',how=\"left\")\ndf1.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:47.429541Z","iopub.execute_input":"2021-07-11T07:45:47.429894Z","iopub.status.idle":"2021-07-11T07:45:47.454771Z","shell.execute_reply.started":"2021-07-11T07:45:47.429859Z","shell.execute_reply":"2021-07-11T07:45:47.453867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('tmp/covid/images/train', exist_ok=True)\nos.makedirs('tmp/covid/images/valid', exist_ok=True)\n\nos.makedirs('tmp/covid/labels/train', exist_ok=True)\nos.makedirs('tmp/covid/labels/valid', exist_ok=True)\n\n! ls tmp/covid/images","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:47.455964Z","iopub.execute_input":"2021-07-11T07:45:47.456451Z","iopub.status.idle":"2021-07-11T07:45:48.119050Z","shell.execute_reply.started":"2021-07-11T07:45:47.456413Z","shell.execute_reply":"2021-07-11T07:45:48.118120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" df=df.merge(meta_df, on='id',how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:48.120610Z","iopub.execute_input":"2021-07-11T07:45:48.120955Z","iopub.status.idle":"2021-07-11T07:45:48.137813Z","shell.execute_reply.started":"2021-07-11T07:45:48.120916Z","shell.execute_reply":"2021-07-11T07:45:48.136798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Move the images to relevant split folder.\nfor i in tqdm(range(len(df1))):\n    row = df1.loc[i]\n    if row.split == 'train':\n        copyfile(row.image_path, f'tmp/covid/images/train/{row.id}.jpg')\n    else:\n        copyfile(row.image_path, f'tmp/covid/images/valid/{row.id}.jpg')","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:45:48.139110Z","iopub.execute_input":"2021-07-11T07:45:48.139614Z","iopub.status.idle":"2021-07-11T07:46:12.964237Z","shell.execute_reply.started":"2021-07-11T07:45:48.139578Z","shell.execute_reply":"2021-07-11T07:46:12.963190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" #Create .yaml file \nimport yaml\n\ndata_yaml = dict(\n    train = '../covid/images/train',\n    val = '../covid/images/valid',\n    nc = 2,\n    names = ['none', 'opacity']\n)\n\n# Note that I am creating the file in the yolov5/data/ directory.\nwith open('tmp/yolov5/data/data.yaml', 'w') as outfile:\n    yaml.dump(data_yaml, outfile, default_flow_style=True)\n    \n%cat tmp/yolov5/data/data.yaml\n","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:47:21.097472Z","iopub.execute_input":"2021-07-11T07:47:21.097830Z","iopub.status.idle":"2021-07-11T07:47:21.735741Z","shell.execute_reply.started":"2021-07-11T07:47:21.097797Z","shell.execute_reply":"2021-07-11T07:47:21.734834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bbox(row):\n    bboxes = []\n    bbox = []\n    for i,l in enumerate(row.label.split(' ')):\n        if (i % 6 == 0 )| (i % 6 == 1):\n            continue \n            bbox.append(float(l))\n            if i % 6 == 5:\n                bboxes.append(bbox)\n                bbox = []\n    return bboxes \n\ndef scale_bbox(row, bboxes):\n    # Get scaling factor\n    scale_x = IMG_SIZE/row.dim1\n    scale_y = IMG_SIZE/row.dim0\n    \n    scaled_bboxes = []\n    for bbox in bboxes:\n        x = int(np.round(bbox[0]*scale_x, 4))\n        y = int(np.round(bbox[1]*scale_y, 4))\n        x1 = int(np.round(bbox[2]*(scale_x), 4))\n        y1= int(np.round(bbox[3]*scale_y, 4))\n\n        scaled_bboxes.append([x, y, x1, y1]) # xmin, ymin, xmax, ymax\n        \n    return scaled_bboxes\n\n# Convert the bounding boxes in YOLO format.\ndef get_yolo_format_bbox(img_w, img_h, bboxes):\n    yolo_boxes = []\n    for bbox in bboxes:\n        w = bbox[2] - bbox[0] # xmax - xmin\n        h = bbox[3] - bbox[1] # ymax - ymin\n        xc = bbox[0] + int(np.round(w/2)) # xmin + width/2\n        yc = bbox[1] + int(np.round(h/2)) # ymin + height/2\n        \n        yolo_boxes.append([xc/img_w, yc/img_h, w/img_w, h/img_h]) # x_center y_center width height\n    \n    return yolo_boxes","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:47:24.536787Z","iopub.execute_input":"2021-07-11T07:47:24.537128Z","iopub.status.idle":"2021-07-11T07:47:24.547162Z","shell.execute_reply.started":"2021-07-11T07:47:24.537093Z","shell.execute_reply":"2021-07-11T07:47:24.546346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare the txt files for bounding box\nfor i in tqdm(range(len(df))):\n    row = df.loc[i]\n    # Get image id\n    img_id = row.id\n    # Get split\n    split = row.split\n    # Get image-level label\n    label = row.image_level\n    \n    if row.split=='train':\n        file_name = f'tmp/covid/labels/train/{row.id}.txt'\n    else:\n        file_name = f'tmp/covid/labels/valid/{row.id}.txt'\n        \n    \n    if label=='opacity':\n        # Get bboxes\n        bboxes = get_bbox(row)\n        # Scale bounding boxes\n        scale_bboxes = scale_bbox(row, bboxes)\n        # Format for YOLOv5\n        yolo_bboxes = get_yolo_format_bbox(IMG_SIZE, IMG_SIZE, scale_bboxes)\n        \n        with open(file_name, 'w') as f:\n            for bbox in yolo_bboxes:\n                bbox = [1]+bbox\n                bbox = [str(i) for i in bbox]\n                bbox = ' '.join(bbox)\n                f.write(bbox)\n                f.write('\\n')\n","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:47:27.710423Z","iopub.execute_input":"2021-07-11T07:47:27.710748Z","iopub.status.idle":"2021-07-11T07:47:29.393953Z","shell.execute_reply.started":"2021-07-11T07:47:27.710716Z","shell.execute_reply":"2021-07-11T07:47:29.392965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd tmp/yolov5/","metadata":{"execution":{"iopub.status.busy":"2021-07-11T07:47:41.120326Z","iopub.execute_input":"2021-07-11T07:47:41.120651Z","iopub.status.idle":"2021-07-11T07:47:41.126637Z","shell.execute_reply.started":"2021-07-11T07:47:41.120620Z","shell.execute_reply":"2021-07-11T07:47:41.125411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!WANDB_MODE=\"dryrun\" python train.py --img 256 \\\n                 --batch 16 \\\n                 --epochs 10 \\\n                 --data data.yaml \\\n                 --weights yolov5x.pt --cache","metadata":{"execution":{"iopub.status.busy":"2021-07-11T08:04:43.691146Z","iopub.execute_input":"2021-07-11T08:04:43.691491Z","iopub.status.idle":"2021-07-11T08:05:13.841533Z","shell.execute_reply.started":"2021-07-11T08:04:43.691456Z","shell.execute_reply":"2021-07-11T08:05:13.840538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}