{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"thanks to:https://www.kaggle.com/awsaf49/vinbigdata-cxr-ad-yolov5-14-class-train  \nAdjust the hyperparameters from hyp.scratch.yaml  \nThanks to [adrielcabral](https://www.kaggle.com/adrielcabral) for hyperparameters  \nhttps://www.kaggle.com/c/vinbigdata-chest-xray-abnormalities-detection/discussion/222707","metadata":{}},{"cell_type":"code","source":"!pip install --upgrade seaborn","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"papermill":{"duration":9.633907,"end_time":"2021-01-01T09:44:53.448657","exception":false,"start_time":"2021-01-01T09:44:43.81475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-11T09:05:01.895923Z","iopub.execute_input":"2021-07-11T09:05:01.896315Z","iopub.status.idle":"2021-07-11T09:05:13.539691Z","shell.execute_reply.started":"2021-07-11T09:05:01.896282Z","shell.execute_reply":"2021-07-11T09:05:13.538648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns","metadata":{"papermill":{"duration":0.926929,"end_time":"2021-01-01T09:44:54.403588","exception":false,"start_time":"2021-01-01T09:44:53.476659","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-11T09:05:13.542615Z","iopub.execute_input":"2021-07-11T09:05:13.543523Z","iopub.status.idle":"2021-07-11T09:05:14.654530Z","shell.execute_reply.started":"2021-07-11T09:05:13.543353Z","shell.execute_reply":"2021-07-11T09:05:14.653585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 0","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:05:14.656156Z","iopub.execute_input":"2021-07-11T09:05:14.656576Z","iopub.status.idle":"2021-07-11T09:05:14.665601Z","shell.execute_reply.started":"2021-07-11T09:05:14.656532Z","shell.execute_reply":"2021-07-11T09:05:14.664216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f'../input/siim-covid19-detection/train_image_level.csv')\ntrain_df.head()","metadata":{"papermill":{"duration":0.262045,"end_time":"2021-01-01T09:44:54.691965","exception":false,"start_time":"2021-01-01T09:44:54.42992","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-11T09:05:14.667855Z","iopub.execute_input":"2021-07-11T09:05:14.668820Z","iopub.status.idle":"2021-07-11T09:05:14.743225Z","shell.execute_reply.started":"2021-07-11T09:05:14.668772Z","shell.execute_reply":"2021-07-11T09:05:14.741978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\ndf = train_df\n\ngkf  = GroupKFold(n_splits = 5)\ndf['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(df, groups = df.StudyInstanceUID.tolist())):\n    df.loc[val_idx, 'fold'] = fold","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:05:14.747083Z","iopub.execute_input":"2021-07-11T09:05:14.747536Z","iopub.status.idle":"2021-07-11T09:05:14.798111Z","shell.execute_reply.started":"2021-07-11T09:05:14.747478Z","shell.execute_reply":"2021-07-11T09:05:14.797083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = df","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:05:14.799988Z","iopub.execute_input":"2021-07-11T09:05:14.800466Z","iopub.status.idle":"2021-07-11T09:05:14.806262Z","shell.execute_reply.started":"2021-07-11T09:05:14.800424Z","shell.execute_reply":"2021-07-11T09:05:14.804803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['image_path'] = f'/kaggle/input/siimcovid19-512-img-png-600-study-png/image/' + train_df.id + '.png'\ntrain_df.head()","metadata":{"papermill":{"duration":0.086788,"end_time":"2021-01-01T09:44:54.805857","exception":false,"start_time":"2021-01-01T09:44:54.719069","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-11T09:05:14.808456Z","iopub.execute_input":"2021-07-11T09:05:14.809553Z","iopub.status.idle":"2021-07-11T09:05:14.846144Z","shell.execute_reply.started":"2021-07-11T09:05:14.809491Z","shell.execute_reply":"2021-07-11T09:05:14.845264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = ['0. opacity']","metadata":{"papermill":{"duration":0.050418,"end_time":"2021-01-01T09:45:03.002944","exception":false,"start_time":"2021-01-01T09:45:02.952526","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-11T09:05:14.847929Z","iopub.execute_input":"2021-07-11T09:05:14.848545Z","iopub.status.idle":"2021-07-11T09:05:14.853918Z","shell.execute_reply.started":"2021-07-11T09:05:14.848504Z","shell.execute_reply":"2021-07-11T09:05:14.852540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nval_files   = []\nval_files += list(train_df[train_df.fold==fold].image_path.unique())\ntrain_files += list(train_df[train_df.fold!=fold].image_path.unique())\nlen(train_files), len(val_files)","metadata":{"papermill":{"duration":0.086817,"end_time":"2021-01-01T09:47:56.443789","exception":false,"start_time":"2021-01-01T09:47:56.356972","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-11T09:05:14.855678Z","iopub.execute_input":"2021-07-11T09:05:14.856522Z","iopub.status.idle":"2021-07-11T09:05:14.875835Z","shell.execute_reply.started":"2021-07-11T09:05:14.856473Z","shell.execute_reply":"2021-07-11T09:05:14.874276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Copying Files","metadata":{"papermill":{"duration":0.083752,"end_time":"2021-01-01T09:47:56.584924","exception":false,"start_time":"2021-01-01T09:47:56.501172","status":"completed"},"tags":[]}},{"cell_type":"code","source":"os.makedirs('/kaggle/working/siim-cov19/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/images/val', exist_ok = True)\nlabel_dir = '/kaggle/input/siim-covid-19-yolo-txt'\nfor file in tqdm(train_files):\n    shutil.copy(file, '/kaggle/working/siim-cov19/images/train')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/siim-cov19/labels/train')\n    \nfor file in tqdm(val_files):\n    shutil.copy(file, '/kaggle/working/siim-cov19/images/val')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/siim-cov19/labels/val')","metadata":{"papermill":{"duration":124.654777,"end_time":"2021-01-01T09:50:01.331041","exception":false,"start_time":"2021-01-01T09:47:56.676264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-11T09:56:40.347365Z","iopub.execute_input":"2021-07-11T09:56:40.347760Z","iopub.status.idle":"2021-07-11T09:58:06.603656Z","shell.execute_reply.started":"2021-07-11T09:56:40.347726Z","shell.execute_reply":"2021-07-11T09:58:06.602263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Class Name","metadata":{"papermill":{"duration":0.068822,"end_time":"2021-01-01T09:50:01.458337","exception":false,"start_time":"2021-01-01T09:50:01.389515","status":"completed"},"tags":[]}},{"cell_type":"code","source":"classes = ['0. opacity']","metadata":{"papermill":{"duration":0.082234,"end_time":"2021-01-01T09:50:01.601574","exception":false,"start_time":"2021-01-01T09:50:01.51934","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-11T10:01:28.776950Z","iopub.execute_input":"2021-07-11T10:01:28.777464Z","iopub.status.idle":"2021-07-11T10:01:28.786477Z","shell.execute_reply.started":"2021-07-11T10:01:28.777416Z","shell.execute_reply":"2021-07-11T10:01:28.784984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# YOLOv5 Stuff","metadata":{"papermill":{"duration":0.055699,"end_time":"2021-01-01T09:50:01.82747","exception":false,"start_time":"2021-01-01T09:50:01.771771","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport yaml\n\ncwd = '/kaggle/working/'\n\nwith open(join( cwd , 'train.txt'), 'w') as f:\n    for path in glob('/kaggle/working/siim-cov19/images/train/*'):\n        f.write(path+'\\n')\n            \nwith open(join( cwd , 'val.txt'), 'w') as f:\n    for path in glob('/kaggle/working/siim-cov19/images/val/*'):\n        f.write(path+'\\n')\n\ndata = dict(\n    train =  join( cwd , 'train.txt') ,\n    val   =  join( cwd , 'val.txt' ),\n    nc    = 1,\n    names = classes\n    )\n\nwith open(join( cwd , 'siim-cov19.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(join( cwd , 'siim-cov19.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"papermill":{"duration":0.113001,"end_time":"2021-01-01T09:50:01.996448","exception":false,"start_time":"2021-01-01T09:50:01.883447","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-11T10:01:32.037160Z","iopub.execute_input":"2021-07-11T10:01:32.037538Z","iopub.status.idle":"2021-07-11T10:01:32.084100Z","shell.execute_reply.started":"2021-07-11T10:01:32.037505Z","shell.execute_reply":"2021-07-11T10:01:32.082839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r /kaggle/working/yolov5","metadata":{"execution":{"iopub.status.busy":"2021-07-11T09:09:15.859589Z","iopub.execute_input":"2021-07-11T09:09:15.859970Z","iopub.status.idle":"2021-07-11T09:09:16.623849Z","shell.execute_reply.started":"2021-07-11T09:09:15.859940Z","shell.execute_reply":"2021-07-11T09:09:16.622670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# https://www.kaggle.com/ultralytics/yolov5\n# !git clone https://github.com/ultralytics/yolov5  # clone repo\n# %cd yolov5\n\nshutil.copytree('/kaggle/input/siim-yolov5x6/yolov5', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5')\n\n# %pip install -qr requirements.txt # install dependencies\n\nimport torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":6.702428,"end_time":"2021-01-01T09:50:08.784153","exception":false,"start_time":"2021-01-01T09:50:02.081725","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-11T09:09:16.627389Z","iopub.execute_input":"2021-07-11T09:09:16.627865Z","iopub.status.idle":"2021-07-11T09:09:37.637490Z","shell.execute_reply.started":"2021-07-11T09:09:16.627829Z","shell.execute_reply":"2021-07-11T09:09:37.636335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pretrained Checkpoints:\n\n| Model | AP<sup>val</sup> | AP<sup>test</sup> | AP<sub>50</sub> | Speed<sub>GPU</sub> | FPS<sub>GPU</sub> || params | FLOPS |\n|---------- |------ |------ |------ | -------- | ------| ------ |------  |  :------: |\n| [YOLOv5s](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | 37.0     | 37.0     | 56.2     | **2.4ms** | **416** || 7.5M   | 13.2B\n| [YOLOv5m](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | 44.3     | 44.3     | 63.2     | 3.4ms     | 294     || 21.8M  | 39.4B\n| [YOLOv5l](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | 47.7     | 47.7     | 66.5     | 4.4ms     | 227     || 47.8M  | 88.1B\n| [YOLOv5x](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | **49.2** | **49.2** | **67.7** | 6.9ms     | 145     || 89.0M  | 166.4B\n| | | | | | || |\n| [YOLOv5x](https://github.com/ultralytics/yolov5/releases/tag/v3.0) + TTA|**50.8**| **50.8** | **68.9** | 25.5ms    | 39      || 89.0M  | 354.3B\n| | | | | | || |\n| [YOLOv3-SPP](https://github.com/ultralytics/yolov5/releases/tag/v3.0) | 45.6     | 45.5     | 65.2     | 4.5ms     | 222     || 63.0M  | 118.0B","metadata":{"papermill":{"duration":0.064911,"end_time":"2021-01-01T09:50:19.435746","exception":false,"start_time":"2021-01-01T09:50:19.370835","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Selecting Models\nIn this notebok I'm using `v5s`. To select your prefered model just replace `--cfg models/yolov5s.yaml --weights yolov5s.pt` with the following command:\n* `v5s` : `--cfg models/yolov5s.yaml --weights yolov5s.pt`\n* `v5m` : `--cfg models/yolov5m.yaml --weights yolov5m.pt`\n* `v5l` : `--cfg models/yolov5l.yaml --weights yolov5l.pt`\n* `v5x` : `--cfg models/yolov5x.yaml --weights yolov5x.pt`","metadata":{"papermill":{"duration":0.064016,"end_time":"2021-01-01T09:50:19.564859","exception":false,"start_time":"2021-01-01T09:50:19.500843","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Train","metadata":{"papermill":{"duration":0.064553,"end_time":"2021-01-01T09:50:19.6938","exception":false,"start_time":"2021-01-01T09:50:19.629247","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install -r requirements.txt","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-11T09:10:24.840793Z","iopub.execute_input":"2021-07-11T09:10:24.841325Z","iopub.status.idle":"2021-07-11T09:12:24.675870Z","shell.execute_reply.started":"2021-07-11T09:10:24.841294Z","shell.execute_reply":"2021-07-11T09:12:24.674564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install wandb","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-11T09:15:51.699125Z","iopub.execute_input":"2021-07-11T09:15:51.699662Z","iopub.status.idle":"2021-07-11T09:16:00.010784Z","shell.execute_reply.started":"2021-07-11T09:15:51.699625Z","shell.execute_reply":"2021-07-11T09:16:00.009509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !WANDB_MODE=\"dryrun\" python train.py --img 640 --batch 16 --epochs 3 --data coco128.yaml --weights yolov5s.pt --nosave --cache \n!WANDB_MODE=\"dryrun\" python train.py --img 512 --batch 24 --epochs 50 --data /kaggle/working/siim-cov19.yaml --hyp /kaggle/input/yolov5-1-yaml/hyp.scratch.yaml --weights yolov5x6.pt --cache","metadata":{"papermill":{"duration":19916.498298,"end_time":"2021-01-01T15:22:16.289734","exception":false,"start_time":"2021-01-01T09:50:19.791436","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-11T09:16:03.069033Z","iopub.execute_input":"2021-07-11T09:16:03.069481Z","iopub.status.idle":"2021-07-11T09:34:09.270998Z","shell.execute_reply.started":"2021-07-11T09:16:03.069436Z","shell.execute_reply":"2021-07-11T09:34:09.268931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion Matrix","metadata":{}},{"cell_type":"code","source":"# plt.figure(figsize=(30,15))\n# plt.axis('off')\n# plt.imshow(plt.imread('runs/train/exp/confusion_matrix.png'));","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-11T09:34:09.275954Z","iopub.execute_input":"2021-07-11T09:34:09.276293Z","iopub.status.idle":"2021-07-11T09:34:09.283606Z","shell.execute_reply.started":"2021-07-11T09:34:09.276263Z","shell.execute_reply":"2021-07-11T09:34:09.282750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{"papermill":{"duration":4.941983,"end_time":"2021-01-01T15:23:33.765831","exception":false,"start_time":"2021-01-01T15:23:28.823848","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!python detect.py --weights 'runs/train/exp2/weights/best.pt'\\\n--img 512\\\n--conf 0.1\\\n--iou 0.5\\\n--source /kaggle/working/siim-cov19/images/val\\\n--exist-ok","metadata":{"_kg_hide-output":true,"papermill":{"duration":10.763143,"end_time":"2021-01-01T15:23:49.800461","exception":false,"start_time":"2021-01-01T15:23:39.037318","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-11T10:01:46.078468Z","iopub.execute_input":"2021-07-11T10:01:46.079052Z","iopub.status.idle":"2021-07-11T10:03:04.007593Z","shell.execute_reply.started":"2021-07-11T10:01:46.078988Z","shell.execute_reply":"2021-07-11T10:03:04.006387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Plot","metadata":{"papermill":{"duration":5.225725,"end_time":"2021-01-01T15:24:00.706026","exception":false,"start_time":"2021-01-01T15:23:55.480301","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# from mpl_toolkits.axes_grid1 import ImageGrid\n# import numpy as np\n# import random\n# import cv2\n# from glob import glob\n# from tqdm import tqdm\n\n# files = glob('runs/detect/exp/*')\n# for _ in range(3):\n#     row = 4\n#     col = 4\n# #     grid_files = ','.join([random.choice(files) for _ in range(row*col)])\n#     grid_files = random.sample(files, k=row*col)\n#     images     = []\n#     for image_path in tqdm(grid_files):\n# #         print(image_path)\n#         img          = cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n#         images.append(img)\n# #     print(images)\n# #     fig = plt.figure(figsize=(col*5, row*5))\n#     ax = plt.figure(figsize=(4., 4.))\n# #     grid = ImageGrid(fig, 111,  # similar to subplot(111)\n# #                      nrows_ncols=(2, 2),  # creates 2x2 grid of axes\n# #                      axes_pad=0.1,  # pad between axes in inch.\n# #                      )\n\n#     for im in zip(images):\n#         # Iterating over the grid returns the Axes.\n#         ax.imshow(im)\n#         ax.set_xticks([])\n#         ax.set_yticks([])\n#     plt.show()","metadata":{"papermill":{"duration":5.31015,"end_time":"2021-01-01T15:24:11.211904","exception":false,"start_time":"2021-01-01T15:24:05.901754","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree('/kaggle/working/siim-cov19')\nshutil.rmtree('runs/detect')\nfor file in (glob('runs/train/exp2/**/*.png', recursive = True)+glob('runs/train/exp2/**/*.jpg', recursive = True)):\n    os.remove(file)","metadata":{"papermill":{"duration":5.709202,"end_time":"2021-01-01T15:24:22.413173","exception":false,"start_time":"2021-01-01T15:24:16.703971","status":"completed"},"tags":[],"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}