{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade seaborn","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:22.90895Z","iopub.execute_input":"2021-05-29T13:49:22.909383Z","iopub.status.idle":"2021-05-29T13:49:29.816882Z","shell.execute_reply.started":"2021-05-29T13:49:22.909308Z","shell.execute_reply":"2021-05-29T13:49:29.815549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns\nimport ast","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:29.825111Z","iopub.execute_input":"2021-05-29T13:49:29.827524Z","iopub.status.idle":"2021-05-29T13:49:30.621201Z","shell.execute_reply.started":"2021-05-29T13:49:29.827481Z","shell.execute_reply":"2021-05-29T13:49:30.620395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dim = 512 #512, 256, 'original'\nfold = 4","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.622883Z","iopub.execute_input":"2021-05-29T13:49:30.623244Z","iopub.status.idle":"2021-05-29T13:49:30.627195Z","shell.execute_reply.started":"2021-05-29T13:49:30.623207Z","shell.execute_reply":"2021-05-29T13:49:30.626116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\", index_col = \"id\")\nid_lst = [ii[:-6] for ii in train_df.index.tolist()]\ntrain_df.index = id_lst\ntrain_df.index.name = \"id\"\nmeta_jpg = pd.read_csv(f\"../input/siimcovid19converttojpg{dim}px/meta.csv\", index_col = \"image_id\")\ntrain_meta = meta_jpg[meta_jpg[\"split\"] == \"train\"]\ntrain_df = pd.concat([train_df, train_meta], axis=1, join=\"inner\")\ntrain_df.index.name = \"image_id\"\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.629119Z","iopub.execute_input":"2021-05-29T13:49:30.629702Z","iopub.status.idle":"2021-05-29T13:49:30.734557Z","shell.execute_reply.started":"2021-05-29T13:49:30.629664Z","shell.execute_reply":"2021-05-29T13:49:30.733773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.dropna(subset=[\"boxes\"])\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.735808Z","iopub.execute_input":"2021-05-29T13:49:30.736146Z","iopub.status.idle":"2021-05-29T13:49:30.762859Z","shell.execute_reply.started":"2021-05-29T13:49:30.736111Z","shell.execute_reply":"2021-05-29T13:49:30.761951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.reset_index(drop=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.764221Z","iopub.execute_input":"2021-05-29T13:49:30.764778Z","iopub.status.idle":"2021-05-29T13:49:30.770283Z","shell.execute_reply.started":"2021-05-29T13:49:30.764741Z","shell.execute_reply":"2021-05-29T13:49:30.769491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['image_path'] = f'/kaggle/input/siimcovid19converttojpg{dim}px/train/'+train_df.image_id+('.jpg' if dim!='original' else '.png')\n#../input/siimcovid19converttojpg1024px/train","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.771619Z","iopub.execute_input":"2021-05-29T13:49:30.772356Z","iopub.status.idle":"2021-05-29T13:49:30.782416Z","shell.execute_reply.started":"2021-05-29T13:49:30.772317Z","shell.execute_reply":"2021-05-29T13:49:30.781566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.785043Z","iopub.execute_input":"2021-05-29T13:49:30.785726Z","iopub.status.idle":"2021-05-29T13:49:30.799029Z","shell.execute_reply.started":"2021-05-29T13:49:30.785689Z","shell.execute_reply":"2021-05-29T13:49:30.798115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = ['opacity']\nclasses","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.800796Z","iopub.execute_input":"2021-05-29T13:49:30.801197Z","iopub.status.idle":"2021-05-29T13:49:30.806407Z","shell.execute_reply.started":"2021-05-29T13:49:30.80115Z","shell.execute_reply":"2021-05-29T13:49:30.805435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gkf  = GroupKFold(n_splits = 5)\ntrain_df['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(train_df, groups = train_df.StudyInstanceUID.tolist())):\n    train_df.loc[val_idx, 'fold'] = fold\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.807914Z","iopub.execute_input":"2021-05-29T13:49:30.808605Z","iopub.status.idle":"2021-05-29T13:49:30.852112Z","shell.execute_reply.started":"2021-05-29T13:49:30.808566Z","shell.execute_reply":"2021-05-29T13:49:30.851131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nval_files   = []\nval_files += list(train_df[train_df.fold==fold].image_path.unique())\ntrain_files += list(train_df[train_df.fold!=fold].image_path.unique())\nlen(train_files), len(val_files)","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.853551Z","iopub.execute_input":"2021-05-29T13:49:30.853899Z","iopub.status.idle":"2021-05-29T13:49:30.866463Z","shell.execute_reply.started":"2021-05-29T13:49:30.853864Z","shell.execute_reply":"2021-05-29T13:49:30.865348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df[\"StudyInstanceUID\"] == \"0572ef0d0c1a\"]","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.868214Z","iopub.execute_input":"2021-05-29T13:49:30.868599Z","iopub.status.idle":"2021-05-29T13:49:30.880172Z","shell.execute_reply.started":"2021-05-29T13:49:30.868558Z","shell.execute_reply":"2021-05-29T13:49:30.879129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.path.exists(\"/kaggle/input/siim-yolov5-labels/labels/0572ef0d0c1a.txt\")","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.881862Z","iopub.execute_input":"2021-05-29T13:49:30.882351Z","iopub.status.idle":"2021-05-29T13:49:30.896411Z","shell.execute_reply.started":"2021-05-29T13:49:30.882313Z","shell.execute_reply":"2021-05-29T13:49:30.895689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.set_index(\"image_id\").loc[\"0572ef0d0c1a\", :]","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.897572Z","iopub.execute_input":"2021-05-29T13:49:30.89809Z","iopub.status.idle":"2021-05-29T13:49:30.909918Z","shell.execute_reply.started":"2021-05-29T13:49:30.898052Z","shell.execute_reply":"2021-05-29T13:49:30.90856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/kaggle/working/siim/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/siim/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/siim/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/siim/images/val', exist_ok = True)\nlabel_dir = '/kaggle/input/siim-yolov5-labels/labels/'\nfor file in tqdm(train_files):\n    shutil.copy(file, '/kaggle/working/siim/images/train')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/siim/labels/train')\n    \nfor file in tqdm(val_files):\n    shutil.copy(file, '/kaggle/working/siim/images/val')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/siim/labels/val')","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:49:30.911132Z","iopub.execute_input":"2021-05-29T13:49:30.911484Z","iopub.status.idle":"2021-05-29T13:50:28.121725Z","shell.execute_reply.started":"2021-05-29T13:49:30.911448Z","shell.execute_reply":"2021-05-29T13:50:28.120965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport yaml\n\ncwd = '/kaggle/working/'\n\nwith open(join( cwd , 'train.txt'), 'w') as f:\n    for path in glob('/kaggle/working/siim/images/train/*'):\n        f.write(path+'\\n')\n            \nwith open(join( cwd , 'val.txt'), 'w') as f:\n    for path in glob('/kaggle/working/siim/images/val/*'):\n        f.write(path+'\\n')\n\ndata = dict(\n    train =  join( cwd , 'train.txt') ,\n    val   =  join( cwd , 'val.txt' ),\n    nc    = 1,\n    names = classes\n    )\n\nwith open(join( cwd , 'siim.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(join( cwd , 'siim.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:50:28.122942Z","iopub.execute_input":"2021-05-29T13:50:28.123303Z","iopub.status.idle":"2021-05-29T13:50:28.16586Z","shell.execute_reply.started":"2021-05-29T13:50:28.123266Z","shell.execute_reply":"2021-05-29T13:50:28.165069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/ultralytics/yolov5\n# !git clone https://github.com/ultralytics/yolov5  # clone repo\n# %cd yolov5\nshutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5')\n# %pip install -qr requirements.txt # install dependencies\n\nimport torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:50:28.16695Z","iopub.execute_input":"2021-05-29T13:50:28.167431Z","iopub.status.idle":"2021-05-29T13:50:30.200165Z","shell.execute_reply.started":"2021-05-29T13:50:28.167396Z","shell.execute_reply":"2021-05-29T13:50:30.199225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --weights yolov5x.pt --img 640 --conf 0.25 --source data/images/\nImage(filename='runs/detect/exp/zidane.jpg', width=600)","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:50:30.201486Z","iopub.execute_input":"2021-05-29T13:50:30.201896Z","iopub.status.idle":"2021-05-29T13:50:46.352432Z","shell.execute_reply.started":"2021-05-29T13:50:30.201854Z","shell.execute_reply":"2021-05-29T13:50:46.349298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !WANDB_MODE=\"dryrun\" python train.py --img 640 --batch 16 --epochs 3 --data coco128.yaml --weights yolov5s.pt --nosave --cache \n!WANDB_MODE=\"dryrun\" python train.py --img $dim --batch 16 --epochs 20 --data /kaggle/working/siim.yaml --weights yolov5x.pt --cache","metadata":{"execution":{"iopub.status.busy":"2021-05-29T13:50:46.353634Z","iopub.execute_input":"2021-05-29T13:50:46.353989Z","iopub.status.idle":"2021-05-29T14:01:34.92643Z","shell.execute_reply.started":"2021-05-29T13:50:46.35395Z","shell.execute_reply":"2021-05-29T14:01:34.925323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (20,20))\nplt.axis('off')\nplt.imshow(plt.imread('runs/train/exp/labels_correlogram.jpg'));","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (20,20))\nplt.axis('off')\nplt.imshow(plt.imread('runs/train/exp/labels.jpg'));","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch0.jpg'))\n\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch1.jpg'))\n\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch2.jpg'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(3, 2, figsize = (2*5,3*5), constrained_layout = True)\nfor row in range(3):\n    ax[row][0].imshow(plt.imread(f'runs/train/exp/test_batch{row}_labels.jpg'))\n    ax[row][0].set_xticks([])\n    ax[row][0].set_yticks([])\n    ax[row][0].set_title(f'runs/train/exp/test_batch{row}_labels.jpg', fontsize = 12)\n    \n    ax[row][1].imshow(plt.imread(f'runs/train/exp/test_batch{row}_pred.jpg'))\n    ax[row][1].set_xticks([])\n    ax[row][1].set_yticks([])\n    ax[row][1].set_title(f'runs/train/exp/test_batch{row}_pred.jpg', fontsize = 12)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,15))\nplt.axis('off')\nplt.imshow(plt.imread('runs/train/exp/results.png'));","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nplt.figure(figsize=(30,15))\nplt.axis('off')\nplt.imshow(plt.imread('runs/train/exp/confusion_matrix.png'));","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --weights 'runs/train/exp/weights/best.pt'\\\n--img $dim\\\n--conf 0.15\\\n--iou 0.5\\\n--source /kaggle/working/siim/images/val\\\n--exist-ok","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nimport numpy as np\nimport random\nimport cv2\nfrom glob import glob\nfrom tqdm import tqdm\n\nfiles = glob('runs/detect/exp/*')\nfor _ in range(3):\n    row = 4\n    col = 4\n    grid_files = random.sample(files, row*col)\n    images     = []\n    for image_path in tqdm(grid_files):\n        img          = cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n        images.append(img)\n\n    fig = plt.figure(figsize=(col*5, row*5))\n    grid = ImageGrid(fig, 111,  # similar to subplot(111)\n                     nrows_ncols=(col, row),  # creates 2x2 grid of axes\n                     axes_pad=0.05,  # pad between axes in inch.\n                     )\n\n    for ax, im in zip(grid, images):\n        # Iterating over the grid returns the Axes.\n        ax.imshow(im)\n        ax.set_xticks([])\n        ax.set_yticks([])\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nshutil.rmtree('/kaggle/working/siim')\nshutil.rmtree('runs/detect')\nfor file in (glob('runs/train/exp/**/*.png', recursive = True)+glob('runs/train/exp/**/*.jpg', recursive = True)):\n    os.remove(file)","metadata":{},"execution_count":null,"outputs":[]}]}