{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # Download YOLOv5\n# !git clone https://github.com/ultralytics/yolov5  # clone repo\n# %cd yolov5\n# # Install dependencies\n# %pip install -qr requirements.txt  # install dependencies","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:24.580473Z","iopub.execute_input":"2021-07-27T16:06:24.580856Z","iopub.status.idle":"2021-07-27T16:06:24.586700Z","shell.execute_reply.started":"2021-07-27T16:06:24.580819Z","shell.execute_reply":"2021-07-27T16:06:24.585199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd ../","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:24.592943Z","iopub.execute_input":"2021-07-27T16:06:24.593558Z","iopub.status.idle":"2021-07-27T16:06:24.600191Z","shell.execute_reply.started":"2021-07-27T16:06:24.593499Z","shell.execute_reply":"2021-07-27T16:06:24.598928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade seaborn","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:24.603347Z","iopub.execute_input":"2021-07-27T16:06:24.603873Z","iopub.status.idle":"2021-07-27T16:06:35.844058Z","shell.execute_reply.started":"2021-07-27T16:06:24.603824Z","shell.execute_reply":"2021-07-27T16:06:35.842952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns","metadata":{"papermill":{"duration":0.926929,"end_time":"2021-01-01T09:44:54.403588","exception":false,"start_time":"2021-01-01T09:44:53.476659","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T16:06:35.847104Z","iopub.execute_input":"2021-07-27T16:06:35.847551Z","iopub.status.idle":"2021-07-27T16:06:36.950398Z","shell.execute_reply.started":"2021-07-27T16:06:35.847503Z","shell.execute_reply":"2021-07-27T16:06:36.949377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 0","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:36.952895Z","iopub.execute_input":"2021-07-27T16:06:36.953254Z","iopub.status.idle":"2021-07-27T16:06:36.960287Z","shell.execute_reply.started":"2021-07-27T16:06:36.953223Z","shell.execute_reply":"2021-07-27T16:06:36.958503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f'../input/siim-covid19-detection/train_image_level.csv')\ntrain_df.head()","metadata":{"papermill":{"duration":0.262045,"end_time":"2021-01-01T09:44:54.691965","exception":false,"start_time":"2021-01-01T09:44:54.42992","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T16:06:36.961962Z","iopub.execute_input":"2021-07-27T16:06:36.962505Z","iopub.status.idle":"2021-07-27T16:06:37.036487Z","shell.execute_reply.started":"2021-07-27T16:06:36.962455Z","shell.execute_reply":"2021-07-27T16:06:37.035601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.id = train_df.id.str[:-6]\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.039966Z","iopub.execute_input":"2021-07-27T16:06:37.040332Z","iopub.status.idle":"2021-07-27T16:06:37.062867Z","shell.execute_reply.started":"2021-07-27T16:06:37.040299Z","shell.execute_reply":"2021-07-27T16:06:37.061964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicates = ['11f46d9df786',\n 'ba57c39bc15d',\n 'bd4b6cfeddee',\n '82c84d5079b7',\n 'a27a126be5c1',\n '3d12cb6aad8b',\n 'b814a1e29517',\n '09236f094647',\n '36b995a9575c',\n '9cdc2af4fbb0',\n '779f0040d1b2',\n 'ef6e312ca719',\n 'b97c6b32105e',\n '93301812b0e7',\n '4c414b793562',\n 'e6cc65d9de1d',\n '9fcbe25a88e0',\n 'f228350bd770',\n '59bd5a00b33e',\n 'fa5e3f933004',\n 'a8e019dcf085',\n 'd205dc170ccc',\n 'a5a364383f34',\n '9939f63af4ff',\n '646a6a0a9bf2',\n '581fd56bc4bf',\n '629c9a257b43',\n '2ebe16a053fe',\n '173c23887f9b',\n 'ea2117b53323',\n '57bf106a9e2e',\n 'b989acc3c046',\n '12fe8eadded1',\n '7f9c98b88aec',\n 'c00c7e8b673c',\n 'a5bbd30ed109',\n 'b622f84403da',\n 'ba375ad92c5f',\n 'b0fd33e251f5',\n 'c6e92e59a0ae',\n 'ef690fe6f288',\n '6970c59656c9',\n 'ab55abb953d1',\n 'd540d6c2a425',\n '262dc03560d6',\n 'fc44ab5de22c',\n 'ce51b397b1a6',\n '6e9fad584bff',\n 'a2429a60f3ea',\n 'b0866caa201a',\n '7e4a4ef3ff9f',\n '3577ee4f26c4',\n '6f54e9cbd180',\n '57faddfc733e',\n '6a93346150a4',\n '611348a721f7',\n 'c4b2940d2334',\n 'e007bc90c441',\n '3cde0ddf01c7',\n '296aaf716ae0',\n 'b7eb26ddf5a3',\n 'e738c549fe8e',\n '39d52f244db3',\n 'a9704ce54780',\n '1701be774ecb',\n '221ff464c9fc',\n 'e3f8e3a3baa3',\n 'be65d1a22de5',\n '103bd9ed3929',\n '9108cdfd43dc',\n '3d2224cb6bb7',\n 'c4b68b29a072',\n '6d36ffbc7864',\n '35da4a5a0201',\n '341787fb81c7',\n '6ff218d10741',\n '68fcf433c6fe',\n '25400558ba1e',\n '149356f04849',\n '39cd89772e5e',\n '99f3642f50f5',\n 'c72e26e9ecae',\n '4b310bbd3cb1',\n 'f208dc529d16',\n '4024e564722b',\n 'cc708f2aaffe',\n 'dcb374d8387e',\n '7f1924880cf8',\n '961f9c368664',\n '1ea01196514a',\n '725328925e0f',\n '35ea2c9841c0',\n '72e5fbd61953',\n '75b52bec817f',\n 'd27bca58e388',\n '44d7b2e7b0f9',\n '830063223a31',\n '0b020a7aff0a',\n '9d8ccbfb99b7',\n '082d3af6ecc0',\n 'eef757610245',\n 'b77430575510',\n '321c93d6503f',\n '0a990c89256a',\n 'c9a028b48504',\n 'c636ac67c19a',\n '76c66ee8e58d',\n 'cdd9e3aaf45a',\n '9a9d89ca4aa0',\n '8b55ea197c78',\n '30b18db28900',\n 'a57bf6dd6119',\n 'e4a78ff8fa73',\n 'e01d23a04187',\n '66dabc6f972d',\n '4cebe94f1fa3',\n '6792b9b0329d',\n 'ddff141123c1',\n '550f057ee0b0',\n '0102b5cac730',\n 'e7e072c73e1d',\n '00e3a7e91a34',\n '8cbf85033da8',\n '6f749e2783e1',\n '4b2f144a1a8b',\n '783b1efb5675',\n 'e96133d06736',\n '869476b0763a',\n 'e496735126c2',\n '00c1515729a8',\n '9a335ce0bca4',\n '9b1de1c45491',\n 'd9456aadecbe',\n '51d04316f2c9',\n 'fb1e24bf6698',\n 'b3ffe59e37c7',\n '16bd0ff2f0cd',\n '8d4b3609ed92',\n 'a174a4f6c807',\n '2a7a456d367e',\n '1b92142f4362',\n 'bfefe70b56ca',\n 'e5fc2fa92e65',\n '8a38c00672d5',\n 'a94171e98807',\n '61d0f2ca0ddd',\n '0bd6cd815ba9',\n '1de0ad5314cc',\n 'e897ef5c203c',\n '0acec7010081',\n '3b6ad60071d4',\n 'b61f3493c551',\n '267a250932bc',\n 'a8013eaf16c1',\n 'e704aebd4ca5',\n '04cc2f7f4c4b',\n '3e7b2ffc97db',\n 'e0e1e1462e56',\n 'b5aa6189d142',\n 'cbf0a27f993e',\n 'ea2688741043',\n '2204baf77803',\n '56ff7c4e1f41',\n '156cb1f5c689',\n '05c063f5cef5',\n 'c3a09e8a600d',\n 'f577c6fb0f49',\n '8093df07a5d0',\n '6728e11290af',\n 'efc93a3917b6',\n 'c843d08b49b8',\n '717ea5155b46',\n '343d445230a1',\n '50f29acd37a6',\n 'ea516e218fe6',\n '582c442e440b',\n 'f9a92f1ba934',\n 'bdd3115879aa',\n '68ad4b624a6d',\n 'a323f1604ae2',\n 'bb52c80c7f45',\n '0b858129adb4',\n '26c6c25e9467',\n '91f72df96e13',\n '19c6397cdec9',\n '6861c5992717',\n 'deea6d6f81a5',\n '106632672b95',\n '6534a837497d',\n '684230477525',\n 'df3d32c8df06',\n '3f0879b045e7',\n '505693d77a18',\n 'a39667fe9a81',\n '2c130ee08736',\n 'c9a9e5e120bd',\n 'bb36f3d3ac71',\n '1c96d9b08487',\n 'daa9bddc5fe6',\n 'e73f2cc047ad',\n 'e60a47d36e62',\n '6366191bfe49',\n 'd180fed57716',\n '866e3622cb24',\n '55839276a9c9',\n 'fa3a7fe547ca',\n '59bc532be971',\n '424118f1afc8',\n '6ddf030ec946',\n '3d0bac303fc8',\n 'f689d30c2f86',\n '8c7bdccb65ed',\n 'df4f1240317e',\n '25ff5d625b35',\n '75f207c4945d',\n '35e398a5a431',\n '5d395fff6278',\n 'e3e2f20e0264',\n '99e8636292c9',\n 'de94a43e947f',\n 'a5beac3d9fb4',\n 'd19ed8d811ad',\n 'bf2417586cf2',\n '72cf260ddf4c',\n 'f5451a98d684',\n 'aad814e52627',\n '32222cc776a2',\n '608d574388ba',\n '9872a8a48f23',\n 'f7f759f7a4f1',\n '88e7702b49c4',\n 'dfd9078a4c6d',\n 'd4131bd4e64a',\n '1a0e966f2d57',\n '9f50edab452b',\n '4cf41c062d0c',\n '13bf9d719ad1',\n '0eb641cb0dcd',\n 'a2ee4b862182',\n '61e618dc7f9a',\n '467f412db5c4',\n '990216aa9f08',\n '3d047361508c',\n '08acae0bf785',\n '0dd32589bdc3',\n '9f180629878a',\n '6e5946091b8a',\n 'f3ce7a4cfd20',\n 'f85fe770048a',\n 'cf0cdff4ef2e',\n 'df2bb22fa871',\n '0d4d6acc9ed3',\n '45ff2dfff073',\n 'd54f6204b044',\n 'eea3a910fa9e',\n '3d19da0d4ee6',\n 'c3510a436bff',\n '4cbc17936e7d',\n 'b121806162c3',\n 'e5380eb2d925',\n '2f973757eaa4',\n 'b5a415f70aa9',\n 'cc69791740bb',\n 'a1fa5f79671d',\n 'dbc311ed04b5',\n '744427c7dff8',\n '49664f078f0e',\n 'bee62c601ae9',\n 'e441b1fb74ee',\n '8df453b7557c',\n '7626c521cad7',\n '33a72bfb5ddb',\n '56ed68c28819',\n '04a0b90d7875',\n '08932dca1447',\n 'daae293a8b40',\n 'c0400d0e413d',\n 'ddb051c1233b',\n 'f13f9ace0a9a',\n '9c0633ef606b']","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.066272Z","iopub.execute_input":"2021-07-27T16:06:37.066563Z","iopub.status.idle":"2021-07-27T16:06:37.097920Z","shell.execute_reply.started":"2021-07-27T16:06:37.066530Z","shell.execute_reply":"2021-07-27T16:06:37.096377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(duplicates)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.099732Z","iopub.execute_input":"2021-07-27T16:06:37.100224Z","iopub.status.idle":"2021-07-27T16:06:37.117449Z","shell.execute_reply.started":"2021-07-27T16:06:37.100178Z","shell.execute_reply":"2021-07-27T16:06:37.116138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_files = list(train_df.id.values)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.119913Z","iopub.execute_input":"2021-07-27T16:06:37.120430Z","iopub.status.idle":"2021-07-27T16:06:37.129364Z","shell.execute_reply.started":"2021-07-27T16:06:37.120383Z","shell.execute_reply":"2021-07-27T16:06:37.128467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in duplicates:\n    all_files.remove(i)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.130890Z","iopub.execute_input":"2021-07-27T16:06:37.131688Z","iopub.status.idle":"2021-07-27T16:06:37.169189Z","shell.execute_reply.started":"2021-07-27T16:06:37.131595Z","shell.execute_reply":"2021-07-27T16:06:37.168314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_files)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.171247Z","iopub.execute_input":"2021-07-27T16:06:37.171837Z","iopub.status.idle":"2021-07-27T16:06:37.181462Z","shell.execute_reply.started":"2021-07-27T16:06:37.171793Z","shell.execute_reply":"2021-07-27T16:06:37.180185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df = pd.DataFrame({'id':all_files})","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.189912Z","iopub.execute_input":"2021-07-27T16:06:37.190264Z","iopub.status.idle":"2021-07-27T16:06:37.196283Z","shell.execute_reply.started":"2021-07-27T16:06:37.190227Z","shell.execute_reply":"2021-07-27T16:06:37.194843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.199731Z","iopub.execute_input":"2021-07-27T16:06:37.200546Z","iopub.status.idle":"2021-07-27T16:06:37.218916Z","shell.execute_reply.started":"2021-07-27T16:06:37.200336Z","shell.execute_reply":"2021-07-27T16:06:37.217500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.merge(final_df, train_df, how='left', on='id')","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.221416Z","iopub.execute_input":"2021-07-27T16:06:37.222162Z","iopub.status.idle":"2021-07-27T16:06:37.244708Z","shell.execute_reply.started":"2021-07-27T16:06:37.222111Z","shell.execute_reply":"2021-07-27T16:06:37.243589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.245943Z","iopub.execute_input":"2021-07-27T16:06:37.246324Z","iopub.status.idle":"2021-07-27T16:06:37.254799Z","shell.execute_reply.started":"2021-07-27T16:06:37.246281Z","shell.execute_reply":"2021-07-27T16:06:37.253062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.257358Z","iopub.execute_input":"2021-07-27T16:06:37.258059Z","iopub.status.idle":"2021-07-27T16:06:37.276291Z","shell.execute_reply.started":"2021-07-27T16:06:37.257990Z","shell.execute_reply":"2021-07-27T16:06:37.274810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv'); print(study_df.shape)\nstudy_df['StudyInstanceUID'] = study_df['id'].apply(lambda x: x.replace('_study', ''))\ndel study_df['id']","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.278424Z","iopub.execute_input":"2021-07-27T16:06:37.278976Z","iopub.status.idle":"2021-07-27T16:06:37.309468Z","shell.execute_reply.started":"2021-07-27T16:06:37.278930Z","shell.execute_reply":"2021-07-27T16:06:37.308246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def hot_to_sparse(row):\n    return(row.index[row.apply(lambda x: x==1)][0])\nstudy_df['diagnosis'] = study_df.apply(lambda row:hot_to_sparse(row), axis=1)\ncls = {\n    'Typical Appearance':1,                    \n    'Negative for Pneumonia':0,                \n    'Indeterminate Appearance':2,                     \n    'Atypical Appearance':3,    \n}\nstudy_df['sparse_gt'] = study_df.diagnosis.map(cls)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:37.311055Z","iopub.execute_input":"2021-07-27T16:06:37.311475Z","iopub.status.idle":"2021-07-27T16:06:38.944974Z","shell.execute_reply.started":"2021-07-27T16:06:37.311430Z","shell.execute_reply":"2021-07-27T16:06:38.944109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:38.946711Z","iopub.execute_input":"2021-07-27T16:06:38.947485Z","iopub.status.idle":"2021-07-27T16:06:38.967588Z","shell.execute_reply.started":"2021-07-27T16:06:38.947435Z","shell.execute_reply":"2021-07-27T16:06:38.966119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_df.merge(study_df, on='StudyInstanceUID')\ntrain['id'] = train['id'].apply(lambda x: x.replace('_image', ''))","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:38.969693Z","iopub.execute_input":"2021-07-27T16:06:38.970408Z","iopub.status.idle":"2021-07-27T16:06:38.996500Z","shell.execute_reply.started":"2021-07-27T16:06:38.970362Z","shell.execute_reply":"2021-07-27T16:06:38.995314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:38.998216Z","iopub.execute_input":"2021-07-27T16:06:38.998748Z","iopub.status.idle":"2021-07-27T16:06:39.020028Z","shell.execute_reply.started":"2021-07-27T16:06:38.998704Z","shell.execute_reply":"2021-07-27T16:06:39.017570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:39.022173Z","iopub.execute_input":"2021-07-27T16:06:39.023075Z","iopub.status.idle":"2021-07-27T16:06:39.032094Z","shell.execute_reply.started":"2021-07-27T16:06:39.022511Z","shell.execute_reply":"2021-07-27T16:06:39.030419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:39.033822Z","iopub.execute_input":"2021-07-27T16:06:39.034603Z","iopub.status.idle":"2021-07-27T16:06:39.041397Z","shell.execute_reply.started":"2021-07-27T16:06:39.034541Z","shell.execute_reply":"2021-07-27T16:06:39.040201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:39.043218Z","iopub.execute_input":"2021-07-27T16:06:39.043864Z","iopub.status.idle":"2021-07-27T16:06:39.069289Z","shell.execute_reply.started":"2021-07-27T16:06:39.043805Z","shell.execute_reply":"2021-07-27T16:06:39.068106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls '../input'","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:39.070844Z","iopub.execute_input":"2021-07-27T16:06:39.071326Z","iopub.status.idle":"2021-07-27T16:06:39.848903Z","shell.execute_reply.started":"2021-07-27T16:06:39.071278Z","shell.execute_reply":"2021-07-27T16:06:39.847977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['image_path'] = f'/kaggle/input/covid19-detection-890pxpng-study/train/' + train.id + '.png'\ntrain.head()","metadata":{"papermill":{"duration":0.086788,"end_time":"2021-01-01T09:44:54.805857","exception":false,"start_time":"2021-01-01T09:44:54.719069","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T16:06:39.853102Z","iopub.execute_input":"2021-07-27T16:06:39.853404Z","iopub.status.idle":"2021-07-27T16:06:39.896338Z","shell.execute_reply.started":"2021-07-27T16:06:39.853370Z","shell.execute_reply":"2021-07-27T16:06:39.895447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nskf = StratifiedKFold(n_splits=6, shuffle=True, random_state=9)\nfor index, (train_index, val_index) in enumerate(skf.split(X=train.index, y=train.sparse_gt)):\n    train.loc[val_index, 'fold'] = index\n    \nprint(train.groupby(['fold', train.sparse_gt]).size())","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:39.899749Z","iopub.execute_input":"2021-07-27T16:06:39.900086Z","iopub.status.idle":"2021-07-27T16:06:39.929453Z","shell.execute_reply.started":"2021-07-27T16:06:39.900054Z","shell.execute_reply":"2021-07-27T16:06:39.928642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = ['0. opacity']","metadata":{"papermill":{"duration":0.050418,"end_time":"2021-01-01T09:45:03.002944","exception":false,"start_time":"2021-01-01T09:45:02.952526","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T16:06:39.931561Z","iopub.execute_input":"2021-07-27T16:06:39.931866Z","iopub.status.idle":"2021-07-27T16:06:39.940851Z","shell.execute_reply.started":"2021-07-27T16:06:39.931836Z","shell.execute_reply":"2021-07-27T16:06:39.939892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n# X_train, X_valid = train_test_split(train_df, test_size=0.1, stratify=train_df['sparse_gt'], random_state=18)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:39.944898Z","iopub.execute_input":"2021-07-27T16:06:39.945252Z","iopub.status.idle":"2021-07-27T16:06:39.952385Z","shell.execute_reply.started":"2021-07-27T16:06:39.945223Z","shell.execute_reply":"2021-07-27T16:06:39.950974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train[train['fold'] != 0]\nX_valid = train[train['fold'] == 0]","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:39.955132Z","iopub.execute_input":"2021-07-27T16:06:39.955459Z","iopub.status.idle":"2021-07-27T16:06:39.976054Z","shell.execute_reply.started":"2021-07-27T16:06:39.955427Z","shell.execute_reply":"2021-07-27T16:06:39.974410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.sparse_gt.value_counts(), X_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:39.982603Z","iopub.execute_input":"2021-07-27T16:06:39.984644Z","iopub.status.idle":"2021-07-27T16:06:40.006453Z","shell.execute_reply.started":"2021-07-27T16:06:39.984594Z","shell.execute_reply":"2021-07-27T16:06:40.004138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid.sparse_gt.value_counts(), X_valid.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:06:40.009970Z","iopub.execute_input":"2021-07-27T16:06:40.010536Z","iopub.status.idle":"2021-07-27T16:06:40.028516Z","shell.execute_reply.started":"2021-07-27T16:06:40.010492Z","shell.execute_reply":"2021-07-27T16:06:40.021192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nval_files   = []\ntrain_files += list(X_train.image_path.unique())\nval_files += list(X_valid.image_path.unique())\nlen(train_files), len(val_files)","metadata":{"papermill":{"duration":0.086817,"end_time":"2021-01-01T09:47:56.443789","exception":false,"start_time":"2021-01-01T09:47:56.356972","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T16:06:40.031457Z","iopub.execute_input":"2021-07-27T16:06:40.032308Z","iopub.status.idle":"2021-07-27T16:06:40.057922Z","shell.execute_reply.started":"2021-07-27T16:06:40.032257Z","shell.execute_reply":"2021-07-27T16:06:40.056395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Copying Files","metadata":{"papermill":{"duration":0.083752,"end_time":"2021-01-01T09:47:56.584924","exception":false,"start_time":"2021-01-01T09:47:56.501172","status":"completed"},"tags":[]}},{"cell_type":"code","source":"os.makedirs('/kaggle/working/siim-cov19/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/images/val', exist_ok = True)\nlabel_dir = '/kaggle/input/siim-covid-19-yolo-txt'\nfor file in tqdm(train_files):\n    shutil.copy(file, '/kaggle/working/siim-cov19/images/train')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/siim-cov19/labels/train')\n    \nfor file in tqdm(val_files):\n    shutil.copy(file, '/kaggle/working/siim-cov19/images/val')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/siim-cov19/labels/val')","metadata":{"papermill":{"duration":124.654777,"end_time":"2021-01-01T09:50:01.331041","exception":false,"start_time":"2021-01-01T09:47:56.676264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T16:06:40.061740Z","iopub.execute_input":"2021-07-27T16:06:40.062524Z","iopub.status.idle":"2021-07-27T16:09:30.179938Z","shell.execute_reply.started":"2021-07-27T16:06:40.062463Z","shell.execute_reply":"2021-07-27T16:09:30.178800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Class Name","metadata":{"papermill":{"duration":0.068822,"end_time":"2021-01-01T09:50:01.458337","exception":false,"start_time":"2021-01-01T09:50:01.389515","status":"completed"},"tags":[]}},{"cell_type":"code","source":"classes = ['0. opacity']","metadata":{"papermill":{"duration":0.082234,"end_time":"2021-01-01T09:50:01.601574","exception":false,"start_time":"2021-01-01T09:50:01.51934","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T16:09:30.181624Z","iopub.execute_input":"2021-07-27T16:09:30.182336Z","iopub.status.idle":"2021-07-27T16:09:30.187792Z","shell.execute_reply.started":"2021-07-27T16:09:30.182287Z","shell.execute_reply":"2021-07-27T16:09:30.186516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# YOLOv5 Stuff","metadata":{"papermill":{"duration":0.055699,"end_time":"2021-01-01T09:50:01.82747","exception":false,"start_time":"2021-01-01T09:50:01.771771","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport yaml\n\ncwd = '/kaggle/working/'\n\nwith open(join( cwd , 'train.txt'), 'w') as f:\n    for path in glob('/kaggle/working/siim-cov19/images/train/*'):\n        f.write(path+'\\n')\n            \nwith open(join( cwd , 'val.txt'), 'w') as f:\n    for path in glob('/kaggle/working/siim-cov19/images/val/*'):\n        f.write(path+'\\n')\n\ndata = dict(\n    train =  join( cwd , 'train.txt') ,\n    val   =  join( cwd , 'val.txt' ),\n    nc    = 1,\n    names = classes\n    )\n\nwith open(join( cwd , 'siim-cov19.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(join( cwd , 'siim-cov19.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"papermill":{"duration":0.113001,"end_time":"2021-01-01T09:50:01.996448","exception":false,"start_time":"2021-01-01T09:50:01.883447","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T16:09:30.190389Z","iopub.execute_input":"2021-07-27T16:09:30.190852Z","iopub.status.idle":"2021-07-27T16:09:30.277063Z","shell.execute_reply.started":"2021-07-27T16:09:30.190793Z","shell.execute_reply":"2021-07-27T16:09:30.275789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls '../input'","metadata":{"execution":{"iopub.status.busy":"2021-07-27T16:09:30.278576Z","iopub.execute_input":"2021-07-27T16:09:30.279112Z","iopub.status.idle":"2021-07-27T16:09:31.032567Z","shell.execute_reply.started":"2021-07-27T16:09:30.279040Z","shell.execute_reply":"2021-07-27T16:09:31.031536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/ultralytics/yolov5\n# !git clone https://github.com/ultralytics/yolov5  # clone repo\n# %cd yolov5\nshutil.copytree('/kaggle/input/yolo-v31/yolov5-metric', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5')\n# %pip install -qr requirements.txt # install dependencies\n\nimport torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":6.702428,"end_time":"2021-01-01T09:50:08.784153","exception":false,"start_time":"2021-01-01T09:50:02.081725","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T16:09:31.034807Z","iopub.execute_input":"2021-07-27T16:09:31.035294Z","iopub.status.idle":"2021-07-27T16:09:33.098864Z","shell.execute_reply.started":"2021-07-27T16:09:31.035239Z","shell.execute_reply":"2021-07-27T16:09:33.097587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pretrained Checkpoints:\n\n| Model | AP<sup>val</sup> | AP<sup>test</sup> | AP<sub>50</sub> | Speed<sub>GPU</sub> | FPS<sub>GPU</sub> || params | FLOPS |\n|---------- |------ |------ |------ | -------- | ------| ------ |------  |  :------: |\n| [YOLOv5s](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | 37.0     | 37.0     | 56.2     | **2.4ms** | **416** || 7.5M   | 13.2B\n| [YOLOv5m](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | 44.3     | 44.3     | 63.2     | 3.4ms     | 294     || 21.8M  | 39.4B\n| [YOLOv5l](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | 47.7     | 47.7     | 66.5     | 4.4ms     | 227     || 47.8M  | 88.1B\n| [YOLOv5x](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | **49.2** | **49.2** | **67.7** | 6.9ms     | 145     || 89.0M  | 166.4B\n| | | | | | || |\n| [YOLOv5x](https://github.com/ultralytics/yolov5/releases/tag/v3.0) + TTA|**50.8**| **50.8** | **68.9** | 25.5ms    | 39      || 89.0M  | 354.3B\n| | | | | | || |\n| [YOLOv3-SPP](https://github.com/ultralytics/yolov5/releases/tag/v3.0) | 45.6     | 45.5     | 65.2     | 4.5ms     | 222     || 63.0M  | 118.0B","metadata":{"papermill":{"duration":0.064911,"end_time":"2021-01-01T09:50:19.435746","exception":false,"start_time":"2021-01-01T09:50:19.370835","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Selecting Models\nIn this notebok I'm using `v5s`. To select your prefered model just replace `--cfg models/yolov5s.yaml --weights yolov5s.pt` with the following command:\n* `v5s` : `--cfg models/yolov5s.yaml --weights yolov5s.pt`\n* `v5m` : `--cfg models/yolov5m.yaml --weights yolov5m.pt`\n* `v5l` : `--cfg models/yolov5l.yaml --weights yolov5l.pt`\n* `v5x` : `--cfg models/yolov5x.yaml --weights yolov5x.pt`","metadata":{"papermill":{"duration":0.064016,"end_time":"2021-01-01T09:50:19.564859","exception":false,"start_time":"2021-01-01T09:50:19.500843","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Train","metadata":{"papermill":{"duration":0.064553,"end_time":"2021-01-01T09:50:19.6938","exception":false,"start_time":"2021-01-01T09:50:19.629247","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!WANDB_MODE=\"dryrun\" python train.py --img 640 --batch 32 --epochs 35 --data /kaggle/working/siim-cov19.yaml --hyp /kaggle/input/yolo-yaml-txt-nv/hyp.scratch.yaml --weights yolov5m.pt --cache","metadata":{"papermill":{"duration":19916.498298,"end_time":"2021-01-01T15:22:16.289734","exception":false,"start_time":"2021-01-01T09:50:19.791436","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T16:09:33.100460Z","iopub.execute_input":"2021-07-27T16:09:33.101156Z","iopub.status.idle":"2021-07-27T17:44:07.839285Z","shell.execute_reply.started":"2021-07-27T16:09:33.101100Z","shell.execute_reply":"2021-07-27T17:44:07.837953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Distribution","metadata":{"papermill":{"duration":4.919442,"end_time":"2021-01-01T15:22:26.398681","exception":false,"start_time":"2021-01-01T15:22:21.479239","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# plt.figure(figsize = (20,20))\n# plt.axis('off')\n# plt.imshow(plt.imread('runs/train/exp/labels_correlogram.jpg'));","metadata":{"papermill":{"duration":6.511035,"end_time":"2021-01-01T15:22:37.753063","exception":false,"start_time":"2021-01-01T15:22:31.242028","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T17:44:07.843358Z","iopub.execute_input":"2021-07-27T17:44:07.843720Z","iopub.status.idle":"2021-07-27T17:44:07.850275Z","shell.execute_reply.started":"2021-07-27T17:44:07.843683Z","shell.execute_reply":"2021-07-27T17:44:07.849263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (20,20))\nplt.axis('off')\nplt.imshow(plt.imread('runs/train/exp/labels.jpg'));","metadata":{"papermill":{"duration":5.977042,"end_time":"2021-01-01T15:22:48.614609","exception":false,"start_time":"2021-01-01T15:22:42.637567","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T17:44:07.853567Z","iopub.execute_input":"2021-07-27T17:44:07.853868Z","iopub.status.idle":"2021-07-27T17:44:08.696721Z","shell.execute_reply.started":"2021-07-27T17:44:07.853836Z","shell.execute_reply":"2021-07-27T17:44:08.695206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Batch Image","metadata":{"papermill":{"duration":5.378338,"end_time":"2021-01-01T15:22:59.482837","exception":false,"start_time":"2021-01-01T15:22:54.104499","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch0.jpg'))\n\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch1.jpg'))\n\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch2.jpg'))","metadata":{"papermill":{"duration":7.317416,"end_time":"2021-01-01T15:23:11.777544","exception":false,"start_time":"2021-01-01T15:23:04.460128","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-27T17:44:08.698129Z","iopub.execute_input":"2021-07-27T17:44:08.698487Z","iopub.status.idle":"2021-07-27T17:44:10.600906Z","shell.execute_reply.started":"2021-07-27T17:44:08.698447Z","shell.execute_reply":"2021-07-27T17:44:10.599794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GT Vs Pred","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(3, 2, figsize = (2*5,3*5), constrained_layout = True)\nfor row in range(3):\n    ax[row][0].imshow(plt.imread(f'runs/train/exp/test_batch{row}_labels.jpg'))\n    ax[row][0].set_xticks([])\n    ax[row][0].set_yticks([])\n    ax[row][0].set_title(f'runs/train/exp/test_batch{row}_labels.jpg', fontsize = 12)\n    \n    ax[row][1].imshow(plt.imread(f'runs/train/exp/test_batch{row}_pred.jpg'))\n    ax[row][1].set_xticks([])\n    ax[row][1].set_yticks([])\n    ax[row][1].set_title(f'runs/train/exp/test_batch{row}_pred.jpg', fontsize = 12)","metadata":{"papermill":{"duration":6.453975,"end_time":"2021-01-01T15:23:23.514717","exception":false,"start_time":"2021-01-01T15:23:17.060742","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-27T17:44:10.603157Z","iopub.execute_input":"2021-07-27T17:44:10.603568Z","iopub.status.idle":"2021-07-27T17:44:12.403833Z","shell.execute_reply.started":"2021-07-27T17:44:10.603525Z","shell.execute_reply":"2021-07-27T17:44:12.402676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (Loss, Map) Vs Epoch","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30,15))\nplt.axis('off')\nplt.imshow(plt.imread('runs/train/exp/results.png'));","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-27T17:44:12.405543Z","iopub.execute_input":"2021-07-27T17:44:12.406540Z","iopub.status.idle":"2021-07-27T17:44:13.270759Z","shell.execute_reply.started":"2021-07-27T17:44:12.406481Z","shell.execute_reply":"2021-07-27T17:44:13.264859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion Matrix","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30,15))\nplt.axis('off')\nplt.imshow(plt.imread('runs/train/exp/confusion_matrix.png'));","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-27T17:44:13.272552Z","iopub.execute_input":"2021-07-27T17:44:13.272973Z","iopub.status.idle":"2021-07-27T17:44:14.495443Z","shell.execute_reply.started":"2021-07-27T17:44:13.272931Z","shell.execute_reply":"2021-07-27T17:44:14.493932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{"papermill":{"duration":4.941983,"end_time":"2021-01-01T15:23:33.765831","exception":false,"start_time":"2021-01-01T15:23:28.823848","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!python detect.py --weights 'runs/train/exp/weights/best.pt'\\\n--img 512\\\n--conf 0.1\\\n--iou 0.5\\\n--source /kaggle/working/siim-cov19/images/val\\\n--exist-ok","metadata":{"_kg_hide-output":true,"papermill":{"duration":10.763143,"end_time":"2021-01-01T15:23:49.800461","exception":false,"start_time":"2021-01-01T15:23:39.037318","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T17:44:14.497539Z","iopub.execute_input":"2021-07-27T17:44:14.498000Z","iopub.status.idle":"2021-07-27T17:45:38.125670Z","shell.execute_reply.started":"2021-07-27T17:44:14.497952Z","shell.execute_reply":"2021-07-27T17:45:38.124188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Plot","metadata":{"papermill":{"duration":5.225725,"end_time":"2021-01-01T15:24:00.706026","exception":false,"start_time":"2021-01-01T15:23:55.480301","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nimport numpy as np\nimport random\nimport cv2\nfrom glob import glob\nfrom tqdm import tqdm\n\nfiles = glob('runs/detect/exp/*')\nfor _ in range(3):\n    row = 4\n    col = 4\n    grid_files = random.sample(files, row*col)\n    images     = []\n    for image_path in tqdm(grid_files):\n        img          = cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n        images.append(img)\n\n    fig = plt.figure(figsize=(col*5, row*5))\n    grid = ImageGrid(fig, 111,  # similar to subplot(111)\n                     nrows_ncols=(col, row),  # creates 2x2 grid of axes\n                     axes_pad=0.05,  # pad between axes in inch.\n                     )\n\n    for ax, im in zip(grid, images):\n        # Iterating over the grid returns the Axes.\n        ax.imshow(im)\n        ax.set_xticks([])\n        ax.set_yticks([])\n    plt.show()","metadata":{"papermill":{"duration":5.31015,"end_time":"2021-01-01T15:24:11.211904","exception":false,"start_time":"2021-01-01T15:24:05.901754","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-27T17:45:38.127803Z","iopub.execute_input":"2021-07-27T17:45:38.128343Z","iopub.status.idle":"2021-07-27T17:45:48.195019Z","shell.execute_reply.started":"2021-07-27T17:45:38.128290Z","shell.execute_reply":"2021-07-27T17:45:48.193765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree('/kaggle/working/siim-cov19')\nshutil.rmtree('runs/detect')\nfor file in (glob('runs/train/exp/**/*.png', recursive = True)+glob('runs/train/exp/**/*.jpg', recursive = True)):\n    os.remove(file)","metadata":{"papermill":{"duration":5.709202,"end_time":"2021-01-01T15:24:22.413173","exception":false,"start_time":"2021-01-01T15:24:16.703971","status":"completed"},"tags":[],"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-27T17:45:48.196736Z","iopub.execute_input":"2021-07-27T17:45:48.197406Z","iopub.status.idle":"2021-07-27T17:45:48.875234Z","shell.execute_reply.started":"2021-07-27T17:45:48.197362Z","shell.execute_reply":"2021-07-27T17:45:48.873914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}