{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # Download YOLOv5\n# !git clone https://github.com/ultralytics/yolov5  # clone repo\n# %cd yolov5\n# # Install dependencies\n# %pip install -qr requirements.txt  # install dependencies","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:12.851666Z","iopub.execute_input":"2021-07-28T14:19:12.852002Z","iopub.status.idle":"2021-07-28T14:19:12.856699Z","shell.execute_reply.started":"2021-07-28T14:19:12.851969Z","shell.execute_reply":"2021-07-28T14:19:12.855360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd ../","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:12.860667Z","iopub.execute_input":"2021-07-28T14:19:12.860915Z","iopub.status.idle":"2021-07-28T14:19:12.867575Z","shell.execute_reply.started":"2021-07-28T14:19:12.860890Z","shell.execute_reply":"2021-07-28T14:19:12.866723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade seaborn","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:12.869598Z","iopub.execute_input":"2021-07-28T14:19:12.870031Z","iopub.status.idle":"2021-07-28T14:19:21.677741Z","shell.execute_reply.started":"2021-07-28T14:19:12.869995Z","shell.execute_reply":"2021-07-28T14:19:21.676732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns","metadata":{"papermill":{"duration":0.926929,"end_time":"2021-01-01T09:44:54.403588","exception":false,"start_time":"2021-01-01T09:44:53.476659","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T14:19:21.680586Z","iopub.execute_input":"2021-07-28T14:19:21.680882Z","iopub.status.idle":"2021-07-28T14:19:22.602981Z","shell.execute_reply.started":"2021-07-28T14:19:21.680853Z","shell.execute_reply":"2021-07-28T14:19:22.602241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 0","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.604809Z","iopub.execute_input":"2021-07-28T14:19:22.605076Z","iopub.status.idle":"2021-07-28T14:19:22.612835Z","shell.execute_reply.started":"2021-07-28T14:19:22.605051Z","shell.execute_reply":"2021-07-28T14:19:22.611929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f'../input/siim-covid19-detection/train_image_level.csv')\ntrain_df.head()","metadata":{"papermill":{"duration":0.262045,"end_time":"2021-01-01T09:44:54.691965","exception":false,"start_time":"2021-01-01T09:44:54.42992","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T14:19:22.615928Z","iopub.execute_input":"2021-07-28T14:19:22.616196Z","iopub.status.idle":"2021-07-28T14:19:22.674252Z","shell.execute_reply.started":"2021-07-28T14:19:22.616170Z","shell.execute_reply":"2021-07-28T14:19:22.673446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.id = train_df.id.str[:-6]\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.676985Z","iopub.execute_input":"2021-07-28T14:19:22.677237Z","iopub.status.idle":"2021-07-28T14:19:22.695126Z","shell.execute_reply.started":"2021-07-28T14:19:22.677212Z","shell.execute_reply":"2021-07-28T14:19:22.694213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicates = ['11f46d9df786',\n 'ba57c39bc15d',\n 'bd4b6cfeddee',\n '82c84d5079b7',\n 'a27a126be5c1',\n '3d12cb6aad8b',\n 'b814a1e29517',\n '09236f094647',\n '36b995a9575c',\n '9cdc2af4fbb0',\n '779f0040d1b2',\n 'ef6e312ca719',\n 'b97c6b32105e',\n '93301812b0e7',\n '4c414b793562',\n 'e6cc65d9de1d',\n '9fcbe25a88e0',\n 'f228350bd770',\n '59bd5a00b33e',\n 'fa5e3f933004',\n 'a8e019dcf085',\n 'd205dc170ccc',\n 'a5a364383f34',\n '9939f63af4ff',\n '646a6a0a9bf2',\n '581fd56bc4bf',\n '629c9a257b43',\n '2ebe16a053fe',\n '173c23887f9b',\n 'ea2117b53323',\n '57bf106a9e2e',\n 'b989acc3c046',\n '12fe8eadded1',\n '7f9c98b88aec',\n 'c00c7e8b673c',\n 'a5bbd30ed109',\n 'b622f84403da',\n 'ba375ad92c5f',\n 'b0fd33e251f5',\n 'c6e92e59a0ae',\n 'ef690fe6f288',\n '6970c59656c9',\n 'ab55abb953d1',\n 'd540d6c2a425',\n '262dc03560d6',\n 'fc44ab5de22c',\n 'ce51b397b1a6',\n '6e9fad584bff',\n 'a2429a60f3ea',\n 'b0866caa201a',\n '7e4a4ef3ff9f',\n '3577ee4f26c4',\n '6f54e9cbd180',\n '57faddfc733e',\n '6a93346150a4',\n '611348a721f7',\n 'c4b2940d2334',\n 'e007bc90c441',\n '3cde0ddf01c7',\n '296aaf716ae0',\n 'b7eb26ddf5a3',\n 'e738c549fe8e',\n '39d52f244db3',\n 'a9704ce54780',\n '1701be774ecb',\n '221ff464c9fc',\n 'e3f8e3a3baa3',\n 'be65d1a22de5',\n '103bd9ed3929',\n '9108cdfd43dc',\n '3d2224cb6bb7',\n 'c4b68b29a072',\n '6d36ffbc7864',\n '35da4a5a0201',\n '341787fb81c7',\n '6ff218d10741',\n '68fcf433c6fe',\n '25400558ba1e',\n '149356f04849',\n '39cd89772e5e',\n '99f3642f50f5',\n 'c72e26e9ecae',\n '4b310bbd3cb1',\n 'f208dc529d16',\n '4024e564722b',\n 'cc708f2aaffe',\n 'dcb374d8387e',\n '7f1924880cf8',\n '961f9c368664',\n '1ea01196514a',\n '725328925e0f',\n '35ea2c9841c0',\n '72e5fbd61953',\n '75b52bec817f',\n 'd27bca58e388',\n '44d7b2e7b0f9',\n '830063223a31',\n '0b020a7aff0a',\n '9d8ccbfb99b7',\n '082d3af6ecc0',\n 'eef757610245',\n 'b77430575510',\n '321c93d6503f',\n '0a990c89256a',\n 'c9a028b48504',\n 'c636ac67c19a',\n '76c66ee8e58d',\n 'cdd9e3aaf45a',\n '9a9d89ca4aa0',\n '8b55ea197c78',\n '30b18db28900',\n 'a57bf6dd6119',\n 'e4a78ff8fa73',\n 'e01d23a04187',\n '66dabc6f972d',\n '4cebe94f1fa3',\n '6792b9b0329d',\n 'ddff141123c1',\n '550f057ee0b0',\n '0102b5cac730',\n 'e7e072c73e1d',\n '00e3a7e91a34',\n '8cbf85033da8',\n '6f749e2783e1',\n '4b2f144a1a8b',\n '783b1efb5675',\n 'e96133d06736',\n '869476b0763a',\n 'e496735126c2',\n '00c1515729a8',\n '9a335ce0bca4',\n '9b1de1c45491',\n 'd9456aadecbe',\n '51d04316f2c9',\n 'fb1e24bf6698',\n 'b3ffe59e37c7',\n '16bd0ff2f0cd',\n '8d4b3609ed92',\n 'a174a4f6c807',\n '2a7a456d367e',\n '1b92142f4362',\n 'bfefe70b56ca',\n 'e5fc2fa92e65',\n '8a38c00672d5',\n 'a94171e98807',\n '61d0f2ca0ddd',\n '0bd6cd815ba9',\n '1de0ad5314cc',\n 'e897ef5c203c',\n '0acec7010081',\n '3b6ad60071d4',\n 'b61f3493c551',\n '267a250932bc',\n 'a8013eaf16c1',\n 'e704aebd4ca5',\n '04cc2f7f4c4b',\n '3e7b2ffc97db',\n 'e0e1e1462e56',\n 'b5aa6189d142',\n 'cbf0a27f993e',\n 'ea2688741043',\n '2204baf77803',\n '56ff7c4e1f41',\n '156cb1f5c689',\n '05c063f5cef5',\n 'c3a09e8a600d',\n 'f577c6fb0f49',\n '8093df07a5d0',\n '6728e11290af',\n 'efc93a3917b6',\n 'c843d08b49b8',\n '717ea5155b46',\n '343d445230a1',\n '50f29acd37a6',\n 'ea516e218fe6',\n '582c442e440b',\n 'f9a92f1ba934',\n 'bdd3115879aa',\n '68ad4b624a6d',\n 'a323f1604ae2',\n 'bb52c80c7f45',\n '0b858129adb4',\n '26c6c25e9467',\n '91f72df96e13',\n '19c6397cdec9',\n '6861c5992717',\n 'deea6d6f81a5',\n '106632672b95',\n '6534a837497d',\n '684230477525',\n 'df3d32c8df06',\n '3f0879b045e7',\n '505693d77a18',\n 'a39667fe9a81',\n '2c130ee08736',\n 'c9a9e5e120bd',\n 'bb36f3d3ac71',\n '1c96d9b08487',\n 'daa9bddc5fe6',\n 'e73f2cc047ad',\n 'e60a47d36e62',\n '6366191bfe49',\n 'd180fed57716',\n '866e3622cb24',\n '55839276a9c9',\n 'fa3a7fe547ca',\n '59bc532be971',\n '424118f1afc8',\n '6ddf030ec946',\n '3d0bac303fc8',\n 'f689d30c2f86',\n '8c7bdccb65ed',\n 'df4f1240317e',\n '25ff5d625b35',\n '75f207c4945d',\n '35e398a5a431',\n '5d395fff6278',\n 'e3e2f20e0264',\n '99e8636292c9',\n 'de94a43e947f',\n 'a5beac3d9fb4',\n 'd19ed8d811ad',\n 'bf2417586cf2',\n '72cf260ddf4c',\n 'f5451a98d684',\n 'aad814e52627',\n '32222cc776a2',\n '608d574388ba',\n '9872a8a48f23',\n 'f7f759f7a4f1',\n '88e7702b49c4',\n 'dfd9078a4c6d',\n 'd4131bd4e64a',\n '1a0e966f2d57',\n '9f50edab452b',\n '4cf41c062d0c',\n '13bf9d719ad1',\n '0eb641cb0dcd',\n 'a2ee4b862182',\n '61e618dc7f9a',\n '467f412db5c4',\n '990216aa9f08',\n '3d047361508c',\n '08acae0bf785',\n '0dd32589bdc3',\n '9f180629878a',\n '6e5946091b8a',\n 'f3ce7a4cfd20',\n 'f85fe770048a',\n 'cf0cdff4ef2e',\n 'df2bb22fa871',\n '0d4d6acc9ed3',\n '45ff2dfff073',\n 'd54f6204b044',\n 'eea3a910fa9e',\n '3d19da0d4ee6',\n 'c3510a436bff',\n '4cbc17936e7d',\n 'b121806162c3',\n 'e5380eb2d925',\n '2f973757eaa4',\n 'b5a415f70aa9',\n 'cc69791740bb',\n 'a1fa5f79671d',\n 'dbc311ed04b5',\n '744427c7dff8',\n '49664f078f0e',\n 'bee62c601ae9',\n 'e441b1fb74ee',\n '8df453b7557c',\n '7626c521cad7',\n '33a72bfb5ddb',\n '56ed68c28819',\n '04a0b90d7875',\n '08932dca1447',\n 'daae293a8b40',\n 'c0400d0e413d',\n 'ddb051c1233b',\n 'f13f9ace0a9a',\n '9c0633ef606b']","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.698766Z","iopub.execute_input":"2021-07-28T14:19:22.699018Z","iopub.status.idle":"2021-07-28T14:19:22.723486Z","shell.execute_reply.started":"2021-07-28T14:19:22.698994Z","shell.execute_reply":"2021-07-28T14:19:22.722415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(duplicates)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.724649Z","iopub.execute_input":"2021-07-28T14:19:22.725090Z","iopub.status.idle":"2021-07-28T14:19:22.737040Z","shell.execute_reply.started":"2021-07-28T14:19:22.725054Z","shell.execute_reply":"2021-07-28T14:19:22.736116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_files = list(train_df.id.values)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.738325Z","iopub.execute_input":"2021-07-28T14:19:22.738636Z","iopub.status.idle":"2021-07-28T14:19:22.746757Z","shell.execute_reply.started":"2021-07-28T14:19:22.738612Z","shell.execute_reply":"2021-07-28T14:19:22.746059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in duplicates:\n    all_files.remove(i)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.747866Z","iopub.execute_input":"2021-07-28T14:19:22.748109Z","iopub.status.idle":"2021-07-28T14:19:22.779458Z","shell.execute_reply.started":"2021-07-28T14:19:22.748087Z","shell.execute_reply":"2021-07-28T14:19:22.778856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_files)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.781114Z","iopub.execute_input":"2021-07-28T14:19:22.781350Z","iopub.status.idle":"2021-07-28T14:19:22.791750Z","shell.execute_reply.started":"2021-07-28T14:19:22.781322Z","shell.execute_reply":"2021-07-28T14:19:22.791089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df = pd.DataFrame({'id':all_files})","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.797349Z","iopub.execute_input":"2021-07-28T14:19:22.797615Z","iopub.status.idle":"2021-07-28T14:19:22.804823Z","shell.execute_reply.started":"2021-07-28T14:19:22.797590Z","shell.execute_reply":"2021-07-28T14:19:22.803911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.807042Z","iopub.execute_input":"2021-07-28T14:19:22.807484Z","iopub.status.idle":"2021-07-28T14:19:22.819270Z","shell.execute_reply.started":"2021-07-28T14:19:22.807447Z","shell.execute_reply":"2021-07-28T14:19:22.818565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.merge(final_df, train_df, how='left', on='id')","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.820910Z","iopub.execute_input":"2021-07-28T14:19:22.821339Z","iopub.status.idle":"2021-07-28T14:19:22.838430Z","shell.execute_reply.started":"2021-07-28T14:19:22.821304Z","shell.execute_reply":"2021-07-28T14:19:22.837810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.839669Z","iopub.execute_input":"2021-07-28T14:19:22.840040Z","iopub.status.idle":"2021-07-28T14:19:22.845534Z","shell.execute_reply.started":"2021-07-28T14:19:22.840003Z","shell.execute_reply":"2021-07-28T14:19:22.844712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.846954Z","iopub.execute_input":"2021-07-28T14:19:22.847555Z","iopub.status.idle":"2021-07-28T14:19:22.862553Z","shell.execute_reply.started":"2021-07-28T14:19:22.847516Z","shell.execute_reply":"2021-07-28T14:19:22.861434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv'); print(study_df.shape)\nstudy_df['StudyInstanceUID'] = study_df['id'].apply(lambda x: x.replace('_study', ''))\ndel study_df['id']","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.863798Z","iopub.execute_input":"2021-07-28T14:19:22.864317Z","iopub.status.idle":"2021-07-28T14:19:22.889880Z","shell.execute_reply.started":"2021-07-28T14:19:22.864271Z","shell.execute_reply":"2021-07-28T14:19:22.888867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def hot_to_sparse(row):\n    return(row.index[row.apply(lambda x: x==1)][0])\nstudy_df['diagnosis'] = study_df.apply(lambda row:hot_to_sparse(row), axis=1)\ncls = {\n    'Typical Appearance':1,                    \n    'Negative for Pneumonia':0,                \n    'Indeterminate Appearance':2,                     \n    'Atypical Appearance':3,    \n}\nstudy_df['sparse_gt'] = study_df.diagnosis.map(cls)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:22.891198Z","iopub.execute_input":"2021-07-28T14:19:22.891570Z","iopub.status.idle":"2021-07-28T14:19:23.938214Z","shell.execute_reply.started":"2021-07-28T14:19:22.891535Z","shell.execute_reply":"2021-07-28T14:19:23.937333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:23.939612Z","iopub.execute_input":"2021-07-28T14:19:23.940006Z","iopub.status.idle":"2021-07-28T14:19:23.953626Z","shell.execute_reply.started":"2021-07-28T14:19:23.939966Z","shell.execute_reply":"2021-07-28T14:19:23.952416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_df.merge(study_df, on='StudyInstanceUID')\ntrain['id'] = train['id'].apply(lambda x: x.replace('_image', ''))","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:23.955624Z","iopub.execute_input":"2021-07-28T14:19:23.956204Z","iopub.status.idle":"2021-07-28T14:19:23.975236Z","shell.execute_reply.started":"2021-07-28T14:19:23.956158Z","shell.execute_reply":"2021-07-28T14:19:23.974567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:23.978071Z","iopub.execute_input":"2021-07-28T14:19:23.978320Z","iopub.status.idle":"2021-07-28T14:19:23.993109Z","shell.execute_reply.started":"2021-07-28T14:19:23.978295Z","shell.execute_reply":"2021-07-28T14:19:23.991954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:23.994823Z","iopub.execute_input":"2021-07-28T14:19:23.995287Z","iopub.status.idle":"2021-07-28T14:19:24.001773Z","shell.execute_reply.started":"2021-07-28T14:19:23.995247Z","shell.execute_reply":"2021-07-28T14:19:24.000696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:24.003653Z","iopub.execute_input":"2021-07-28T14:19:24.004295Z","iopub.status.idle":"2021-07-28T14:19:24.009689Z","shell.execute_reply.started":"2021-07-28T14:19:24.004257Z","shell.execute_reply":"2021-07-28T14:19:24.008596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:24.011107Z","iopub.execute_input":"2021-07-28T14:19:24.011547Z","iopub.status.idle":"2021-07-28T14:19:24.028522Z","shell.execute_reply.started":"2021-07-28T14:19:24.011512Z","shell.execute_reply":"2021-07-28T14:19:24.027683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls '../input'","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:24.030267Z","iopub.execute_input":"2021-07-28T14:19:24.030939Z","iopub.status.idle":"2021-07-28T14:19:24.665516Z","shell.execute_reply.started":"2021-07-28T14:19:24.030898Z","shell.execute_reply":"2021-07-28T14:19:24.664704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['image_path'] = f'/kaggle/input/covid19-detection-890pxpng-study/train/' + train.id + '.png'\ntrain.head()","metadata":{"papermill":{"duration":0.086788,"end_time":"2021-01-01T09:44:54.805857","exception":false,"start_time":"2021-01-01T09:44:54.719069","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T14:19:24.668846Z","iopub.execute_input":"2021-07-28T14:19:24.669118Z","iopub.status.idle":"2021-07-28T14:19:24.703273Z","shell.execute_reply.started":"2021-07-28T14:19:24.669090Z","shell.execute_reply":"2021-07-28T14:19:24.702443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nskf = StratifiedKFold(n_splits=6, shuffle=True, random_state=9)\nfor index, (train_index, val_index) in enumerate(skf.split(X=train.index, y=train.sparse_gt)):\n    train.loc[val_index, 'fold'] = index\n    \nprint(train.groupby(['fold', train.sparse_gt]).size())","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:24.704618Z","iopub.execute_input":"2021-07-28T14:19:24.705153Z","iopub.status.idle":"2021-07-28T14:19:24.728066Z","shell.execute_reply.started":"2021-07-28T14:19:24.705115Z","shell.execute_reply":"2021-07-28T14:19:24.727134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = ['0. opacity']","metadata":{"papermill":{"duration":0.050418,"end_time":"2021-01-01T09:45:03.002944","exception":false,"start_time":"2021-01-01T09:45:02.952526","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T14:19:24.729261Z","iopub.execute_input":"2021-07-28T14:19:24.729824Z","iopub.status.idle":"2021-07-28T14:19:24.733876Z","shell.execute_reply.started":"2021-07-28T14:19:24.729786Z","shell.execute_reply":"2021-07-28T14:19:24.732749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n# X_train, X_valid = train_test_split(train_df, test_size=0.1, stratify=train_df['sparse_gt'], random_state=18)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:24.735517Z","iopub.execute_input":"2021-07-28T14:19:24.736146Z","iopub.status.idle":"2021-07-28T14:19:24.742580Z","shell.execute_reply.started":"2021-07-28T14:19:24.736096Z","shell.execute_reply":"2021-07-28T14:19:24.741783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train[train['fold'] != 2]\nX_valid = train[train['fold'] == 2]","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:24.744079Z","iopub.execute_input":"2021-07-28T14:19:24.744731Z","iopub.status.idle":"2021-07-28T14:19:24.754075Z","shell.execute_reply.started":"2021-07-28T14:19:24.744693Z","shell.execute_reply":"2021-07-28T14:19:24.753219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.sparse_gt.value_counts(), X_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:24.755598Z","iopub.execute_input":"2021-07-28T14:19:24.756229Z","iopub.status.idle":"2021-07-28T14:19:24.764505Z","shell.execute_reply.started":"2021-07-28T14:19:24.756189Z","shell.execute_reply":"2021-07-28T14:19:24.763509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid.sparse_gt.value_counts(), X_valid.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:19:24.766106Z","iopub.execute_input":"2021-07-28T14:19:24.766903Z","iopub.status.idle":"2021-07-28T14:19:24.775161Z","shell.execute_reply.started":"2021-07-28T14:19:24.766750Z","shell.execute_reply":"2021-07-28T14:19:24.774209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nval_files   = []\ntrain_files += list(X_train.image_path.unique())\nval_files += list(X_valid.image_path.unique())\nlen(train_files), len(val_files)","metadata":{"papermill":{"duration":0.086817,"end_time":"2021-01-01T09:47:56.443789","exception":false,"start_time":"2021-01-01T09:47:56.356972","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T14:19:24.776628Z","iopub.execute_input":"2021-07-28T14:19:24.777261Z","iopub.status.idle":"2021-07-28T14:19:24.789567Z","shell.execute_reply.started":"2021-07-28T14:19:24.777222Z","shell.execute_reply":"2021-07-28T14:19:24.788468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Copying Files","metadata":{"papermill":{"duration":0.083752,"end_time":"2021-01-01T09:47:56.584924","exception":false,"start_time":"2021-01-01T09:47:56.501172","status":"completed"},"tags":[]}},{"cell_type":"code","source":"os.makedirs('/kaggle/working/siim-cov19/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/images/val', exist_ok = True)\nlabel_dir = '/kaggle/input/siim-covid-19-yolo-txt'\nfor file in tqdm(train_files):\n    shutil.copy(file, '/kaggle/working/siim-cov19/images/train')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/siim-cov19/labels/train')\n    \nfor file in tqdm(val_files):\n    shutil.copy(file, '/kaggle/working/siim-cov19/images/val')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/siim-cov19/labels/val')","metadata":{"papermill":{"duration":124.654777,"end_time":"2021-01-01T09:50:01.331041","exception":false,"start_time":"2021-01-01T09:47:56.676264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T14:19:24.791088Z","iopub.execute_input":"2021-07-28T14:19:24.791613Z","iopub.status.idle":"2021-07-28T14:20:58.694444Z","shell.execute_reply.started":"2021-07-28T14:19:24.791576Z","shell.execute_reply":"2021-07-28T14:20:58.693730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Class Name","metadata":{"papermill":{"duration":0.068822,"end_time":"2021-01-01T09:50:01.458337","exception":false,"start_time":"2021-01-01T09:50:01.389515","status":"completed"},"tags":[]}},{"cell_type":"code","source":"classes = ['0. opacity']","metadata":{"papermill":{"duration":0.082234,"end_time":"2021-01-01T09:50:01.601574","exception":false,"start_time":"2021-01-01T09:50:01.51934","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T14:20:58.696131Z","iopub.execute_input":"2021-07-28T14:20:58.696527Z","iopub.status.idle":"2021-07-28T14:20:58.700852Z","shell.execute_reply.started":"2021-07-28T14:20:58.696487Z","shell.execute_reply":"2021-07-28T14:20:58.699763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# YOLOv5 Stuff","metadata":{"papermill":{"duration":0.055699,"end_time":"2021-01-01T09:50:01.82747","exception":false,"start_time":"2021-01-01T09:50:01.771771","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport yaml\n\ncwd = '/kaggle/working/'\n\nwith open(join( cwd , 'train.txt'), 'w') as f:\n    for path in glob('/kaggle/working/siim-cov19/images/train/*'):\n        f.write(path+'\\n')\n            \nwith open(join( cwd , 'val.txt'), 'w') as f:\n    for path in glob('/kaggle/working/siim-cov19/images/val/*'):\n        f.write(path+'\\n')\n\ndata = dict(\n    train =  join( cwd , 'train.txt') ,\n    val   =  join( cwd , 'val.txt' ),\n    nc    = 1,\n    names = classes\n    )\n\nwith open(join( cwd , 'siim-cov19.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(join( cwd , 'siim-cov19.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"papermill":{"duration":0.113001,"end_time":"2021-01-01T09:50:01.996448","exception":false,"start_time":"2021-01-01T09:50:01.883447","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T14:20:58.702869Z","iopub.execute_input":"2021-07-28T14:20:58.703351Z","iopub.status.idle":"2021-07-28T14:20:58.819888Z","shell.execute_reply.started":"2021-07-28T14:20:58.703313Z","shell.execute_reply":"2021-07-28T14:20:58.818932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls '../input'","metadata":{"execution":{"iopub.status.busy":"2021-07-28T14:20:58.824182Z","iopub.execute_input":"2021-07-28T14:20:58.826424Z","iopub.status.idle":"2021-07-28T14:20:59.567996Z","shell.execute_reply.started":"2021-07-28T14:20:58.826364Z","shell.execute_reply":"2021-07-28T14:20:59.567163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/ultralytics/yolov5\n# !git clone https://github.com/ultralytics/yolov5  # clone repo\n# %cd yolov5\nshutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5')\n# %pip install -qr requirements.txt # install dependencies\n\nimport torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":6.702428,"end_time":"2021-01-01T09:50:08.784153","exception":false,"start_time":"2021-01-01T09:50:02.081725","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T14:20:59.571421Z","iopub.execute_input":"2021-07-28T14:20:59.571703Z","iopub.status.idle":"2021-07-28T14:21:01.861879Z","shell.execute_reply.started":"2021-07-28T14:20:59.571674Z","shell.execute_reply":"2021-07-28T14:21:01.860941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pretrained Checkpoints:\n\n| Model | AP<sup>val</sup> | AP<sup>test</sup> | AP<sub>50</sub> | Speed<sub>GPU</sub> | FPS<sub>GPU</sub> || params | FLOPS |\n|---------- |------ |------ |------ | -------- | ------| ------ |------  |  :------: |\n| [YOLOv5s](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | 37.0     | 37.0     | 56.2     | **2.4ms** | **416** || 7.5M   | 13.2B\n| [YOLOv5m](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | 44.3     | 44.3     | 63.2     | 3.4ms     | 294     || 21.8M  | 39.4B\n| [YOLOv5l](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | 47.7     | 47.7     | 66.5     | 4.4ms     | 227     || 47.8M  | 88.1B\n| [YOLOv5x](https://github.com/ultralytics/yolov5/releases/tag/v3.0)    | **49.2** | **49.2** | **67.7** | 6.9ms     | 145     || 89.0M  | 166.4B\n| | | | | | || |\n| [YOLOv5x](https://github.com/ultralytics/yolov5/releases/tag/v3.0) + TTA|**50.8**| **50.8** | **68.9** | 25.5ms    | 39      || 89.0M  | 354.3B\n| | | | | | || |\n| [YOLOv3-SPP](https://github.com/ultralytics/yolov5/releases/tag/v3.0) | 45.6     | 45.5     | 65.2     | 4.5ms     | 222     || 63.0M  | 118.0B","metadata":{"papermill":{"duration":0.064911,"end_time":"2021-01-01T09:50:19.435746","exception":false,"start_time":"2021-01-01T09:50:19.370835","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Selecting Models\nIn this notebok I'm using `v5s`. To select your prefered model just replace `--cfg models/yolov5s.yaml --weights yolov5s.pt` with the following command:\n* `v5s` : `--cfg models/yolov5s.yaml --weights yolov5s.pt`\n* `v5m` : `--cfg models/yolov5m.yaml --weights yolov5m.pt`\n* `v5l` : `--cfg models/yolov5l.yaml --weights yolov5l.pt`\n* `v5x` : `--cfg models/yolov5x.yaml --weights yolov5x.pt`","metadata":{"papermill":{"duration":0.064016,"end_time":"2021-01-01T09:50:19.564859","exception":false,"start_time":"2021-01-01T09:50:19.500843","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Train","metadata":{"papermill":{"duration":0.064553,"end_time":"2021-01-01T09:50:19.6938","exception":false,"start_time":"2021-01-01T09:50:19.629247","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!WANDB_MODE=\"dryrun\" python train.py --img 640 --batch 24 --epochs 35 --data /kaggle/working/siim-cov19.yaml --hyp /kaggle/input/yolo-yaml-txt-nv/hyp.scratch.yaml --weights yolov5m.pt --cache","metadata":{"papermill":{"duration":19916.498298,"end_time":"2021-01-01T15:22:16.289734","exception":false,"start_time":"2021-01-01T09:50:19.791436","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T14:21:01.863221Z","iopub.execute_input":"2021-07-28T14:21:01.863749Z","iopub.status.idle":"2021-07-28T15:55:27.963289Z","shell.execute_reply.started":"2021-07-28T14:21:01.863706Z","shell.execute_reply":"2021-07-28T15:55:27.962335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Distribution","metadata":{"papermill":{"duration":4.919442,"end_time":"2021-01-01T15:22:26.398681","exception":false,"start_time":"2021-01-01T15:22:21.479239","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# plt.figure(figsize = (20,20))\n# plt.axis('off')\n# plt.imshow(plt.imread('runs/train/exp/labels_correlogram.jpg'));","metadata":{"papermill":{"duration":6.511035,"end_time":"2021-01-01T15:22:37.753063","exception":false,"start_time":"2021-01-01T15:22:31.242028","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T15:55:27.965300Z","iopub.execute_input":"2021-07-28T15:55:27.965688Z","iopub.status.idle":"2021-07-28T15:55:27.973285Z","shell.execute_reply.started":"2021-07-28T15:55:27.965647Z","shell.execute_reply":"2021-07-28T15:55:27.972443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (20,20))\nplt.axis('off')\nplt.imshow(plt.imread('runs/train/exp/labels.jpg'));","metadata":{"papermill":{"duration":5.977042,"end_time":"2021-01-01T15:22:48.614609","exception":false,"start_time":"2021-01-01T15:22:42.637567","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T15:55:27.974525Z","iopub.execute_input":"2021-07-28T15:55:27.974808Z","iopub.status.idle":"2021-07-28T15:55:28.785783Z","shell.execute_reply.started":"2021-07-28T15:55:27.974781Z","shell.execute_reply":"2021-07-28T15:55:28.784602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Batch Image","metadata":{"papermill":{"duration":5.378338,"end_time":"2021-01-01T15:22:59.482837","exception":false,"start_time":"2021-01-01T15:22:54.104499","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch0.jpg'))\n\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch1.jpg'))\n\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch2.jpg'))","metadata":{"papermill":{"duration":7.317416,"end_time":"2021-01-01T15:23:11.777544","exception":false,"start_time":"2021-01-01T15:23:04.460128","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-28T15:55:28.787075Z","iopub.execute_input":"2021-07-28T15:55:28.787471Z","iopub.status.idle":"2021-07-28T15:55:30.605741Z","shell.execute_reply.started":"2021-07-28T15:55:28.787433Z","shell.execute_reply":"2021-07-28T15:55:30.604417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GT Vs Pred","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(3, 2, figsize = (2*5,3*5), constrained_layout = True)\nfor row in range(3):\n    ax[row][0].imshow(plt.imread(f'runs/train/exp/test_batch{row}_labels.jpg'))\n    ax[row][0].set_xticks([])\n    ax[row][0].set_yticks([])\n    ax[row][0].set_title(f'runs/train/exp/test_batch{row}_labels.jpg', fontsize = 12)\n    \n    ax[row][1].imshow(plt.imread(f'runs/train/exp/test_batch{row}_pred.jpg'))\n    ax[row][1].set_xticks([])\n    ax[row][1].set_yticks([])\n    ax[row][1].set_title(f'runs/train/exp/test_batch{row}_pred.jpg', fontsize = 12)","metadata":{"papermill":{"duration":6.453975,"end_time":"2021-01-01T15:23:23.514717","exception":false,"start_time":"2021-01-01T15:23:17.060742","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-28T15:55:30.607458Z","iopub.execute_input":"2021-07-28T15:55:30.607831Z","iopub.status.idle":"2021-07-28T15:55:32.408791Z","shell.execute_reply.started":"2021-07-28T15:55:30.607794Z","shell.execute_reply":"2021-07-28T15:55:32.407366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (Loss, Map) Vs Epoch","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30,15))\nplt.axis('off')\nplt.imshow(plt.imread('runs/train/exp/results.png'));","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-28T15:55:32.410657Z","iopub.execute_input":"2021-07-28T15:55:32.411032Z","iopub.status.idle":"2021-07-28T15:55:33.153231Z","shell.execute_reply.started":"2021-07-28T15:55:32.410996Z","shell.execute_reply":"2021-07-28T15:55:33.152276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion Matrix","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30,15))\nplt.axis('off')\nplt.imshow(plt.imread('runs/train/exp/confusion_matrix.png'));","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-28T15:55:33.154920Z","iopub.execute_input":"2021-07-28T15:55:33.155297Z","iopub.status.idle":"2021-07-28T15:55:34.256609Z","shell.execute_reply.started":"2021-07-28T15:55:33.155258Z","shell.execute_reply":"2021-07-28T15:55:34.255838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{"papermill":{"duration":4.941983,"end_time":"2021-01-01T15:23:33.765831","exception":false,"start_time":"2021-01-01T15:23:28.823848","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!python detect.py --weights 'runs/train/exp/weights/best.pt'\\\n--img 512\\\n--conf 0.1\\\n--iou 0.5\\\n--source /kaggle/working/siim-cov19/images/val\\\n--exist-ok","metadata":{"_kg_hide-output":true,"papermill":{"duration":10.763143,"end_time":"2021-01-01T15:23:49.800461","exception":false,"start_time":"2021-01-01T15:23:39.037318","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-28T15:55:34.257960Z","iopub.execute_input":"2021-07-28T15:55:34.258300Z","iopub.status.idle":"2021-07-28T15:56:46.744657Z","shell.execute_reply.started":"2021-07-28T15:55:34.258271Z","shell.execute_reply":"2021-07-28T15:56:46.743757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Plot","metadata":{"papermill":{"duration":5.225725,"end_time":"2021-01-01T15:24:00.706026","exception":false,"start_time":"2021-01-01T15:23:55.480301","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nimport numpy as np\nimport random\nimport cv2\nfrom glob import glob\nfrom tqdm import tqdm\n\nfiles = glob('runs/detect/exp/*')\nfor _ in range(3):\n    row = 4\n    col = 4\n    grid_files = random.sample(files, row*col)\n    images     = []\n    for image_path in tqdm(grid_files):\n        img          = cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n        images.append(img)\n\n    fig = plt.figure(figsize=(col*5, row*5))\n    grid = ImageGrid(fig, 111,  # similar to subplot(111)\n                     nrows_ncols=(col, row),  # creates 2x2 grid of axes\n                     axes_pad=0.05,  # pad between axes in inch.\n                     )\n\n    for ax, im in zip(grid, images):\n        # Iterating over the grid returns the Axes.\n        ax.imshow(im)\n        ax.set_xticks([])\n        ax.set_yticks([])\n    plt.show()","metadata":{"papermill":{"duration":5.31015,"end_time":"2021-01-01T15:24:11.211904","exception":false,"start_time":"2021-01-01T15:24:05.901754","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-28T15:56:46.746519Z","iopub.execute_input":"2021-07-28T15:56:46.746883Z","iopub.status.idle":"2021-07-28T15:56:55.644955Z","shell.execute_reply.started":"2021-07-28T15:56:46.746852Z","shell.execute_reply":"2021-07-28T15:56:55.637477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree('/kaggle/working/siim-cov19')\nshutil.rmtree('runs/detect')\nfor file in (glob('runs/train/exp/**/*.png', recursive = True)+glob('runs/train/exp/**/*.jpg', recursive = True)):\n    os.remove(file)","metadata":{"papermill":{"duration":5.709202,"end_time":"2021-01-01T15:24:22.413173","exception":false,"start_time":"2021-01-01T15:24:16.703971","status":"completed"},"tags":[],"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-28T15:56:55.646556Z","iopub.execute_input":"2021-07-28T15:56:55.646908Z","iopub.status.idle":"2021-07-28T15:56:56.188618Z","shell.execute_reply.started":"2021-07-28T15:56:55.646873Z","shell.execute_reply":"2021-07-28T15:56:56.187772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}