{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!cp /kaggle/input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-17T14:31:37.232963Z","iopub.execute_input":"2021-09-17T14:31:37.23327Z","iopub.status.idle":"2021-09-17T14:32:00.143748Z","shell.execute_reply.started":"2021-09-17T14:31:37.233197Z","shell.execute_reply":"2021-09-17T14:32:00.142697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:00.145601Z","iopub.execute_input":"2021-09-17T14:32:00.145955Z","iopub.status.idle":"2021-09-17T14:32:00.817243Z","shell.execute_reply.started":"2021-09-17T14:32:00.145913Z","shell.execute_reply":"2021-09-17T14:32:00.816356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:00.819313Z","iopub.execute_input":"2021-09-17T14:32:00.819717Z","iopub.status.idle":"2021-09-17T14:32:00.824159Z","shell.execute_reply.started":"2021-09-17T14:32:00.819675Z","shell.execute_reply":"2021-09-17T14:32:00.82295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntot_m, used_m, free_m = map(int, os.popen('free -t -m').readlines()[-1].split()[1:])\nprint('tot_m, used_m, free_m',tot_m, used_m, free_m )","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:00.826259Z","iopub.execute_input":"2021-09-17T14:32:00.826899Z","iopub.status.idle":"2021-09-17T14:32:00.855008Z","shell.execute_reply.started":"2021-09-17T14:32:00.826862Z","shell.execute_reply":"2021-09-17T14:32:00.853979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import display_html\ndef restartkernel() :\n    #display_html(\"<script>Jupyter.notebook.kernel.restart()</script>\",raw=True)\n    display_html(\"\",raw=True)\n    \nrestartkernel()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:00.856362Z","iopub.execute_input":"2021-09-17T14:32:00.856719Z","iopub.status.idle":"2021-09-17T14:32:00.864156Z","shell.execute_reply.started":"2021-09-17T14:32:00.856681Z","shell.execute_reply":"2021-09-17T14:32:00.862997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE=608","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:00.865693Z","iopub.execute_input":"2021-09-17T14:32:00.866102Z","iopub.status.idle":"2021-09-17T14:32:00.870338Z","shell.execute_reply.started":"2021-09-17T14:32:00.866068Z","shell.execute_reply":"2021-09-17T14:32:00.869326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2   \nimport pandas as pd\nimport numpy as np\nimport PIL.Image as Image\nimport numpy as np\nimport time\nimport pandas as pd\nimport numpy as np\nimport xgboost as xgb\nimport tensorflow as tf\nimport tensorflow.keras as keras\nfrom sklearn.preprocessing import StandardScaler\nimport math\nimport sklearn\nfrom sklearn.ensemble import RandomForestClassifier\nimport warnings\nimport eli5\nimport datatable as dt\nimport matplotlib.pyplot as plt\nimport os\nimport csv\nimport cv2\nfrom tensorflow.keras.preprocessing import image\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D\nfrom tensorflow.keras.models import Model\nimport tensorflow_addons as tfa\nimport pickle\nimport re\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport time\nfrom keras.models import model_from_json\n\n'''\n# specify your image path\nimage_path = '../input/siim-covid19-detection/train/00086460a852/9e8302230c91/65761e66de9f.dcm'\nds1 = dicom.dcmread(image_path)\nprint(type(ds1))#, ds1)\n'''","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:00.871995Z","iopub.execute_input":"2021-09-17T14:32:00.872487Z","iopub.status.idle":"2021-09-17T14:32:08.186499Z","shell.execute_reply.started":"2021-09-17T14:32:00.872448Z","shell.execute_reply":"2021-09-17T14:32:08.1857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import model_from_json","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:08.189178Z","iopub.execute_input":"2021-09-17T14:32:08.189503Z","iopub.status.idle":"2021-09-17T14:32:08.193349Z","shell.execute_reply.started":"2021-09-17T14:32:08.189476Z","shell.execute_reply":"2021-09-17T14:32:08.192316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.dcmread(path)\n    #print(dicom)\n    #print('dicom ',dicom.pixel_array.shape)\n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n        #print('in If ', data.shape)\n    else:\n        data = dicom.pixel_array\n        #print('in else ', data.shape)\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        #print('In fix_monochrome ', data.shape)\n        \n    data = data - np.min(data)\n    #print('minus ', data.shape)\n    data = data / np.max(data)\n    #print('divisn ', data.shape)\n    data = (data * 255).astype(np.uint8)\n    #print('* ', data.shape)\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:08.19563Z","iopub.execute_input":"2021-09-17T14:32:08.196118Z","iopub.status.idle":"2021-09-17T14:32:08.203677Z","shell.execute_reply.started":"2021-09-17T14:32:08.19608Z","shell.execute_reply":"2021-09-17T14:32:08.202751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:08.205188Z","iopub.execute_input":"2021-09-17T14:32:08.205553Z","iopub.status.idle":"2021-09-17T14:32:08.214993Z","shell.execute_reply.started":"2021-09-17T14:32:08.205503Z","shell.execute_reply":"2021-09-17T14:32:08.214224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_smpl_subm=pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')\ndf_smpl_subm","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:08.216116Z","iopub.execute_input":"2021-09-17T14:32:08.216456Z","iopub.status.idle":"2021-09-17T14:32:08.253058Z","shell.execute_reply.started":"2021-09-17T14:32:08.21642Z","shell.execute_reply":"2021-09-17T14:32:08.252166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_files=0\ndir_len=0\n\nbroad_dir_len =0\n\nfor dirname, _, filenames in os.walk('../input/siim-covid19-detection/test'):\n\n    if len(_) ==0:\n        dir_len+=1\n    j=0\n    \n    if len(_)>0:\n        broad_dir_len+=1\n    \n    \n    for filename in filenames:\n        j+=1\n        '''\n        if num_files in {3728, 3938, 3939}:\n            print(os.path.join(dirname, filename))\n        '''\n        \n        '''\n        if j >1:\n            print(' num_files is ', num_files, ' and j is ', j, 'and dir_len is ', dir_len)\n            print(os.path.join(dirname, filename))\n        '''\n        \n        num_files+=1\n    '''\n    if dir_len in {3728, 3937}:\n        print(dir_len)\n        print('dirname ', dirname)\n        print('_ ', _, len(_))\n        print('filenames ', filenames)\n    '''\n\nprint('dir_len is ', dir_len)\nprint('broad_dir_len is ', broad_dir_len)\nprint('Total # of files ', num_files)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:08.254302Z","iopub.execute_input":"2021-09-17T14:32:08.254646Z","iopub.status.idle":"2021-09-17T14:32:13.64513Z","shell.execute_reply.started":"2021-09-17T14:32:08.254598Z","shell.execute_reply":"2021-09-17T14:32:13.644246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"studyid_list=[['a','b','c']]*(broad_dir_len-1)\n\ni=0\nfor dirname, _, filenames in os.walk('../input/siim-covid19-detection/test'):\n    \n    dirname_list=re.split('/', dirname)\n    #print(len(dirname_list), dirname_list)\n    if len(dirname_list)==5:\n        studyid=[dirname, dirname_list[4], dirname_list[4]+'_study']\n        studyid_list[i]= studyid\n        i+=1\n    '''\n    if len(dirname_list)==6:\n        image_list=[]\n        for filename in filenames:\n            dir_=os.path.join(dirname, filename)\n            image_list.append(dir_)\n    '''\n        \nlen(studyid_list), '\\n', studyid_list[:2]\n\ndf_test_studyid = pd.DataFrame(studyid_list, columns=['study_path', 'study_id', 'StudyUID' ])\nprint(len(df_test_studyid), '\\n')\n\ndf_test_studyid.sort_values(by=['study_id'], inplace=True)\ndf_test_studyid.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:13.646507Z","iopub.execute_input":"2021-09-17T14:32:13.646878Z","iopub.status.idle":"2021-09-17T14:32:14.671527Z","shell.execute_reply.started":"2021-09-17T14:32:13.646835Z","shell.execute_reply":"2021-09-17T14:32:14.670746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_colwidth', None)\nimageid_list=[['a', 'b', 'c', 'd']]*(num_files)\n\ni=0\n\nfor dirname, _, filenames in os.walk('../input/siim-covid19-detection/test'):\n    \n    if len(_)==0:        \n        for filename in filenames:\n            dirname_list=re.split('/', dirname)\n            study_id=dirname_list[len(dirname_list)-2]\n            image_path = os.path.join(dirname, filename)\n            image_id=filename.replace('.dcm', '')\n            imageid=[image_path, image_id, image_id + '_image', study_id]\n            imageid_list[i]= imageid\n            i+=1\nprint(i, '\\n')\n#len(imageid_list), '\\n', imageid_list[:10], imageid_list[-10:]\n\ndf_test_imageid = pd.DataFrame(imageid_list, columns=['image_path', 'ImageUID', 'image_id' ,'study_id'])\nprint(len(df_test_imageid), '\\n')\n\ndf_test_imageid.sort_values(by=['study_id'], inplace=True)\ndf_test_imageid","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:14.672852Z","iopub.execute_input":"2021-09-17T14:32:14.673199Z","iopub.status.idle":"2021-09-17T14:32:15.766906Z","shell.execute_reply.started":"2021-09-17T14:32:14.673162Z","shell.execute_reply":"2021-09-17T14:32:15.766079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_study_path=pd.merge(df_test_studyid, df_test_imageid, how='inner', left_on='study_id', right_on='study_id')#, left_index=True, right_index=True )\ndf_test_study_path","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:15.770794Z","iopub.execute_input":"2021-09-17T14:32:15.772997Z","iopub.status.idle":"2021-09-17T14:32:15.809556Z","shell.execute_reply.started":"2021-09-17T14:32:15.772954Z","shell.execute_reply":"2021-09-17T14:32:15.808649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_study_path.drop(columns=['study_path', 'study_id', 'ImageUID'], inplace=True)\ndf_test_study_path","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:15.813305Z","iopub.execute_input":"2021-09-17T14:32:15.815484Z","iopub.status.idle":"2021-09-17T14:32:15.837205Z","shell.execute_reply.started":"2021-09-17T14:32:15.815443Z","shell.execute_reply":"2021-09-17T14:32:15.836449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np_test_study=df_test_study_path.to_numpy()\nnp_test_study.shape, np_test_study[:2]","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:15.840975Z","iopub.execute_input":"2021-09-17T14:32:15.843099Z","iopub.status.idle":"2021-09-17T14:32:15.853377Z","shell.execute_reply.started":"2021-09-17T14:32:15.843061Z","shell.execute_reply":"2021-09-17T14:32:15.852397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/tmp/')","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:15.857685Z","iopub.execute_input":"2021-09-17T14:32:15.859798Z","iopub.status.idle":"2021-09-17T14:32:15.865505Z","shell.execute_reply.started":"2021-09-17T14:32:15.859761Z","shell.execute_reply":"2021-09-17T14:32:15.864702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/tmp/test_images/')","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:15.870438Z","iopub.execute_input":"2021-09-17T14:32:15.872517Z","iopub.status.idle":"2021-09-17T14:32:15.878014Z","shell.execute_reply.started":"2021-09-17T14:32:15.872452Z","shell.execute_reply":"2021-09-17T14:32:15.877172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd /kaggle/working","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:15.88247Z","iopub.execute_input":"2021-09-17T14:32:15.883967Z","iopub.status.idle":"2021-09-17T14:32:15.892486Z","shell.execute_reply.started":"2021-09-17T14:32:15.883168Z","shell.execute_reply":"2021-09-17T14:32:15.891263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os.mkdir('./test_images')","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:15.895915Z","iopub.execute_input":"2021-09-17T14:32:15.896646Z","iopub.status.idle":"2021-09-17T14:32:15.901583Z","shell.execute_reply.started":"2021-09-17T14:32:15.896588Z","shell.execute_reply":"2021-09-17T14:32:15.900528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start=time.time()\n\n#shape_list=np.array()\nfor i in range(len(np_test_study)):\n    #print(image_path[i])\n    #print(i)\n    \n\n    img=read_xray(np_test_study[i][1])\n    #print(img.shape)\n    img_resized=resize(img, IMG_SIZE)\n    \n    np_img_resize = np.asarray(img_resized)\n    img_resized=np_img_resize.reshape([1,np_img_resize.shape[0], np_img_resize.shape[1],1 ])\n    \n    img_id=re.sub( '_image', '', np_test_study[i][2] )\n    path=os.path.join('/kaggle/tmp/test_images/', (img_id + '.jpg'))\n    cv2.imwrite(path, np_img_resize)\n\n    end=time.time()\n\n    \n    if i == 0:\n        test_image_data = img_resized\n        \n    else:\n        test_image_data=np.concatenate((test_image_data, img_resized), axis=0)\n    \n    end=time.time()\n    \n    \n    if i%100 ==0:\n        print(i)\n        print(f'Time taken for {i} images is ', (end-start)/60., 'mins')\n        start=end\n\nend=time.time()\nprint(f'Time taken for {i} images   is ', (end-start)/60., 'mins')\n\n\nprint(type(test_image_data), test_image_data.shape)#, test_image_data[:2])","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:32:15.903254Z","iopub.execute_input":"2021-09-17T14:32:15.903928Z","iopub.status.idle":"2021-09-17T14:33:03.549384Z","shell.execute_reply.started":"2021-09-17T14:32:15.903892Z","shell.execute_reply":"2021-09-17T14:33:03.548479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_PATH = '/kaggle/tmp/test_images' # absolute path\nMODEL_PATH = '/kaggle/input/best9-sept21/best_9Sept2.pt'","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:03.554472Z","iopub.execute_input":"2021-09-17T14:33:03.555006Z","iopub.status.idle":"2021-09-17T14:33:03.564298Z","shell.execute_reply.started":"2021-09-17T14:33:03.554958Z","shell.execute_reply":"2021-09-17T14:33:03.562443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('./detect')","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:03.570873Z","iopub.execute_input":"2021-09-17T14:33:03.571423Z","iopub.status.idle":"2021-09-17T14:33:03.57721Z","shell.execute_reply.started":"2021-09-17T14:33:03.571386Z","shell.execute_reply":"2021-09-17T14:33:03.576266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('./detect/exp')","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:03.579231Z","iopub.execute_input":"2021-09-17T14:33:03.580949Z","iopub.status.idle":"2021-09-17T14:33:03.591237Z","shell.execute_reply.started":"2021-09-17T14:33:03.580914Z","shell.execute_reply":"2021-09-17T14:33:03.589853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nwith open('/kaggle/input/requirements/requirements.txt') as req_txt:\n    k= req_txt.read()\n    print(k)\n    with open('/kaggle/working/requirements.txt', 'w') as txtfile:\n        txtfile.write(k)\n","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:03.593225Z","iopub.execute_input":"2021-09-17T14:33:03.594428Z","iopub.status.idle":"2021-09-17T14:33:03.622647Z","shell.execute_reply.started":"2021-09-17T14:33:03.593892Z","shell.execute_reply":"2021-09-17T14:33:03.621892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python /kaggle/input/ultralyticsyolov5a/detect.py --weights {MODEL_PATH} \\\n                  --source {TEST_PATH} \\\n                  --img {IMG_SIZE} \\\n                  --conf 0.281 \\\n                  --iou-thres 0.5 \\\n                  --max-det 3 \\\n                  --save-txt \\\n                  --save-conf \\\n                  --project {'detect'} \\\n                  --name {'exp'}","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:03.623832Z","iopub.execute_input":"2021-09-17T14:33:03.624346Z","iopub.status.idle":"2021-09-17T14:33:17.347916Z","shell.execute_reply.started":"2021-09-17T14:33:03.624313Z","shell.execute_reply":"2021-09-17T14:33:17.34697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntot_m, used_m, free_m = map(int, os.popen('free -t -m').readlines()[-1].split()[1:])\nprint('tot_m, used_m, free_m',tot_m, used_m, free_m )","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:17.35127Z","iopub.execute_input":"2021-09-17T14:33:17.351653Z","iopub.status.idle":"2021-09-17T14:33:17.376301Z","shell.execute_reply.started":"2021-09-17T14:33:17.351622Z","shell.execute_reply":"2021-09-17T14:33:17.375171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:17.379245Z","iopub.execute_input":"2021-09-17T14:33:17.379498Z","iopub.status.idle":"2021-09-17T14:33:17.385423Z","shell.execute_reply.started":"2021-09-17T14:33:17.37947Z","shell.execute_reply":"2021-09-17T14:33:17.384628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:17.387519Z","iopub.execute_input":"2021-09-17T14:33:17.387806Z","iopub.status.idle":"2021-09-17T14:33:17.396278Z","shell.execute_reply.started":"2021-09-17T14:33:17.38778Z","shell.execute_reply":"2021-09-17T14:33:17.395278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/siim-vgg19-vv3/json_model','r') as json_modelfile:\n    loaded_json_model=json_modelfile.read()\nloaded_model=model_from_json(loaded_json_model)\nloaded_model.load_weights('/kaggle/input/siim-vgg19-vv3/modelv2.h5')","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:17.39761Z","iopub.execute_input":"2021-09-17T14:33:17.397972Z","iopub.status.idle":"2021-09-17T14:33:21.364797Z","shell.execute_reply.started":"2021-09-17T14:33:17.397936Z","shell.execute_reply":"2021-09-17T14:33:21.363755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loaded_model","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:21.366231Z","iopub.execute_input":"2021-09-17T14:33:21.366638Z","iopub.status.idle":"2021-09-17T14:33:21.373843Z","shell.execute_reply.started":"2021-09-17T14:33:21.366584Z","shell.execute_reply":"2021-09-17T14:33:21.372856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#VGG_19=keras.models.load_model('../input/vgg19-v1/VGG19_model\"//\"saved_model.pb')","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:21.375206Z","iopub.execute_input":"2021-09-17T14:33:21.375816Z","iopub.status.idle":"2021-09-17T14:33:21.384367Z","shell.execute_reply.started":"2021-09-17T14:33:21.37578Z","shell.execute_reply":"2021-09-17T14:33:21.383387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start=time.time()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:21.385571Z","iopub.execute_input":"2021-09-17T14:33:21.386085Z","iopub.status.idle":"2021-09-17T14:33:21.392369Z","shell.execute_reply.started":"2021-09-17T14:33:21.386043Z","shell.execute_reply":"2021-09-17T14:33:21.391584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_predctn=loaded_model.predict(test_image_data)\nmodel_predctn.shape, model_predctn[:5]","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:21.393556Z","iopub.execute_input":"2021-09-17T14:33:21.393927Z","iopub.status.idle":"2021-09-17T14:33:41.595131Z","shell.execute_reply.started":"2021-09-17T14:33:21.393892Z","shell.execute_reply":"2021-09-17T14:33:41.59421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end=time.time()\nprint(f'Time taken in predictn of {len(test_image_data)} images is ', (end-start)/60., 'mins')\n","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.598291Z","iopub.execute_input":"2021-09-17T14:33:41.598569Z","iopub.status.idle":"2021-09-17T14:33:41.606994Z","shell.execute_reply.started":"2021-09-17T14:33:41.598539Z","shell.execute_reply":"2021-09-17T14:33:41.60612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_predctn_arg=np.argmax(model_predctn, axis=1)\nmodel_predctn_arg.shape, model_predctn_arg[:5]","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.608453Z","iopub.execute_input":"2021-09-17T14:33:41.608885Z","iopub.status.idle":"2021-09-17T14:33:41.616734Z","shell.execute_reply.started":"2021-09-17T14:33:41.608848Z","shell.execute_reply":"2021-09-17T14:33:41.615678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(model_predctn_arg)):\n    \n    predctn=np.array([[0,0,0,0]])\n    predctn[0][model_predctn_arg[i]]=1\n    \n    \n    if i == 0:\n        test_predctn = predctn\n        \n    else:\n        test_predctn=np.concatenate((test_predctn, predctn), axis=0)\n        \ntest_predctn.shape, test_predctn[:2]","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.618275Z","iopub.execute_input":"2021-09-17T14:33:41.618924Z","iopub.status.idle":"2021-09-17T14:33:41.629161Z","shell.execute_reply.started":"2021-09-17T14:33:41.618886Z","shell.execute_reply":"2021-09-17T14:33:41.628158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_predctn = pd.DataFrame(test_predctn, columns=['Negative for Pneumonia','Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance'])\ndf_test_predctn","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.632274Z","iopub.execute_input":"2021-09-17T14:33:41.632539Z","iopub.status.idle":"2021-09-17T14:33:41.648532Z","shell.execute_reply.started":"2021-09-17T14:33:41.632515Z","shell.execute_reply":"2021-09-17T14:33:41.6477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_path=pd.DataFrame(np_test_study, columns=['study_id','image_path', 'image_id'])\ndf_test_path.head(2), len(df_test_path)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.649836Z","iopub.execute_input":"2021-09-17T14:33:41.650208Z","iopub.status.idle":"2021-09-17T14:33:41.659812Z","shell.execute_reply.started":"2021-09-17T14:33:41.650183Z","shell.execute_reply":"2021-09-17T14:33:41.65879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_study_predctn=pd.concat([df_test_path, df_test_predctn], axis=1)# ignore_index=True, sort=False)\ndf_study_predctn.drop(columns=['image_path','image_id'], inplace=True)\ndf_study_predctn","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.661336Z","iopub.execute_input":"2021-09-17T14:33:41.661719Z","iopub.status.idle":"2021-09-17T14:33:41.685249Z","shell.execute_reply.started":"2021-09-17T14:33:41.661685Z","shell.execute_reply":"2021-09-17T14:33:41.684545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_study_predctn.drop_duplicates(subset='study_id', inplace=True)\ndf_study_predctn","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.686179Z","iopub.execute_input":"2021-09-17T14:33:41.686407Z","iopub.status.idle":"2021-09-17T14:33:41.708188Z","shell.execute_reply.started":"2021-09-17T14:33:41.686383Z","shell.execute_reply":"2021-09-17T14:33:41.707339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample_submission=pd.read_csv('/kaggle/input/siim-covid19-detection/sample_submission.csv')\ndf_sample_submission","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.709327Z","iopub.execute_input":"2021-09-17T14:33:41.709798Z","iopub.status.idle":"2021-09-17T14:33:41.728244Z","shell.execute_reply.started":"2021-09-17T14:33:41.70976Z","shell.execute_reply":"2021-09-17T14:33:41.727524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_study_path=pd.merge(df_sample_submission, df_study_predctn, how='inner', left_on='id', right_on='study_id')#, left_index=True, right_index=True )\ndf_test_study_path","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.729273Z","iopub.execute_input":"2021-09-17T14:33:41.729597Z","iopub.status.idle":"2021-09-17T14:33:41.753896Z","shell.execute_reply.started":"2021-09-17T14:33:41.729563Z","shell.execute_reply":"2021-09-17T14:33:41.752995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_study_path['PredictionString2'] = df_test_study_path['PredictionString'] \ndf_test_study_path.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.755105Z","iopub.execute_input":"2021-09-17T14:33:41.755441Z","iopub.status.idle":"2021-09-17T14:33:41.76898Z","shell.execute_reply.started":"2021-09-17T14:33:41.755406Z","shell.execute_reply":"2021-09-17T14:33:41.768123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_study_path.loc[df_test_study_path['Negative for Pneumonia'] == 1, 'PredictionString2'] = 'negative 1 0 0 1 1' \ndf_test_study_path.loc[df_test_study_path['Typical Appearance'] == 1, 'PredictionString2'] = 'typical 1 0 0 1 1' \ndf_test_study_path.loc[df_test_study_path['Indeterminate Appearance'] == 1, 'PredictionString2'] = 'indeterminate 1 0 0 1 1' \ndf_test_study_path.loc[df_test_study_path['Atypical Appearance'] == 1, 'PredictionString2'] = 'atypical 1 0 0 1 1' \ndf_test_study_path.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.77026Z","iopub.execute_input":"2021-09-17T14:33:41.770851Z","iopub.status.idle":"2021-09-17T14:33:41.790032Z","shell.execute_reply.started":"2021-09-17T14:33:41.770783Z","shell.execute_reply":"2021-09-17T14:33:41.789034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_study_path.drop(columns=['PredictionString', 'study_id', 'Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance'], inplace=True)\ndf_test_study_path","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.791402Z","iopub.execute_input":"2021-09-17T14:33:41.791788Z","iopub.status.idle":"2021-09-17T14:33:41.807929Z","shell.execute_reply.started":"2021-09-17T14:33:41.791751Z","shell.execute_reply":"2021-09-17T14:33:41.806986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_study_path.rename(columns={\"PredictionString2\": \"PredictionString\"}, inplace=True)\ndf_test_study_path","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.809366Z","iopub.execute_input":"2021-09-17T14:33:41.809759Z","iopub.status.idle":"2021-09-17T14:33:41.824591Z","shell.execute_reply.started":"2021-09-17T14:33:41.809724Z","shell.execute_reply":"2021-09-17T14:33:41.823798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n!pip install -q --upgrade wandb\n# Login \nimport wandb\nwandb.login()\n'''","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.82773Z","iopub.execute_input":"2021-09-17T14:33:41.827999Z","iopub.status.idle":"2021-09-17T14:33:41.833371Z","shell.execute_reply.started":"2021-09-17T14:33:41.827967Z","shell.execute_reply":"2021-09-17T14:33:41.832351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nprint(f\"Setup complete. Using torch {torch.__version__} ({torch.cuda.get_device_properties(0).name if torch.cuda.is_available() else 'CPU'})\")\n\n# Necessary/extra dependencies. \nimport os\nimport gc\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom shutil import copyfile\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport re\n\n#customize iPython writefile so we can write variables\nfrom IPython.core.magic import register_line_cell_magic\n\n@register_line_cell_magic\ndef writetemplate(line, cell):\n    with open(line, 'w') as f:\n        f.write(cell.format(**globals()))","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:41.83492Z","iopub.execute_input":"2021-09-17T14:33:41.835267Z","iopub.status.idle":"2021-09-17T14:33:42.194955Z","shell.execute_reply.started":"2021-09-17T14:33:41.835232Z","shell.execute_reply":"2021-09-17T14:33:42.193328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TEST_PATH = '/kaggle/tmp/test_images' # absolute path","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.196322Z","iopub.execute_input":"2021-09-17T14:33:42.196711Z","iopub.status.idle":"2021-09-17T14:33:42.200739Z","shell.execute_reply.started":"2021-09-17T14:33:42.19667Z","shell.execute_reply":"2021-09-17T14:33:42.19964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#MODEL_PATH = '/kaggle/input/bestpt2/best.pt'","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.202263Z","iopub.execute_input":"2021-09-17T14:33:42.202906Z","iopub.status.idle":"2021-09-17T14:33:42.209743Z","shell.execute_reply.started":"2021-09-17T14:33:42.202865Z","shell.execute_reply":"2021-09-17T14:33:42.208833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os.mkdir('./detect')","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.211276Z","iopub.execute_input":"2021-09-17T14:33:42.211736Z","iopub.status.idle":"2021-09-17T14:33:42.218214Z","shell.execute_reply.started":"2021-09-17T14:33:42.211608Z","shell.execute_reply":"2021-09-17T14:33:42.21731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os.mkdir('./detect/exp')","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.220901Z","iopub.execute_input":"2021-09-17T14:33:42.221164Z","iopub.status.idle":"2021-09-17T14:33:42.22666Z","shell.execute_reply.started":"2021-09-17T14:33:42.221139Z","shell.execute_reply":"2021-09-17T14:33:42.225847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start=time.time()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.228064Z","iopub.execute_input":"2021-09-17T14:33:42.228468Z","iopub.status.idle":"2021-09-17T14:33:42.234882Z","shell.execute_reply.started":"2021-09-17T14:33:42.228431Z","shell.execute_reply":"2021-09-17T14:33:42.234031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.236343Z","iopub.execute_input":"2021-09-17T14:33:42.236881Z","iopub.status.idle":"2021-09-17T14:33:42.90827Z","shell.execute_reply.started":"2021-09-17T14:33:42.236705Z","shell.execute_reply":"2021-09-17T14:33:42.907401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nwith open('../input/requirements/requirements.txt') as req_txt:\n    k= req_txt.read()\n    print(k)\n    with open('/kaggle/working/requirements.txt', 'w') as txtfile:\n        txtfile.write(k)\n'''","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.91118Z","iopub.execute_input":"2021-09-17T14:33:42.911443Z","iopub.status.idle":"2021-09-17T14:33:42.917178Z","shell.execute_reply.started":"2021-09-17T14:33:42.911414Z","shell.execute_reply":"2021-09-17T14:33:42.916145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.918919Z","iopub.execute_input":"2021-09-17T14:33:42.919587Z","iopub.status.idle":"2021-09-17T14:33:42.927081Z","shell.execute_reply.started":"2021-09-17T14:33:42.919548Z","shell.execute_reply":"2021-09-17T14:33:42.926021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n!python /kaggle/input/ultralyticsyolov5a/detect.py --weights {MODEL_PATH} \\\n                  --source {TEST_PATH} \\\n                  --img {IMG_SIZE} \\\n                  --conf 0.281 \\\n                  --iou-thres 0.5 \\\n                  --max-det 3 \\\n                  --save-txt \\\n                  --save-conf \\\n                  --project {'detect'} \\\n                  --name {'exp'}\n'''","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.928573Z","iopub.execute_input":"2021-09-17T14:33:42.928991Z","iopub.status.idle":"2021-09-17T14:33:42.936035Z","shell.execute_reply.started":"2021-09-17T14:33:42.928946Z","shell.execute_reply":"2021-09-17T14:33:42.935036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end=time.time()\nprint(f'Time taken in object detectn of {len(test_image_data)} images is ', (end-start)/60., 'mins')\n","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.937777Z","iopub.execute_input":"2021-09-17T14:33:42.938362Z","iopub.status.idle":"2021-09-17T14:33:42.945743Z","shell.execute_reply.started":"2021-09-17T14:33:42.938323Z","shell.execute_reply":"2021-09-17T14:33:42.94466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname,_,filenames in os.walk('/kaggle/working/detect/'):\n    #print('dirname',dirname,'_',_,)#'filenames ',filenames)\n    if _ ==['labels'] :\n        exp=dirname\n        labels_path=os.path.join(exp, 'labels')\n        print('labels_path',labels_path)\n        image_dir=dirname\n        print('image_dir', image_dir)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.947397Z","iopub.execute_input":"2021-09-17T14:33:42.947866Z","iopub.status.idle":"2021-09-17T14:33:42.9569Z","shell.execute_reply.started":"2021-09-17T14:33:42.947832Z","shell.execute_reply":"2021-09-17T14:33:42.955835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PRED_PATH = labels_path\nprint(PRED_PATH)\n!ls {PRED_PATH}","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:42.958461Z","iopub.execute_input":"2021-09-17T14:33:42.959007Z","iopub.status.idle":"2021-09-17T14:33:43.636504Z","shell.execute_reply.started":"2021-09-17T14:33:42.95896Z","shell.execute_reply":"2021-09-17T14:33:43.63535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname,_,filenames in os.walk(PRED_PATH):\n    print(dirname, _)\n    i=0\n    for filename in filenames:\n        \n        text_file=filename\n        image_id=re.sub('.txt', '_image', text_file)\n        #print(i)\n        with open(os.path.join(dirname, text_file)) as txt_file:\n            predictn=txt_file.read()\n            #print(predictn, type(predictn))\n            predictn_np=re.split(' |\\n', predictn)\n            #print(predictn_np)\n            predictn_img=[image_id, predictn_np]\n            if i==0:\n                predictn_array=[predictn_img]\n            else:\n                predictn_array.append(predictn_img)\n        i+=1\nprint(len(predictn_array), predictn_array[:5])","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.637961Z","iopub.execute_input":"2021-09-17T14:33:43.64256Z","iopub.status.idle":"2021-09-17T14:33:43.662051Z","shell.execute_reply.started":"2021-09-17T14:33:43.642512Z","shell.execute_reply":"2021-09-17T14:33:43.660894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.666879Z","iopub.execute_input":"2021-09-17T14:33:43.669559Z","iopub.status.idle":"2021-09-17T14:33:43.681187Z","shell.execute_reply.started":"2021-09-17T14:33:43.669516Z","shell.execute_reply":"2021-09-17T14:33:43.679969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(predictn_array)):\n    x_c1=float(predictn_array[i][1][1])\n    y_c1=float(predictn_array[i][1][2])\n    w1=float(predictn_array[i][1][3])\n    h1=float(predictn_array[i][1][4])\n    \n    x1_min=int((x_c1-w1/2)*IMG_SIZE)\n    x1_max=int((x_c1+w1/2)*IMG_SIZE)\n\n    y1_min=int((y_c1-h1/2)*IMG_SIZE)\n    y1_max=int((y_c1+h1/2)*IMG_SIZE)\n\n\n    if len(predictn_array[i][1])>7:\n        x_c2=float(predictn_array[i][1][7])\n        y_c2=float(predictn_array[i][1][8])\n        w2=float(predictn_array[i][1][9])\n        h2=float(predictn_array[i][1][10])\n        \n        x2_min=int((x_c2-w2/2)*IMG_SIZE)\n        x2_max=int((x_c2+w2/2)*IMG_SIZE)\n        \n        y2_min=int((y_c2-h2/2)*IMG_SIZE)\n        y2_max=int((y_c2+h2/2)*IMG_SIZE)\n        #Write confidence score\n        \n        bb=['opacity', float(predictn_array[i][1][5]),  x1_min, y1_min, x1_max, y1_max, 'opacity', float(predictn_array[i][1][11]),  x2_min, y2_min, x2_max, y2_max   ]\n        \n        bb_f=[predictn_array[i][0], ' '.join(str(b) for b in bb) ]\n\n\n    else:\n        bb=['opacity', float(predictn_array[i][1][5]),  x1_min, y1_min, x1_max, y1_max  ]\n        bb_f=[ predictn_array[i][0], ' '.join(str(b) for b in bb)]\n    \n\n    if i==0:\n        bb_arr=[bb_f]\n    else:\n        #np_label_norm.append(norm)\n        bb_arr.append(bb_f)\n\n    \n#np_label_norm[i][0]=norm\n\n#print(np_label_norm.shape, np_label_norm)\n#np_label_norm=np_label_norm.reshape()\n#np_label_norm=np_label_norm.reshape([len(np_label_norm),]) \nlen(bb_arr), bb_arr[:5]","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.685602Z","iopub.execute_input":"2021-09-17T14:33:43.687701Z","iopub.status.idle":"2021-09-17T14:33:43.718485Z","shell.execute_reply.started":"2021-09-17T14:33:43.687658Z","shell.execute_reply":"2021-09-17T14:33:43.717462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Checks\nfor i in range(len(bb_arr)):\n    \n    bb=re.split(' ' ,bb_arr[i][1])\n    \n    x1_min=int(bb[2])\n    y1_min=int(bb[3])\n    x1_max=int(bb[4])\n    y1_max=int(bb[5])\n    \n    if x1_min > x1_max or y1_min > y1_max or x1_min<=0 or x1_max<=0 or y1_min<=0 or y1_max<=0 or x1_min>608 or x1_max>608 or y1_min>608 or y1_max>608:\n        print(i, bb)\n        \n    \n    \n    if len(bb)>8:\n        x2_min=int(bb[8])\n        y2_min=int(bb[9])\n        x2_max=int(bb[10])\n        y2_max=int(bb[11])\n        \n        if x2_min > x2_max or y2_min > y2_max or x2_min<=0 or x2_max<=0 or y2_min<=0 or y2_max<=0 or x2_min>608 or x2_max>608 or y2_min>608 or y2_max>608:\n            print(i,bb)\n            \n    if i%1000==0:\n        print(i)\n    \n\n        \n        \n","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.722439Z","iopub.execute_input":"2021-09-17T14:33:43.722776Z","iopub.status.idle":"2021-09-17T14:33:43.741175Z","shell.execute_reply.started":"2021-09-17T14:33:43.722743Z","shell.execute_reply":"2021-09-17T14:33:43.73995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_bb=pd.DataFrame(bb_arr, columns=['id', 'PredictionString2'])\ndf_bb['img_id']=1\ndf_bb","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.745348Z","iopub.execute_input":"2021-09-17T14:33:43.745686Z","iopub.status.idle":"2021-09-17T14:33:43.772651Z","shell.execute_reply.started":"2021-09-17T14:33:43.745653Z","shell.execute_reply":"2021-09-17T14:33:43.771673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_check=df_sample_submission[df_sample_submission['id'].str.contains('82589da89e9')]\ndf_check","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.783474Z","iopub.execute_input":"2021-09-17T14:33:43.783814Z","iopub.status.idle":"2021-09-17T14:33:43.804218Z","shell.execute_reply.started":"2021-09-17T14:33:43.783782Z","shell.execute_reply":"2021-09-17T14:33:43.80328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_smpl_subm_img=df_sample_submission[df_sample_submission['id'].str.contains('_image')]\ndf_smpl_subm_img['img_id']=0\ndf_smpl_subm_img","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.80621Z","iopub.execute_input":"2021-09-17T14:33:43.806548Z","iopub.status.idle":"2021-09-17T14:33:43.83109Z","shell.execute_reply.started":"2021-09-17T14:33:43.806513Z","shell.execute_reply":"2021-09-17T14:33:43.830336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm_img=pd.merge(df_smpl_subm_img, df_bb, how='left',  left_on='id', right_on='id')#, left_index=True, right_index=True )\ndf_subm_img","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.832165Z","iopub.execute_input":"2021-09-17T14:33:43.836125Z","iopub.status.idle":"2021-09-17T14:33:43.867086Z","shell.execute_reply.started":"2021-09-17T14:33:43.836087Z","shell.execute_reply":"2021-09-17T14:33:43.866194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_check=df_subm_img[df_subm_img['img_id_y']==1]\ndf_check","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.870701Z","iopub.execute_input":"2021-09-17T14:33:43.872744Z","iopub.status.idle":"2021-09-17T14:33:43.903687Z","shell.execute_reply.started":"2021-09-17T14:33:43.872702Z","shell.execute_reply":"2021-09-17T14:33:43.902789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm_img.loc[df_subm_img['img_id_y']==1, 'PredictionString3']  = df_subm_img['PredictionString2']\ndf_subm_img.loc[df_subm_img['img_id_y']!=1, 'PredictionString3']  = df_subm_img['PredictionString']\ndf_subm_img","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.907192Z","iopub.execute_input":"2021-09-17T14:33:43.909296Z","iopub.status.idle":"2021-09-17T14:33:43.941003Z","shell.execute_reply.started":"2021-09-17T14:33:43.909257Z","shell.execute_reply":"2021-09-17T14:33:43.940266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_check=df_subm_img[df_subm_img['PredictionString3'].str.contains('opacity')]\ndf_check","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.944761Z","iopub.execute_input":"2021-09-17T14:33:43.947086Z","iopub.status.idle":"2021-09-17T14:33:43.982685Z","shell.execute_reply.started":"2021-09-17T14:33:43.947045Z","shell.execute_reply":"2021-09-17T14:33:43.98191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm_img.drop(columns=[\"PredictionString\",\"img_id_x\",\"PredictionString2\",\"img_id_y\"], inplace=True)\ndf_subm_img","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:43.986181Z","iopub.execute_input":"2021-09-17T14:33:43.988235Z","iopub.status.idle":"2021-09-17T14:33:44.006601Z","shell.execute_reply.started":"2021-09-17T14:33:43.988195Z","shell.execute_reply":"2021-09-17T14:33:44.005739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm_img.rename(columns={\"PredictionString3\": \"PredictionString\"}, errors=\"raise\", inplace=True)\ndf_subm_img","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:44.01077Z","iopub.execute_input":"2021-09-17T14:33:44.012687Z","iopub.status.idle":"2021-09-17T14:33:44.031658Z","shell.execute_reply.started":"2021-09-17T14:33:44.012646Z","shell.execute_reply":"2021-09-17T14:33:44.03069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df_test_study_path.append(df_subm_img)\ndf","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:44.035683Z","iopub.execute_input":"2021-09-17T14:33:44.037674Z","iopub.status.idle":"2021-09-17T14:33:44.055962Z","shell.execute_reply.started":"2021-09-17T14:33:44.037629Z","shell.execute_reply":"2021-09-17T14:33:44.055206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm_array=df.to_numpy()\nsubm_array","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:44.059269Z","iopub.execute_input":"2021-09-17T14:33:44.059744Z","iopub.status.idle":"2021-09-17T14:33:44.073847Z","shell.execute_reply.started":"2021-09-17T14:33:44.059711Z","shell.execute_reply":"2021-09-17T14:33:44.07303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tot_m, used_m, free_m = map(int, os.popen('free -t -m').readlines()[-1].split()[1:])\nprint('tot_m, used_m, free_m',tot_m, used_m, free_m )","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:44.077741Z","iopub.execute_input":"2021-09-17T14:33:44.078283Z","iopub.status.idle":"2021-09-17T14:33:44.134804Z","shell.execute_reply.started":"2021-09-17T14:33:44.078245Z","shell.execute_reply":"2021-09-17T14:33:44.133967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:44.136265Z","iopub.execute_input":"2021-09-17T14:33:44.13661Z","iopub.status.idle":"2021-09-17T14:33:44.810808Z","shell.execute_reply.started":"2021-09-17T14:33:44.136573Z","shell.execute_reply":"2021-09-17T14:33:44.809794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree('/kaggle/tmp/test_images')","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:44.814009Z","iopub.execute_input":"2021-09-17T14:33:44.81429Z","iopub.status.idle":"2021-09-17T14:33:44.825975Z","shell.execute_reply.started":"2021-09-17T14:33:44.814261Z","shell.execute_reply":"2021-09-17T14:33:44.825015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree(image_dir)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:44.829102Z","iopub.execute_input":"2021-09-17T14:33:44.829356Z","iopub.status.idle":"2021-09-17T14:33:44.840362Z","shell.execute_reply.started":"2021-09-17T14:33:44.829331Z","shell.execute_reply":"2021-09-17T14:33:44.83957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:44.843034Z","iopub.execute_input":"2021-09-17T14:33:44.843301Z","iopub.status.idle":"2021-09-17T14:33:45.512414Z","shell.execute_reply.started":"2021-09-17T14:33:44.843276Z","shell.execute_reply":"2021-09-17T14:33:45.511437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tot_m, used_m, free_m = map(int, os.popen('free -t -m').readlines()[-1].split()[1:])\nprint('tot_m, used_m, free_m',tot_m, used_m, free_m )","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:45.51568Z","iopub.execute_input":"2021-09-17T14:33:45.515939Z","iopub.status.idle":"2021-09-17T14:33:45.566648Z","shell.execute_reply.started":"2021-09-17T14:33:45.515911Z","shell.execute_reply":"2021-09-17T14:33:45.565586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fields=['id', 'PredictionString']\nwith open('/kaggle/working/submission.csv', 'w') as f:\n      \n    # using csv.writer method from CSV package\n    write = csv.writer(f)\n    \n    write.writerow(fields)\n    write.writerows(subm_array)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:45.569676Z","iopub.execute_input":"2021-09-17T14:33:45.569954Z","iopub.status.idle":"2021-09-17T14:33:45.583371Z","shell.execute_reply.started":"2021-09-17T14:33:45.569916Z","shell.execute_reply":"2021-09-17T14:33:45.582652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cd detect","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:45.584598Z","iopub.execute_input":"2021-09-17T14:33:45.584962Z","iopub.status.idle":"2021-09-17T14:33:45.589109Z","shell.execute_reply.started":"2021-09-17T14:33:45.584926Z","shell.execute_reply":"2021-09-17T14:33:45.588116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ls","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:45.590757Z","iopub.execute_input":"2021-09-17T14:33:45.591284Z","iopub.status.idle":"2021-09-17T14:33:45.597252Z","shell.execute_reply.started":"2021-09-17T14:33:45.591246Z","shell.execute_reply":"2021-09-17T14:33:45.59624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cd exp3","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:45.598555Z","iopub.execute_input":"2021-09-17T14:33:45.599013Z","iopub.status.idle":"2021-09-17T14:33:45.605445Z","shell.execute_reply.started":"2021-09-17T14:33:45.598968Z","shell.execute_reply":"2021-09-17T14:33:45.604682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ls","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:45.606693Z","iopub.execute_input":"2021-09-17T14:33:45.60709Z","iopub.status.idle":"2021-09-17T14:33:45.613848Z","shell.execute_reply.started":"2021-09-17T14:33:45.607054Z","shell.execute_reply":"2021-09-17T14:33:45.613012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_check=pd.read_csv('/kaggle/working/submission.csv')\ndf_check","metadata":{"execution":{"iopub.status.busy":"2021-09-17T14:33:45.616612Z","iopub.execute_input":"2021-09-17T14:33:45.616902Z","iopub.status.idle":"2021-09-17T14:33:45.637012Z","shell.execute_reply.started":"2021-09-17T14:33:45.616866Z","shell.execute_reply":"2021-09-17T14:33:45.636157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}