{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! conda install -c conda-forge gdcm -y","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-05T14:34:22.019374Z","iopub.execute_input":"2021-06-05T14:34:22.019982Z","iopub.status.idle":"2021-06-05T14:35:08.508838Z","shell.execute_reply.started":"2021-06-05T14:34:22.019793Z","shell.execute_reply":"2021-06-05T14:35:08.507665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom ast import literal_eval\nimport json","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-05T14:35:08.510797Z","iopub.execute_input":"2021-06-05T14:35:08.511128Z","iopub.status.idle":"2021-06-05T14:35:08.516453Z","shell.execute_reply.started":"2021-06-05T14:35:08.511093Z","shell.execute_reply":"2021-06-05T14:35:08.515014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extract_path = \"/kaggle/input/siim-covid-512-reshaped-corrected-bounded-box/Extracted_Study_Series_Img.csv\"\ndf = pd.read_csv(extract_path)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:08.518798Z","iopub.execute_input":"2021-06-05T14:35:08.519088Z","iopub.status.idle":"2021-06-05T14:35:08.71317Z","shell.execute_reply.started":"2021-06-05T14:35:08.519059Z","shell.execute_reply":"2021-06-05T14:35:08.712417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.tail()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:08.71447Z","iopub.execute_input":"2021-06-05T14:35:08.714914Z","iopub.status.idle":"2021-06-05T14:35:08.735666Z","shell.execute_reply.started":"2021-06-05T14:35:08.714868Z","shell.execute_reply":"2021-06-05T14:35:08.734833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Resized Data Paths","metadata":{}},{"cell_type":"code","source":"import os\ndata_dir = \"/kaggle/input/siim-covid-512-reshaped-corrected-bounded-box/resized_data/resized_data\"\nNew_Image_Path = []\nfor index, row in df.iterrows():\n    img_path = row[\"Image_Name\"]\n    curr_set = row[\"Set_Name\"]\n    png_path = img_path.replace('.dcm','.png').zfill(16)\n    new_path = os.path.join(data_dir,curr_set,png_path)\n    New_Image_Path.append(new_path)\ndf[\"New_Image_Path\"] = New_Image_Path","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:08.736928Z","iopub.execute_input":"2021-06-05T14:35:08.737232Z","iopub.status.idle":"2021-06-05T14:35:09.504911Z","shell.execute_reply.started":"2021-06-05T14:35:08.737203Z","shell.execute_reply":"2021-06-05T14:35:09.503479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(\"0026720152f5.png\")","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:09.506436Z","iopub.execute_input":"2021-06-05T14:35:09.506844Z","iopub.status.idle":"2021-06-05T14:35:09.518191Z","shell.execute_reply.started":"2021-06-05T14:35:09.506799Z","shell.execute_reply":"2021-06-05T14:35:09.516985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = 20\ntrain_df = df[df[\"Set_Name\"] == \"train\"]\nsample_train = train_df.sample(n)\nsample_train.reset_index(inplace = True)\n\ntest_df = df[df[\"Set_Name\"] == \"test\"]\nsample_test = test_df.sample(n)\nsample_test.reset_index(inplace = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:09.520249Z","iopub.execute_input":"2021-06-05T14:35:09.520662Z","iopub.status.idle":"2021-06-05T14:35:09.547604Z","shell.execute_reply.started":"2021-06-05T14:35:09.520619Z","shell.execute_reply":"2021-06-05T14:35:09.546247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:09.551524Z","iopub.execute_input":"2021-06-05T14:35:09.552071Z","iopub.status.idle":"2021-06-05T14:35:09.579427Z","shell.execute_reply.started":"2021-06-05T14:35:09.552021Z","shell.execute_reply":"2021-06-05T14:35:09.578434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\n%matplotlib inline\nimport cv2\nplt.style.use(\"dark_background\")","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:09.581321Z","iopub.execute_input":"2021-06-05T14:35:09.581638Z","iopub.status.idle":"2021-06-05T14:35:09.656169Z","shell.execute_reply.started":"2021-06-05T14:35:09.581606Z","shell.execute_reply":"2021-06-05T14:35:09.655131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom as dicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport PIL\nfrom PIL import Image\nfrom colorama import Fore, Back, Style\nviz_counter=0\n\ndef create_dir(dir, v=1):\n    \"\"\"\n    Creates a directory without throwing an error if directory already exists.\n    dir : The directory to be created.\n    v : Verbosity\n    \"\"\"\n    if not os.path.exists(dir):\n        os.makedirs(dir)\n        if v:\n            print(\"Created Directory : \", dir)\n        return 1\n    else:\n        if v:\n            print(\"Directory already existed : \", dir)\n        return 0\n\nvoi_lut=True\nfix_monochrome=True\n\ndef dicom_dataset_to_dict(filename):\n    \"\"\"Credit: https://github.com/pydicom/pydicom/issues/319\n               https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    \"\"\"\n    \n    dicom_header = dicom.dcmread(filename) \n    \n    #====== DICOM FILE DATA ======\n    dicom_dict = {}\n    repr(dicom_header)\n    for dicom_value in dicom_header.values():\n        if dicom_value.tag == (0x7fe0, 0x0010):\n            #discard pixel data\n            continue\n        if type(dicom_value.value) == dicom.dataset.Dataset:\n            dicom_dict[dicom_value.name] = dicom_dataset_to_dict(dicom_value.value)\n        else:\n            v = _convert_value(dicom_value.value)\n            dicom_dict[dicom_value.name] = v\n      \n    del dicom_dict['Pixel Representation']\n    \n    #====== DICOM IMAGE DATA ======\n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom_header.pixel_array, dicom_header)\n    else:\n        data = dicom_header.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom_header.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    modified_image_data = (data * 255).astype(np.uint8)\n    \n    return dicom_dict, modified_image_data\n\ndef _sanitise_unicode(s):\n    return s.replace(u\"\\u0000\", \"\").strip()\n\ndef _convert_value(v):\n    t = type(v)\n    if t in (list, int, float):\n        cv = v\n    elif t == str:\n        cv = _sanitise_unicode(v)\n    elif t == bytes:\n        s = v.decode('ascii', 'replace')\n        cv = _sanitise_unicode(s)\n    elif t == dicom.valuerep.DSfloat:\n        cv = float(v)\n    elif t == dicom.valuerep.IS:\n        cv = int(v)\n    else:\n        cv = repr(v)\n    return cv\n\"\"\"\ndef rgb2gray(rgb):\n    return np.dot(rgb[...,:3], [0.2989, 0.5870, 0.1140])\n\"\"\"\ndef view_data(sample_train,view=\"original\"):\n    if view==\"resized\":\n        fig, axs = plt.subplots(4, 5, figsize=(20,20))\n        fig.subplots_adjust(hspace=.2, wspace=.2)\n        axs = axs.ravel()\n        for i in range(n):\n            img = cv2.imread(sample_train['New_Image_Path'][i])\n            axs[i].imshow(img,cmap='gray')\n            if type(sample_train['corrected_boxes'][i])==str and (not sample_train['corrected_boxes'][i]==\"\") and (not sample_train['corrected_boxes'][i]==np.nan) :\n                boxes = literal_eval(sample_train['corrected_boxes'][i])\n                for box in boxes:\n                    axs[i].add_patch(Rectangle((box['x'], box['y']), box['width'], box['height'], fill=0, color='y', linewidth=2))\n                axs[i].set_title(sample_train['study_label'][i])\n            else:\n                axs[i].set_title(sample_train['study_label'][i])\n    if view==\"original\":\n        fig, axs = plt.subplots(4, 5, figsize=(20,20))\n        fig.subplots_adjust(hspace=.2, wspace=.2)\n        axs = axs.ravel()\n        for i in range(n):\n            fpath = sample_train['Image_Path'][i]\n            dicom_dict, modified_image_data = dicom_dataset_to_dict(fpath)\n            # print(modified_image_data.shape)\n            # rgb2gray = lambda rgb : np.dot(rgb[... , :3] , [0.299 , 0.587, 0.114])   \n            img = modified_image_data\n            axs[i].imshow(img,cmap='gray')\n            if type(sample_train['boxes'][i])==str and (not sample_train['boxes'][i]==\"\") and (not sample_train['boxes'][i]==np.nan):\n                boxes = literal_eval(sample_train['boxes'][i])\n                for box in boxes:\n                    axs[i].add_patch(Rectangle((box['x'], box['y']), box['width'], box['height'], fill=0, color='y', linewidth=2))\n                axs[i].set_title(sample_train['study_label'][i])\n            else:\n                axs[i].set_title(sample_train['study_label'][i])\n        ","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:09.657808Z","iopub.execute_input":"2021-06-05T14:35:09.658209Z","iopub.status.idle":"2021-06-05T14:35:09.849651Z","shell.execute_reply.started":"2021-06-05T14:35:09.658166Z","shell.execute_reply":"2021-06-05T14:35:09.848481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# View Original Train Data","metadata":{}},{"cell_type":"code","source":"view_data(sample_train,view=\"original\")","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:09.850918Z","iopub.execute_input":"2021-06-05T14:35:09.851232Z","iopub.status.idle":"2021-06-05T14:35:32.041232Z","shell.execute_reply.started":"2021-06-05T14:35:09.851201Z","shell.execute_reply":"2021-06-05T14:35:32.040012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# View Resized Train Data","metadata":{}},{"cell_type":"code","source":"view_data(sample_train,view=\"resized\")","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:32.042433Z","iopub.execute_input":"2021-06-05T14:35:32.042847Z","iopub.status.idle":"2021-06-05T14:35:35.408448Z","shell.execute_reply.started":"2021-06-05T14:35:32.042807Z","shell.execute_reply":"2021-06-05T14:35:35.407387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Now go ahead and build some awesome Object Detection Models!\n\nHurray! It displays the same data! Also let's try to build a mock submission file that takes in predictions from a random model and then create the `submission.csv` file. ","metadata":{}},{"cell_type":"code","source":"sample_test.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:35.410165Z","iopub.execute_input":"2021-06-05T14:35:35.41056Z","iopub.status.idle":"2021-06-05T14:35:35.434833Z","shell.execute_reply.started":"2021-06-05T14:35:35.410522Z","shell.execute_reply":"2021-06-05T14:35:35.433963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"view_data(sample_test,view=\"resized\")","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:35.436084Z","iopub.execute_input":"2021-06-05T14:35:35.436498Z","iopub.status.idle":"2021-06-05T14:35:38.517056Z","shell.execute_reply.started":"2021-06-05T14:35:35.436435Z","shell.execute_reply":"2021-06-05T14:35:38.516029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.tail(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:38.518335Z","iopub.execute_input":"2021-06-05T14:35:38.518624Z","iopub.status.idle":"2021-06-05T14:35:38.539906Z","shell.execute_reply.started":"2021-06-05T14:35:38.518596Z","shell.execute_reply":"2021-06-05T14:35:38.538812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install openpyxl","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-05T14:35:38.541105Z","iopub.execute_input":"2021-06-05T14:35:38.541382Z","iopub.status.idle":"2021-06-05T14:35:45.172938Z","shell.execute_reply.started":"2021-06-05T14:35:38.541355Z","shell.execute_reply":"2021-06-05T14:35:45.171781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('Resized_MetaData.csv',index=False)\ndf.to_excel('Resized_MetaData.xlsx',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:45.174727Z","iopub.execute_input":"2021-06-05T14:35:45.175056Z","iopub.status.idle":"2021-06-05T14:35:50.09111Z","shell.execute_reply.started":"2021-06-05T14:35:45.175023Z","shell.execute_reply":"2021-06-05T14:35:50.09Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def subtract_lists(x,y):\n    \"\"\"Subtract Two Lists (List Difference)\"\"\"\n    return [item for item in x if item not in y]\ndef merge_list_to_dict(test_keys,test_values):\n    \"\"\"Using dictionary comprehension to merge two lists to dictionary\"\"\"\n    merged_dict = {test_keys[i]: test_values[i] for i in range(len(test_keys))}\n    return merged_dict\n# CLASSES = subtract_lists(list(set(df[\"study_label\"])),[np.nan])\nCLASSES = ['negative','indeterminate', 'typical',  'atypical'] # keep negative at start","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.092675Z","iopub.execute_input":"2021-06-05T14:35:50.093131Z","iopub.status.idle":"2021-06-05T14:35:50.099422Z","shell.execute_reply.started":"2021-06-05T14:35:50.093082Z","shell.execute_reply":"2021-06-05T14:35:50.098424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IMAGE_LABELS = subtract_lists(list(set(df[\"image_label\"])),[np.nan])\nIMAGE_LABELS = ['none','opacity']","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.100941Z","iopub.execute_input":"2021-06-05T14:35:50.101515Z","iopub.status.idle":"2021-06-05T14:35:50.155033Z","shell.execute_reply.started":"2021-06-05T14:35:50.101469Z","shell.execute_reply":"2021-06-05T14:35:50.154053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(42)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.156474Z","iopub.execute_input":"2021-06-05T14:35:50.157133Z","iopub.status.idle":"2021-06-05T14:35:50.172806Z","shell.execute_reply.started":"2021-06-05T14:35:50.157086Z","shell.execute_reply":"2021-06-05T14:35:50.171435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Dummy Predictions","metadata":{}},{"cell_type":"code","source":"sub_pth = \"../input/siim-covid19-detection/sample_submission.csv\"\ndf_sub = pd.read_csv(sub_pth)\ndf_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.174013Z","iopub.execute_input":"2021-06-05T14:35:50.174432Z","iopub.status.idle":"2021-06-05T14:35:50.205827Z","shell.execute_reply.started":"2021-06-05T14:35:50.174402Z","shell.execute_reply":"2021-06-05T14:35:50.204792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.tail()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.207397Z","iopub.execute_input":"2021-06-05T14:35:50.207722Z","iopub.status.idle":"2021-06-05T14:35:50.216914Z","shell.execute_reply.started":"2021-06-05T14:35:50.207686Z","shell.execute_reply":"2021-06-05T14:35:50.216123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx_img = []\nidx_std = []\n\nfor index, row in df_sub.iterrows():\n    if row[\"id\"].endswith('_image'):\n        idx_img.append(int(index))\n    if row[\"id\"].endswith('_study'):\n        idx_std.append(int(index))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.221288Z","iopub.execute_input":"2021-06-05T14:35:50.221846Z","iopub.status.idle":"2021-06-05T14:35:50.461996Z","shell.execute_reply.started":"2021-06-05T14:35:50.221699Z","shell.execute_reply":"2021-06-05T14:35:50.460359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(idx_img)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.464343Z","iopub.execute_input":"2021-06-05T14:35:50.464677Z","iopub.status.idle":"2021-06-05T14:35:50.470484Z","shell.execute_reply.started":"2021-06-05T14:35:50.464646Z","shell.execute_reply":"2021-06-05T14:35:50.469354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(idx_std)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.472094Z","iopub.execute_input":"2021-06-05T14:35:50.472426Z","iopub.status.idle":"2021-06-05T14:35:50.489809Z","shell.execute_reply.started":"2021-06-05T14:35:50.472394Z","shell.execute_reply":"2021-06-05T14:35:50.488749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.491238Z","iopub.execute_input":"2021-06-05T14:35:50.491523Z","iopub.status.idle":"2021-06-05T14:35:50.509322Z","shell.execute_reply.started":"2021-06-05T14:35:50.491495Z","shell.execute_reply":"2021-06-05T14:35:50.508229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub_img = df_sub.iloc[idx_img]\ndf_sub_std = df_sub.iloc[idx_std]","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.510615Z","iopub.execute_input":"2021-06-05T14:35:50.510943Z","iopub.status.idle":"2021-06-05T14:35:50.52622Z","shell.execute_reply.started":"2021-06-05T14:35:50.510911Z","shell.execute_reply":"2021-06-05T14:35:50.524931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub_img.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.527462Z","iopub.execute_input":"2021-06-05T14:35:50.527836Z","iopub.status.idle":"2021-06-05T14:35:50.54917Z","shell.execute_reply.started":"2021-06-05T14:35:50.527803Z","shell.execute_reply":"2021-06-05T14:35:50.547992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.550362Z","iopub.execute_input":"2021-06-05T14:35:50.550825Z","iopub.status.idle":"2021-06-05T14:35:50.582882Z","shell.execute_reply.started":"2021-06-05T14:35:50.550777Z","shell.execute_reply":"2021-06-05T14:35:50.581823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub_img[\"Image_ID\"] = df_sub_img[\"id\"].str.replace(\"_image\",\"\")","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.584361Z","iopub.execute_input":"2021-06-05T14:35:50.584979Z","iopub.status.idle":"2021-06-05T14:35:50.60517Z","shell.execute_reply.started":"2021-06-05T14:35:50.584931Z","shell.execute_reply":"2021-06-05T14:35:50.60299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_img = df_sub_img.merge(df,on=\"Image_ID\")\ndf_img.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.6066Z","iopub.execute_input":"2021-06-05T14:35:50.607083Z","iopub.status.idle":"2021-06-05T14:35:50.654999Z","shell.execute_reply.started":"2021-06-05T14:35:50.607049Z","shell.execute_reply":"2021-06-05T14:35:50.65397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub_img.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.65623Z","iopub.execute_input":"2021-06-05T14:35:50.656533Z","iopub.status.idle":"2021-06-05T14:35:50.662605Z","shell.execute_reply.started":"2021-06-05T14:35:50.656502Z","shell.execute_reply":"2021-06-05T14:35:50.661404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_img.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.664027Z","iopub.execute_input":"2021-06-05T14:35:50.664341Z","iopub.status.idle":"2021-06-05T14:35:50.682882Z","shell.execute_reply.started":"2021-06-05T14:35:50.664311Z","shell.execute_reply":"2021-06-05T14:35:50.682052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSES","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.683958Z","iopub.execute_input":"2021-06-05T14:35:50.68425Z","iopub.status.idle":"2021-06-05T14:35:50.703073Z","shell.execute_reply.started":"2021-06-05T14:35:50.684222Z","shell.execute_reply":"2021-06-05T14:35:50.701873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASS_LABELLINGS = merge_list_to_dict(CLASSES,list(range(len(CLASSES))))\nCLASS_LABELLINGS","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.704362Z","iopub.execute_input":"2021-06-05T14:35:50.704667Z","iopub.status.idle":"2021-06-05T14:35:50.720166Z","shell.execute_reply.started":"2021-06-05T14:35:50.704637Z","shell.execute_reply":"2021-06-05T14:35:50.719274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_preds(img = np.zeros((512,512))):\n    \"\"\"\n    Returns Classes and BBoxes\n    \"\"\"\n    Shape_X,Shape_Y = img.shape\n    Height_X = Shape_X/4\n    Height_Y = Shape_Y/4\n    num_preds = np.random.randint(low=0,high=4)\n    # print(num_preds4\n    CLASSES_IDX = []\n    CONFS = []\n    \n    BBOX = []\n    for i in range(num_preds):\n        CLS = np.random.randint(low=1,high=4) # 1,2,3\n        conf = np.random.randint(low=7,high=11)/10 # Confidence > 7 - 7-10\n        x = np.random.randint(low=40,high=61) * Height_X/100\n        y = np.random.randint(low=40,high=61) * Height_Y/100\n\n        init_x = np.random.randint(low=Height_X,high=(Shape_X-Height_X))\n        init_y = np.random.randint(low=Height_Y,high=(Shape_Y-Height_Y))\n\n        data = {\"x\":init_x,\n                \"y\":init_y,\n                \"width\":x,\n                \"height\":y}\n        \n        BBOX.append(data)\n        CLASSES_IDX.append(CLS)\n        CONFS.append(conf)\n    \n    return CLASSES_IDX,BBOX,CONFS\n\n    \nget_preds()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.721158Z","iopub.execute_input":"2021-06-05T14:35:50.721435Z","iopub.status.idle":"2021-06-05T14:35:50.740233Z","shell.execute_reply.started":"2021-06-05T14:35:50.721408Z","shell.execute_reply":"2021-06-05T14:35:50.739099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Target:\n\n```\nId,PredictionString\n2b95d54e4be65_study,negative 1 0 0 1 1\n2b95d54e4be66_study,typical 1 0 0 1 1\n2b95d54e4be67_study,indeterminate 1 0 0 1 1 atypical 1 0 0 1 1\n2b95d54e4be68_image,none 1 0 0 1 1\n2b95d54e4be69_image,opacity 0.5 100 100 200 200 opacity 0.7 10 10 20 20\netc.\n```","metadata":{}},{"cell_type":"code","source":"df_img[\"confidences\"] = str([])\ndf_img.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.741732Z","iopub.execute_input":"2021-06-05T14:35:50.742079Z","iopub.status.idle":"2021-06-05T14:35:50.784263Z","shell.execute_reply.started":"2021-06-05T14:35:50.742048Z","shell.execute_reply":"2021-06-05T14:35:50.782901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"std_value_cnts = df_img[\"Study_Name\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.785923Z","iopub.execute_input":"2021-06-05T14:35:50.786281Z","iopub.status.idle":"2021-06-05T14:35:50.801794Z","shell.execute_reply.started":"2021-06-05T14:35:50.786245Z","shell.execute_reply":"2021-06-05T14:35:50.800459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"greater_than_one = std_value_cnts>1\ngreater_than_one.sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.803293Z","iopub.execute_input":"2021-06-05T14:35:50.803672Z","iopub.status.idle":"2021-06-05T14:35:50.8238Z","shell.execute_reply.started":"2021-06-05T14:35:50.803589Z","shell.execute_reply":"2021-06-05T14:35:50.822576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ignoring these for now.\nWill be clarified once the confusion is dealt with.\n\nSee Discussion : https://www.kaggle.com/c/siim-covid19-detection/discussion/244189 for more intricate details.","metadata":{}},{"cell_type":"code","source":"CLASS_LABELLINGS","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.825747Z","iopub.execute_input":"2021-06-05T14:35:50.826062Z","iopub.status.idle":"2021-06-05T14:35:50.836519Z","shell.execute_reply.started":"2021-06-05T14:35:50.826033Z","shell.execute_reply":"2021-06-05T14:35:50.835424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASS_VALS = {v: k for k, v in CLASS_LABELLINGS.items()}\nCLASS_VALS","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.83813Z","iopub.execute_input":"2021-06-05T14:35:50.838442Z","iopub.status.idle":"2021-06-05T14:35:50.854114Z","shell.execute_reply.started":"2021-06-05T14:35:50.838411Z","shell.execute_reply":"2021-06-05T14:35:50.853052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nIMG_PREDS = []\nSTUDY_PREDS = []\n\"\"\"\nfor index, row in df_img.iterrows():\n    img_path = row[\"New_Image_Path\"]\n    img = cv2.imread(img_path,0)\n    # print(img.shape)\n    CLASSES_IDX,BBOX,CONFS = get_preds(img)\n    \n    if CLASSES_IDX==[]:\n        df_img.loc[index, \"image_label\"] = \"none\"\n        df_img.loc[index, \"study_label\"] = \"negative\"\n        df_img.loc[index, \"confidences\"] = str([])\n        df_img.loc[index, \"corrected_boxes\"] = str([])\n    else:\n        PRED_CLASSES = []\n        for cls_idx in CLASSES_IDX:\n            e_cls = CLASS_VALS[cls_idx]\n            PRED_CLASSES.append(e_cls)\n        df_img.loc[index, \"image_label\"] = \"opacity\"\n        df_img.loc[index, \"study_label\"] = \",\".join(PRED_CLASSES)\n        df_img.loc[index, \"confidences\"] = str(CONFS)\n        df_img.loc[index, \"corrected_boxes\"] = str(BBOX)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:50.855615Z","iopub.execute_input":"2021-06-05T14:35:50.856058Z","iopub.status.idle":"2021-06-05T14:35:59.949456Z","shell.execute_reply.started":"2021-06-05T14:35:50.85601Z","shell.execute_reply":"2021-06-05T14:35:59.948658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:59.95068Z","iopub.execute_input":"2021-06-05T14:35:59.951282Z","iopub.status.idle":"2021-06-05T14:35:59.956434Z","shell.execute_reply.started":"2021-06-05T14:35:59.951237Z","shell.execute_reply":"2021-06-05T14:35:59.955315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_img.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:59.957958Z","iopub.execute_input":"2021-06-05T14:35:59.95855Z","iopub.status.idle":"2021-06-05T14:35:59.989475Z","shell.execute_reply.started":"2021-06-05T14:35:59.958517Z","shell.execute_reply":"2021-06-05T14:35:59.988577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:35:59.990662Z","iopub.execute_input":"2021-06-05T14:35:59.990959Z","iopub.status.idle":"2021-06-05T14:36:00.007555Z","shell.execute_reply.started":"2021-06-05T14:35:59.99093Z","shell.execute_reply":"2021-06-05T14:36:00.006869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Upscale the Predictions\n\nRemember that we have downscaled the image to a `(512,512)` shape before and we need to adjust the bounding boxes accordingly again.","metadata":{}},{"cell_type":"code","source":"df_img['ecfy'] = 1/df_img['cfy']\ndf_img['ecfx'] = 1/df_img['cfx']","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:36:00.008664Z","iopub.execute_input":"2021-06-05T14:36:00.009149Z","iopub.status.idle":"2021-06-05T14:36:00.023132Z","shell.execute_reply.started":"2021-06-05T14:36:00.009117Z","shell.execute_reply":"2021-06-05T14:36:00.022171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ast import literal_eval\nfor index, row in df_img.iterrows():\n    cbbox =  literal_eval(row[\"corrected_boxes\"])\n    OLD_BOXES = []\n    for each_box in cbbox:\n        data = {\"x\":each_box[\"x\"]*row[\"ecfy\"],\n                \"y\":each_box[\"y\"]*row[\"ecfx\"],\n                \"width\":each_box[\"width\"]*row[\"ecfy\"],\n                \"height\":each_box[\"height\"]*row[\"ecfx\"]}\n        OLD_BOXES.append(data)\n    df_img.loc[index, \"boxes\"] = str(OLD_BOXES)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:36:00.024978Z","iopub.execute_input":"2021-06-05T14:36:00.025435Z","iopub.status.idle":"2021-06-05T14:36:00.607368Z","shell.execute_reply.started":"2021-06-05T14:36:00.025386Z","shell.execute_reply":"2021-06-05T14:36:00.606347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_img.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:36:00.608562Z","iopub.execute_input":"2021-06-05T14:36:00.608862Z","iopub.status.idle":"2021-06-05T14:36:00.634484Z","shell.execute_reply.started":"2021-06-05T14:36:00.608833Z","shell.execute_reply":"2021-06-05T14:36:00.633705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Re-Verification of Scaling","metadata":{}},{"cell_type":"code","source":"view_data(df_img,view=\"resized\")","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:36:00.63558Z","iopub.execute_input":"2021-06-05T14:36:00.636085Z","iopub.status.idle":"2021-06-05T14:36:03.970238Z","shell.execute_reply.started":"2021-06-05T14:36:00.636051Z","shell.execute_reply":"2021-06-05T14:36:03.969104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"view_data(df_img,view=\"original\")","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:36:03.971704Z","iopub.execute_input":"2021-06-05T14:36:03.9721Z","iopub.status.idle":"2021-06-05T14:36:24.012235Z","shell.execute_reply.started":"2021-06-05T14:36:03.972061Z","shell.execute_reply":"2021-06-05T14:36:24.011152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hooray! The Scaled Images have been brought back to the original size","metadata":{}},{"cell_type":"markdown","source":"# Prepare Submission\n\n### Image Level","metadata":{}},{"cell_type":"code","source":"df_img.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:36:24.013523Z","iopub.execute_input":"2021-06-05T14:36:24.013824Z","iopub.status.idle":"2021-06-05T14:36:24.040681Z","shell.execute_reply.started":"2021-06-05T14:36:24.013794Z","shell.execute_reply":"2021-06-05T14:36:24.039792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_img[\"Pred_Img\"] = \"\"\ndf_img[\"Pred_Std\"] = \"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:36:24.041775Z","iopub.execute_input":"2021-06-05T14:36:24.042064Z","iopub.status.idle":"2021-06-05T14:36:24.047447Z","shell.execute_reply.started":"2021-06-05T14:36:24.042037Z","shell.execute_reply":"2021-06-05T14:36:24.046499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_img.tail(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T14:36:24.048942Z","iopub.execute_input":"2021-06-05T14:36:24.04963Z","iopub.status.idle":"2021-06-05T14:36:24.085938Z","shell.execute_reply.started":"2021-06-05T14:36:24.049588Z","shell.execute_reply":"2021-06-05T14:36:24.084944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```\nId,PredictionString\n2b95d54e4be65_study,negative 1 0 0 1 1\n2b95d54e4be66_study,typical 1 0 0 1 1\n2b95d54e4be67_study,indeterminate 1 0 0 1 1 atypical 1 0 0 1 1\n2b95d54e4be68_image,none 1 0 0 1 1\n2b95d54e4be69_image,opacity 0.5 100 100 200 200 opacity 0.7 10 10 20 20\netc.\n```","metadata":{}},{"cell_type":"code","source":"for index, row in df_img.iterrows():\n    \n    Pred_Img = \"\"\n    Pred_Std = \"\"\n    study_label = row[\"study_label\"]\n    if study_label == \"negative\":\n        Pred_Img = \"negative 1 0 0 1 1\"\n        Pred_Std = \"none 1 0 0 1 1\"\n        # maybe insert confidences here (instead of the initial 1)!\n    else:\n        all_cls = study_label.split(\",\") if \",\" in study_label else [study_label]\n        bboxes =  literal_eval(row[\"boxes\"])\n        confs = literal_eval(row[\"confidences\"])\n        for each_class,each_box,each_conf in zip(all_cls,bboxes,confs):\n            # opacity 0.5 100 100 200 200 opacity 0.7 10 10 20 20\n            Pred_Img += \"opacity \"+str(round(each_conf,1))+\" \" + str(round(each_box[\"x\"],2))\n            Pred_Img +=  \" \"+ str(round(each_box[\"y\"],2))\n            Pred_Img +=  \" \" + str(round(each_box[\"width\"],2))+\" \" + str(round(each_box[\"height\"],2)) + \" \"\n            # indeterminate 1 0 0 1 1 atypical 1 0 0 1 1\n            Pred_Std += each_class + \" 1 0 0 1 1 \"\n    df_img.loc[index, \"Pred_Img\"] = str(Pred_Img)\n    df_img.loc[index, \"Pred_Std\"] = str(Pred_Std)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:05:08.111157Z","iopub.execute_input":"2021-06-05T15:05:08.111522Z","iopub.status.idle":"2021-06-05T15:05:08.982421Z","shell.execute_reply.started":"2021-06-05T15:05:08.111492Z","shell.execute_reply":"2021-06-05T15:05:08.981434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_img[\"Pred_Img\"] = df_img[\"Pred_Img\"].str.strip()\ndf_img[\"Pred_Std\"] = df_img[\"Pred_Std\"].str.strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:05:10.742772Z","iopub.execute_input":"2021-06-05T15:05:10.743151Z","iopub.status.idle":"2021-06-05T15:05:10.752018Z","shell.execute_reply.started":"2021-06-05T15:05:10.743119Z","shell.execute_reply":"2021-06-05T15:05:10.750977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_img.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:05:12.132669Z","iopub.execute_input":"2021-06-05T15:05:12.133252Z","iopub.status.idle":"2021-06-05T15:05:12.169068Z","shell.execute_reply.started":"2021-06-05T15:05:12.1332Z","shell.execute_reply":"2021-06-05T15:05:12.168074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_img.to_csv('Prediction_Data.csv',index=False)\ndf_img.to_excel('Prediction_Data.xlsx',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:05:24.543554Z","iopub.execute_input":"2021-06-05T15:05:24.544067Z","iopub.status.idle":"2021-06-05T15:05:25.721881Z","shell.execute_reply.started":"2021-06-05T15:05:24.544016Z","shell.execute_reply":"2021-06-05T15:05:25.720758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index, row in df_sub.iterrows():\n    img_id = row[\"id\"]\n    try:\n        if img_id.endswith('_study'):\n            idx = df_img.index[df_img['Study_Name'] == img_id.replace(\"_study\",\"\")].tolist()\n            p = df_img[\"Pred_Std\"][idx[0]]\n        elif img_id.endswith('_image'):\n            idx = df_img.index[df_img['Image_ID'] == img_id.replace(\"_image\",\"\")].tolist()\n            p = df_img[\"Pred_Img\"][idx[0]]\n        df_sub.loc[index, \"PredictionString\"] = str(p)\n    except:\n        continue","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:05:31.212807Z","iopub.execute_input":"2021-06-05T15:05:31.213226Z","iopub.status.idle":"2021-06-05T15:05:32.696856Z","shell.execute_reply.started":"2021-06-05T15:05:31.21319Z","shell.execute_reply":"2021-06-05T15:05:32.695883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:05:33.857776Z","iopub.execute_input":"2021-06-05T15:05:33.858332Z","iopub.status.idle":"2021-06-05T15:05:33.867417Z","shell.execute_reply.started":"2021-06-05T15:05:33.858297Z","shell.execute_reply":"2021-06-05T15:05:33.866434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.tail()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:05:34.967559Z","iopub.execute_input":"2021-06-05T15:05:34.967952Z","iopub.status.idle":"2021-06-05T15:05:34.977764Z","shell.execute_reply.started":"2021-06-05T15:05:34.967914Z","shell.execute_reply":"2021-06-05T15:05:34.976758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.to_csv('Random_Submission.csv',index=False)\ndf_sub.to_excel('Random_Submission.xlsx',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T15:05:59.648463Z","iopub.execute_input":"2021-06-05T15:05:59.648838Z","iopub.status.idle":"2021-06-05T15:06:00.616235Z","shell.execute_reply.started":"2021-06-05T15:05:59.648804Z","shell.execute_reply":"2021-06-05T15:06:00.615187Z"},"trusted":true},"execution_count":null,"outputs":[]}]}