{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom PIL import Image\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport shutil\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","scrolled":true,"execution":{"iopub.status.busy":"2021-08-08T07:57:13.406299Z","iopub.execute_input":"2021-08-08T07:57:13.406725Z","iopub.status.idle":"2021-08-08T07:57:13.416272Z","shell.execute_reply.started":"2021-08-08T07:57:13.406619Z","shell.execute_reply":"2021-08-08T07:57:13.41531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom glob import glob\nimport gc\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom shutil import copyfile\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\n#customize iPython writefile so we can write variables\nfrom IPython.core.magic import register_line_cell_magic\n\n@register_line_cell_magic\ndef writetemplate(line, cell):\n    with open(line, 'w') as f:\n        f.write(cell.format(**globals()))\n        \nimport torch\nprint(f\"Setup complete. Using torch {torch.__version__} ({torch.cuda.get_device_properties(0).name if torch.cuda.is_available() else 'CPU'})\")\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2021-08-08T07:57:13.518955Z","iopub.execute_input":"2021-08-08T07:57:13.519229Z","iopub.status.idle":"2021-08-08T07:57:20.391246Z","shell.execute_reply.started":"2021-08-08T07:57:13.519202Z","shell.execute_reply":"2021-08-08T07:57:20.390258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"!conda install '/kaggle/input/pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:14:56.711118Z","iopub.execute_input":"2021-08-07T10:14:56.711471Z","iopub.status.idle":"2021-08-07T10:16:07.144877Z","shell.execute_reply.started":"2021-08-07T10:14:56.71144Z","shell.execute_reply":"2021-08-07T10:16:07.143931Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fast_df = False\nIMG_SIZE = 600","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:16:07.146693Z","iopub.execute_input":"2021-08-07T10:16:07.147069Z","iopub.status.idle":"2021-08-07T10:16:07.153712Z","shell.execute_reply.started":"2021-08-07T10:16:07.147029Z","shell.execute_reply":"2021-08-07T10:16:07.152758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:16:07.156001Z","iopub.execute_input":"2021-08-07T10:16:07.156583Z","iopub.status.idle":"2021-08-07T10:16:07.419158Z","shell.execute_reply.started":"2021-08-07T10:16:07.156536Z","shell.execute_reply":"2021-08-07T10:16:07.418273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:16:07.42082Z","iopub.execute_input":"2021-08-07T10:16:07.421188Z","iopub.status.idle":"2021-08-07T10:16:07.429326Z","shell.execute_reply.started":"2021-08-07T10:16:07.421151Z","shell.execute_reply":"2021-08-07T10:16:07.428403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"split = 'test'\nimage_id = []\ndim0 = []\ndim1 = []\nsplits = []\nsave_dir = f'/kaggle/tmp/{split}/image/'\nos.makedirs(save_dir, exist_ok=True)\n\nfor dirname, _, filenames in tqdm(os.walk(f'/kaggle/input/siim-covid19-detection/{split}')):\n    for file in filenames:\n        # set keep_ratio=True to have original aspect ratio\n        xray = read_xray(os.path.join(dirname, file))\n        im = resize(xray, size=IMG_SIZE)  \n        im.save(os.path.join(save_dir, file.replace('.dcm', '_image.jpeg')))\n        image_id.append(file.replace('.dcm', ''))\n        dim0.append(xray.shape[0])\n        dim1.append(xray.shape[1])\n        splits.append(split)\nmeta = pd.DataFrame.from_dict({'image_id': image_id, 'dim0': dim0, 'dim1': dim1, 'split': splits})","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:16:07.430661Z","iopub.execute_input":"2021-08-07T10:16:07.431138Z","iopub.status.idle":"2021-08-07T10:23:55.302833Z","shell.execute_reply.started":"2021-08-07T10:16:07.431096Z","shell.execute_reply":"2021-08-07T10:23:55.301824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MODEL_PATH = \"artifacts/run_1qerm3x5_model:v24/last.pt\"\n# MODEL_PATH = \"/kaggle/input/yolov5-150-epochs-covid-localization/best.pt\"\n# MODEL_PATH = \"/kaggle/input/yolo-notebook/tmp/yolov5/artifacts/run_2yubez04_model:v29/best.pt\"\nMODEL_PATH = \"artifacts/run_3l287lxi_model:v108/last.pt\"\nTEST_PATH = '/kaggle/tmp/test/image'","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:23:55.304244Z","iopub.execute_input":"2021-08-07T10:23:55.304617Z","iopub.status.idle":"2021-08-07T10:23:55.309033Z","shell.execute_reply.started":"2021-08-07T10:23:55.304577Z","shell.execute_reply":"2021-08-07T10:23:55.307883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from numba import cuda\n# import torch\n# cuda.select_device(0)\n# cuda.close()\n# cuda.select_device(0)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:23:55.310335Z","iopub.execute_input":"2021-08-07T10:23:55.310877Z","iopub.status.idle":"2021-08-07T10:23:55.322559Z","shell.execute_reply.started":"2021-08-07T10:23:55.310818Z","shell.execute_reply":"2021-08-07T10:23:55.32155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/input/yolo-notebook-2/tmp/yolov5","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:23:55.325025Z","iopub.execute_input":"2021-08-07T10:23:55.325446Z","iopub.status.idle":"2021-08-07T10:23:55.341516Z","shell.execute_reply.started":"2021-08-07T10:23:55.325407Z","shell.execute_reply":"2021-08-07T10:23:55.34069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"all the detection will be stored in '/kaggle/tmp/detect'","metadata":{}},{"cell_type":"code","source":"!python detect.py --weights {MODEL_PATH} \\\n                  --source {TEST_PATH} \\\n                  --img {640} \\\n                  --conf 0.3\\\n                  --iou 0.4 \\\n                  --save-txt \\\n                  --save-conf \\\n                  --project \"/kaggle/tmp/detect\"","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:28:07.282597Z","iopub.execute_input":"2021-08-07T10:28:07.282937Z","iopub.status.idle":"2021-08-07T10:29:47.351943Z","shell.execute_reply.started":"2021-08-07T10:28:07.282891Z","shell.execute_reply":"2021-08-07T10:29:47.351003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model =  tf.keras.models.load_model(\"/kaggle/input/k/sushantjha78/efficientnetb7/weights/my_model.h5\") \n#83%","metadata":{"execution":{"iopub.status.busy":"2021-08-08T07:59:25.752772Z","iopub.execute_input":"2021-08-08T07:59:25.753099Z","iopub.status.idle":"2021-08-08T07:59:36.986294Z","shell.execute_reply.started":"2021-08-08T07:59:25.75307Z","shell.execute_reply":"2021-08-08T07:59:36.985327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Model =  tf.keras.models.load_model(\"/kaggle/input/efficientnetweights/weights/my_model.h5\") \n#75%","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:46:45.41863Z","iopub.execute_input":"2021-08-07T10:46:45.41901Z","iopub.status.idle":"2021-08-07T10:46:56.445809Z","shell.execute_reply.started":"2021-08-07T10:46:45.418972Z","shell.execute_reply":"2021-08-07T10:46:56.444897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model =  tf.keras.models.load_model(\"/kaggle/input/efnetb7-layers-increased-more-trainable-layers/weights/my_model.h5\")\n#65%","metadata":{"execution":{"iopub.status.busy":"2021-07-23T09:41:49.40045Z","iopub.execute_input":"2021-07-23T09:41:49.400861Z","iopub.status.idle":"2021-07-23T09:41:49.405322Z","shell.execute_reply.started":"2021-07-23T09:41:49.400778Z","shell.execute_reply":"2021-07-23T09:41:49.404136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name2label = { \n    'negative': 2,\n    'indeterminate': 1,\n    'atypical': 0,\n    'typical': 3}\nlabel2name  = {v:k for k, v in name2label.items()}","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:30:02.405534Z","iopub.execute_input":"2021-08-07T10:30:02.405871Z","iopub.status.idle":"2021-08-07T10:30:02.411178Z","shell.execute_reply.started":"2021-08-07T10:30:02.405822Z","shell.execute_reply":"2021-08-07T10:30:02.409745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepaths = glob('/kaggle/input/siim-covid19-detection/test/**/*dcm',recursive=True)\ntest_df = pd.DataFrame({'filepath':filepaths,})\ntest_df['image_id'] = test_df.filepath.map(lambda x: x.split('/')[-1].replace('.dcm', '')+'_image')\ntest_df['study_id'] = test_df.filepath.map(lambda x: x.split('/')[-3].replace('.dcm', '')+'_study')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:30:02.413122Z","iopub.execute_input":"2021-08-07T10:30:02.413472Z","iopub.status.idle":"2021-08-07T10:30:04.141236Z","shell.execute_reply.started":"2021-08-07T10:30:02.413435Z","shell.execute_reply":"2021-08-07T10:30:04.140435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"path_modified\"] = test_df.filepath.map(lambda x: \"/kaggle/tmp/test/image/\" +\n                                                x.split('/')[-1].replace('.dcm', '')+\"_image.jpeg\")","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:30:04.142476Z","iopub.execute_input":"2021-08-07T10:30:04.142824Z","iopub.status.idle":"2021-08-07T10:30:04.149452Z","shell.execute_reply.started":"2021-08-07T10:30:04.142786Z","shell.execute_reply":"2021-08-07T10:30:04.148267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:30:04.15091Z","iopub.execute_input":"2021-08-07T10:30:04.151493Z","iopub.status.idle":"2021-08-07T10:30:04.172058Z","shell.execute_reply.started":"2021-08-07T10:30:04.151413Z","shell.execute_reply":"2021-08-07T10:30:04.170887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PRED_PATH = '/kaggle/tmp/detect/exp/labels'\n!ls {PRED_PATH}","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:30:04.173863Z","iopub.execute_input":"2021-08-07T10:30:04.174247Z","iopub.status.idle":"2021-08-07T10:30:04.878461Z","shell.execute_reply.started":"2021-08-07T10:30:04.17421Z","shell.execute_reply":"2021-08-07T10:30:04.877271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_files = os.listdir(PRED_PATH)\nprint('Number of test images predicted as opaque: ', len(prediction_files))","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:30:04.882955Z","iopub.execute_input":"2021-08-07T10:30:04.883273Z","iopub.status.idle":"2021-08-07T10:30:04.889182Z","shell.execute_reply.started":"2021-08-07T10:30:04.883239Z","shell.execute_reply":"2021-08-07T10:30:04.887791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The submisison requires xmin, ymin, xmax, ymax format. \n# YOLOv5 returns x_center, y_center, width, height\ndef correct_bbox_format(bboxes):\n    correct_bboxes = []\n    for b in bboxes:\n        xc, yc = int(np.round(b[0]*IMG_SIZE)), int(np.round(b[1]*IMG_SIZE))\n        w, h = int(np.round(b[2]*IMG_SIZE)), int(np.round(b[3]*IMG_SIZE))\n\n        xmin = xc - int(np.round(w/2))\n        xmax = xc + int(np.round(w/2))\n        ymin = yc - int(np.round(h/2))\n        ymax = yc + int(np.round(h/2))\n        \n        correct_bboxes.append([xmin, xmax, ymin, ymax])\n        \n    return correct_bboxes\n\n# Read the txt file generated by YOLOv5 during inference and extract \n# confidence and bounding box coordinates.\ndef get_conf_bboxes(file_path):\n    confidence = []\n    bboxes = []\n    with open(file_path, 'r') as file:\n        for line in file:\n            preds = line.strip('\\n').split(' ')\n            preds = list(map(float, preds))\n            confidence.append(preds[-1])\n            bboxes.append(preds[1:-1])\n    return confidence, bboxes","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:30:04.891086Z","iopub.execute_input":"2021-08-07T10:30:04.891837Z","iopub.status.idle":"2021-08-07T10:30:04.904544Z","shell.execute_reply.started":"2021-08-07T10:30:04.891795Z","shell.execute_reply":"2021-08-07T10:30:04.90352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the submisison file\nsub_df = pd.read_csv('/kaggle/input/siim-covid19-detection/sample_submission.csv')\nsub_df","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:39:36.356168Z","iopub.execute_input":"2021-08-07T10:39:36.356503Z","iopub.status.idle":"2021-08-07T10:39:36.378475Z","shell.execute_reply.started":"2021-08-07T10:39:36.356473Z","shell.execute_reply":"2021-08-07T10:39:36.377388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction loop for submission\npredictions = []\n# iterating through submission.csv.\nfor i in tqdm(range(len(sub_df))):\n    row = sub_df.loc[i]\n    id_name = row.id.split('_')[0]\n    id_level = row.id.split('_')[-1]\n    # passing all images under this id through Efficientneb7 model if id_level is 'study' \n    if id_level == 'study':\n        # one study can contain more than one images. multiple images can have same predictions. appending all predictions \n        # in set and submitting only distinct predictions. \n        temp = set()\n        ans =\"\"\n        for i in test_df.loc[test_df[\"study_id\"]==row.id][\"path_modified\"] :\n            # dcm to jpeg\n            img = tf.io.read_file(i)\n            img = tf.image.decode_jpeg(img, channels=1)\n            img = tf.reshape(img,(1,IMG_SIZE,IMG_SIZE,1))\n            # output predictions\n            op = Model(img)\n            label = label2name[op.numpy().argmax()]+\" 1 0 0 1 1\" \n            temp.add(label)\n        for i in temp:\n            ans = ans +i+\" \"\n            \n        predictions.append(ans.strip())\n\n    elif id_level == 'image':\n        # we can do image-level classification here.\n        # also we can rely on the object detector's classification head.\n        # for this example submisison we will use YOLO's classification head. \n        # since we already ran the inference we know which test images belong to opacity.\n        if f'{id_name}_{id_level}.txt' in prediction_files:\n            # opacity label\n            confidence, bboxes = get_conf_bboxes(f'{PRED_PATH}/{id_name}_{id_level}.txt')\n            bboxes = correct_bbox_format(bboxes)\n            pred_string = ''\n            for j, conf in enumerate(confidence):\n                pred_string += f'opacity {conf} ' + ' '.join(map(str, bboxes[j])) + ' '\n            predictions.append(pred_string[:-1]) \n        else:\n            predictions.append(\"none 1 0 0 1 1\")","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:39:40.020504Z","iopub.execute_input":"2021-08-07T10:39:40.020837Z","iopub.status.idle":"2021-08-07T10:40:06.510172Z","shell.execute_reply.started":"2021-08-07T10:39:40.020803Z","shell.execute_reply":"2021-08-07T10:40:06.507976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df['PredictionString'] = predictions\nsub_df.to_csv('/kaggle/working/submission.csv', index=False)\nsub_df","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:35:51.664058Z","iopub.execute_input":"2021-08-07T10:35:51.664438Z","iopub.status.idle":"2021-08-07T10:35:51.950932Z","shell.execute_reply.started":"2021-08-07T10:35:51.664397Z","shell.execute_reply":"2021-08-07T10:35:51.950022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-07-21T04:14:54.805171Z","iopub.execute_input":"2021-07-21T04:14:54.805523Z","iopub.status.idle":"2021-07-21T04:14:54.858906Z","shell.execute_reply.started":"2021-07-21T04:14:54.805489Z","shell.execute_reply":"2021-07-21T04:14:54.857922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}