{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install Dependencies","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)\nimport torch\nprint(f\"Setup complete. Using torch {torch.__version__} ({torch.cuda.get_device_properties(0).name if torch.cuda.is_available() else 'CPU'})\")\n\nimport os\nimport gc\nimport cv2\nimport glob\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom shutil import copyfile\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-13T14:04:25.135871Z","iopub.execute_input":"2021-06-13T14:04:25.136225Z","iopub.status.idle":"2021-06-13T14:04:25.145757Z","shell.execute_reply.started":"2021-06-13T14:04:25.136194Z","shell.execute_reply":"2021-06-13T14:04:25.144965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpus = tf.config.list_physical_devices('GPU')\nif gpus:\n  try:\n    # Currently, memory growth needs to be the same across GPUs\n    for gpu in gpus:\n      tf.config.experimental.set_memory_growth(gpu, True)\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n  except RuntimeError as e:\n    # Memory growth must be set before GPUs have been initialized\n    print(e)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:04:25.656427Z","iopub.execute_input":"2021-06-13T14:04:25.656771Z","iopub.status.idle":"2021-06-13T14:04:25.663256Z","shell.execute_reply.started":"2021-06-13T14:04:25.656733Z","shell.execute_reply":"2021-06-13T14:04:25.662397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Submission ","metadata":{}},{"cell_type":"code","source":"# Read the submisison file\nsub_df = pd.read_csv('/kaggle/input/siim-covid19-detection/sample_submission.csv')\nprint(len(sub_df))\nsub_df","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:04:27.178667Z","iopub.execute_input":"2021-06-13T14:04:27.178998Z","iopub.status.idle":"2021-06-13T14:04:27.200321Z","shell.execute_reply.started":"2021-06-13T14:04:27.178967Z","shell.execute_reply":"2021-06-13T14:04:27.199534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df = sub_df.loc[sub_df.id.str.contains('_study')]\nlen(study_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:04:27.901463Z","iopub.execute_input":"2021-06-13T14:04:27.901800Z","iopub.status.idle":"2021-06-13T14:04:27.909867Z","shell.execute_reply.started":"2021-06-13T14:04:27.901769Z","shell.execute_reply":"2021-06-13T14:04:27.909059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df = sub_df.loc[sub_df.id.str.contains('_image')]\nlen(image_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:04:28.535740Z","iopub.execute_input":"2021-06-13T14:04:28.536098Z","iopub.status.idle":"2021-06-13T14:04:28.544103Z","shell.execute_reply.started":"2021-06-13T14:04:28.536065Z","shell.execute_reply":"2021-06-13T14:04:28.543238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils to extract images and resize them","metadata":{}},{"cell_type":"code","source":"\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\ndef resize_xray(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:04:34.633988Z","iopub.execute_input":"2021-06-13T14:04:34.634305Z","iopub.status.idle":"2021-06-13T14:04:34.642270Z","shell.execute_reply.started":"2021-06-13T14:04:34.634276Z","shell.execute_reply":"2021-06-13T14:04:34.641145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_PATH = f'/kaggle/tmp/test/'\nIMG_SIZE = 1024\n\ndef prepare_test_images():\n    image_id = []\n    dim0 = []\n    dim1 = []\n\n    os.makedirs(TEST_PATH, exist_ok=True)\n\n    for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/test')):\n        for file in filenames:\n            # set keep_ratio=True to have original aspect ratio\n            xray = read_xray(os.path.join(dirname, file))\n            im = resize_xray(xray, size=IMG_SIZE)  \n            im.save(os.path.join(TEST_PATH, file.replace('dcm', 'png')))\n\n            image_id.append(file.replace('.dcm', ''))\n            dim0.append(xray.shape[0])\n            dim1.append(xray.shape[1])\n            \n    return image_id, dim0, dim1","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:04:36.522735Z","iopub.execute_input":"2021-06-13T14:04:36.523075Z","iopub.status.idle":"2021-06-13T14:04:36.529726Z","shell.execute_reply.started":"2021-06-13T14:04:36.523045Z","shell.execute_reply":"2021-06-13T14:04:36.528770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare Image Level Test Images","metadata":{}},{"cell_type":"markdown","source":"# YOLOv5 Inferencecontains","metadata":{}},{"cell_type":"code","source":"YOLO_MODEL_PATH = '../input/happyhappyyolov5xtrain/best.pt'\nTEST_PATH = '../input/siim-covide-1024-resized-happyhappy/test'\n!python ../input/kaggle-yolov5/detect.py --weights {YOLO_MODEL_PATH} \\\n                                      --source {TEST_PATH} \\\n                                      --img {IMG_SIZE} \\\n                                      --conf 0.16  \\\n                                      --iou-thres 0.5 \\\n                                      --max-det 10 \\\n                                      --save-txt \\\n                                      --save-conf","metadata":{"scrolled":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-13T14:04:39.880238Z","iopub.execute_input":"2021-06-13T14:04:39.880678Z","iopub.status.idle":"2021-06-13T14:07:43.740743Z","shell.execute_reply.started":"2021-06-13T14:04:39.880637Z","shell.execute_reply":"2021-06-13T14:07:43.739835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PRED_PATH = 'runs/detect/exp2/labels'\nprediction_files = os.listdir(PRED_PATH)\nprint(f'Number of opacity predicted by YOLOv5: {len(prediction_files)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:07:56.819498Z","iopub.execute_input":"2021-06-13T14:07:56.819823Z","iopub.status.idle":"2021-06-13T14:07:56.825537Z","shell.execute_reply.started":"2021-06-13T14:07:56.819790Z","shell.execute_reply":"2021-06-13T14:07:56.824490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# The submisison requires xmin, ymin, xmax, ymax format. \n# YOLOv5 returns x_center, y_center, width, height\ndef correct_bbox_format(bboxes):\n    correct_bboxes = []\n    for b in bboxes:\n        xc, yc = int(np.round(b[0]*IMG_SIZE)), int(np.round(b[1]*IMG_SIZE))\n        w, h = int(np.round(b[2]*IMG_SIZE)), int(np.round(b[3]*IMG_SIZE))\n\n        xmin = xc - int(np.round(w/2))\n        xmax = xc + int(np.round(w/2))\n        ymin = yc - int(np.round(h/2))\n        ymax = yc + int(np.round(h/2))\n        \n        correct_bboxes.append([xmin, xmax, ymin, ymax])\n        \n    return correct_bboxes\n\n# Read the txt file generated by YOLOv5 during inference and extract \n# confidence and bounding box coordinates.\ndef get_conf_bboxes(file_path):\n    confidence = []\n    bboxes = []\n    with open(file_path, 'r') as file:\n        for line in file:\n            preds = line.strip('\\n').split(' ')\n            preds = list(map(float, preds))\n            confidence.append(preds[-1])\n            bboxes.append(preds[1:-1])\n    return confidence, bboxes\n\n# Read the submisison file\nsub_df = pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')\nsub_df.tail()\n\n# Prediction loop for submission\npredictions = []\n\nfor i in tqdm(range(len(sub_df))):\n    row = sub_df.loc[i]\n    id_name = row.id.split('_')[0]\n    id_level = row.id.split('_')[-1]\n    \n    if id_level == 'study':\n        # do study-level classification\n        predictions.append(\"Negative 1 0 0 1 1\") # dummy prediction\n        \n    elif id_level == 'image':\n        # we can do image-level classification here.\n        # also we can rely on the object detector's classification head.\n        # for this example submisison we will use YOLO's classification head. \n        # since we already ran the inference we know which test images belong to opacity.\n        if f'{id_name}.txt' in prediction_files:\n            # opacity label\n            confidence, bboxes = get_conf_bboxes(f'{PRED_PATH}/{id_name}.txt')\n            bboxes = correct_bbox_format(bboxes)\n            pred_string = ''\n            for j, conf in enumerate(confidence):\n                pred_string += f'opacity {conf} ' + ' '.join(map(str, bboxes[j])) + ' '\n            predictions.append(pred_string[:-1]) \n        else:\n            predictions.append(\"None 1 0 0 1 1\")\n\n\n\nsub_df['PredictionString'] = predictions","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:08:14.237487Z","iopub.execute_input":"2021-06-13T14:08:14.237834Z","iopub.status.idle":"2021-06-13T14:08:14.693904Z","shell.execute_reply.started":"2021-06-13T14:08:14.237796Z","shell.execute_reply":"2021-06-13T14:08:14.692931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:08:22.583171Z","iopub.execute_input":"2021-06-13T14:08:22.583514Z","iopub.status.idle":"2021-06-13T14:08:22.595109Z","shell.execute_reply.started":"2021-06-13T14:08:22.583480Z","shell.execute_reply":"2021-06-13T14:08:22.594193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf ./runs/detect","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:08:57.138658Z","iopub.execute_input":"2021-06-13T14:08:57.138998Z","iopub.status.idle":"2021-06-13T14:08:58.006921Z","shell.execute_reply.started":"2021-06-13T14:08:57.138964Z","shell.execute_reply":"2021-06-13T14:08:58.005942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# study_df = pd.read_csv('../input/siimhappyhappysubmission/study_submission.csv')\n# study_df = study_df[study_df.apply(lambda x : x['id'].endswith(\"study\"),axis=1)]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:26:28.331267Z","iopub.execute_input":"2021-06-13T13:26:28.331596Z","iopub.status.idle":"2021-06-13T13:26:28.381776Z","shell.execute_reply.started":"2021-06-13T13:26:28.331561Z","shell.execute_reply":"2021-06-13T13:26:28.381075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # image_df = pd.read_csv('image_submission.csv')\n# image_df = image_df[image_df.apply(lambda x : x['id'].endswith(\"image\"),axis=1)]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:27:55.412427Z","iopub.execute_input":"2021-06-13T13:27:55.412765Z","iopub.status.idle":"2021-06-13T13:27:55.431191Z","shell.execute_reply.started":"2021-06-13T13:27:55.412731Z","shell.execute_reply":"2021-06-13T13:27:55.430300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub_df = pd.concat([study_df, image_df])\nsub_df.to_csv('./submission.csv', index=False)\nsub_df","metadata":{"execution":{"iopub.status.busy":"2021-06-13T14:09:09.865729Z","iopub.execute_input":"2021-06-13T14:09:09.866114Z","iopub.status.idle":"2021-06-13T14:09:09.890240Z","shell.execute_reply.started":"2021-06-13T14:09:09.866076Z","shell.execute_reply":"2021-06-13T14:09:09.889250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}