{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport os\nimport matplotlib.pyplot as plt\nimport cv2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/weighted-box-fusion","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /kaggle/input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/siim-covid-inference-models ./models","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/yolov5 ./\n!cp -r /kaggle/input/yolov4 ./","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare test data","metadata":{}},{"cell_type":"code","source":"data_path = \"/kaggle/input/siim-covid19-detection/\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv(os.path.join(data_path, \"sample_submission.csv\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = os.path.join(data_path, \"test\")\ntest_filenames = [os.path.join(dirname,filename) for dirname,_,filenames in os.walk(test_path) for filename in filenames]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_mapping = {}\n\nfor study_dir in os.listdir(test_path):\n    for sub_dir in os.listdir(os.path.join(test_path, study_dir)):\n        for image_name in os.listdir(os.path.join(os.path.join(test_path, study_dir),\n                                                 sub_dir)):\n            image_id = image_name[:-4] \n            study_mapping[image_id] = study_dir","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs(\"dataset/test\", exist_ok=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nfrom PIL import Image\nimport pandas as pd\nfrom tqdm.auto import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig_shapes = {}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_dcm_as_png(source, dest, size = 832):\n    image = read_xray(source)\n    \n    orig_shapes[source.split(\"/\")[-1][:-4]] = (image.shape[1], image.shape[0])\n    \n    image = resize(image, size)\n    image.save(dest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x in test_filenames:\n    save_dcm_as_png(x, os.path.join(\"dataset/test\", \n                                    x.split(\"/\")[-1][:-3] + \"png\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Level Inference","metadata":{}},{"cell_type":"markdown","source":"## YOLOv5","metadata":{}},{"cell_type":"code","source":"!python ./yolov5/detect.py --augment --weights ./models/yolov5x_v2.pt \\\n                                      --source ./dataset/test \\\n                                      --img 608 \\\n                                      --conf 0.005 \\\n                                      --iou-thres 0.5 \\\n                                      --save-txt \\\n                                      --save-conf \\\n                                      --nosave","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_path = 'runs/detect/exp/labels'\nprediction_files = os.listdir(preds_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_list = os.listdir(\"dataset/test\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yolov5_boxes = {image_name[:-4] : list() for image_name in images_list}\nyolov5_scores = {image_name[:-4] : list() for image_name in images_list}\nyolov5_labels = {image_name[:-4] : list() for image_name in images_list}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for pred_file in prediction_files:\n    id = pred_file[:-4]\n    \n    image_width, image_height = orig_shapes[id]\n    \n    pred_file_path = os.path.join(preds_path, pred_file)\n    \n    pred_str = \"\"\n    \n    with open(pred_file_path, \"r\") as f:\n        preds = f.readlines()\n        for pred in preds:\n            pred = np.array(pred.replace(\"\\n\", \"\").split(\" \"), dtype = np.float)\n            x_c, y_c, w_b, h_b = pred[1:5] #* np.array([image_width, image_height, image_width, image_height])\n            \n            xmin = x_c - (w_b / 2)\n            xmax = x_c + (w_b / 2)\n            ymin = y_c - (h_b / 2)\n            ymax = y_c + (h_b / 2)\n\n            score = pred[-1]\n            #opacity 0.44043 1568 446 2146 1793 opacity 0.543457 689 405 1290 1588\n            #pred_str += \"opacity {} {} {} {} {} \".format(score, xmin, ymin, xmax, ymax)\n            \n            yolov5_boxes[id].append([xmin, ymin, xmax, ymax])\n            yolov5_scores[id].append(score)\n            yolov5_labels[id].append(0)\n    #sub_df.loc[sub_df[\"id\"] == id + \"_image\", \"PredictionString\"] = pred_str","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## YOLOv4","metadata":{}},{"cell_type":"code","source":"%cd yolov4\n!sed -i 's/OPENCV=0/OPENCV=1/' Makefile\n!sed -i 's/GPU=0/GPU=1/' Makefile\n!sed -i 's/CUDNN=0/CUDNN=1/' Makefile\n!sed -i 's/CUDNN_HALF=0/CUDNN_HALF=1/' Makefile","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# verify CUDA\n!/usr/local/cuda/bin/nvcc --version","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /usr/lib/gcc/x86_64-linux-gnu/7/../../../x86_64-linux-gnu/\n!rm libcuda.so","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /kaggle/input/libcuda/libcuda.so .","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/yolov4\n!make","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs(\"../runs/v4\", exist_ok=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile ../models/obj.data\nclasses = 1\ntrain = ./data/train.txt\nvalid = ./data/valid.txt\nnames = ../models/obj.names\nbackup = /content/drive/My Drive/siim_covid/models/yolov4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r ../dataset/test ../runs/v4 ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimage_files = []\n\nfor filename in os.listdir(os.path.join(\"../runs/v4\", \"test\")):\n    if filename.endswith(\".png\"):\n        image_files.append(\"../runs/v4/test/\" + filename)\n\nwith open(\"../runs/v4/test.txt\", \"w\") as outfile:\n    for image in image_files:\n        outfile.write(image)\n        outfile.write(\"\\n\")\n    outfile.close()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!./darknet detector test ../models/obj.data ../models/yolov4-obj.cfg \\\n../models/yolov4.weights -ext_output -dont_show -thresh 0.005 \\\n-out ../runs/v4/result.json < ../runs/v4/test.txt","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nwith open (\"../runs/v4/result.json\", \"r\") as f:\n    results = json.load(f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yolov4_boxes = {result[\"filename\"].split(\"/\")[-1][:-4] : list() for result in results}\nyolov4_scores = {result[\"filename\"].split(\"/\")[-1][:-4] : list() for result in results}\nyolov4_labels = {result[\"filename\"].split(\"/\")[-1][:-4] : list() for result in results}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for result in results:\n    id = result[\"filename\"].split(\"/\")[-1][:-4]\n    json_det = result[\"objects\"]\n    if len(json_det) > 0:\n        for det in json_det:\n            coords = det[\"relative_coordinates\"]\n            x_c = float(coords[\"center_x\"])\n            y_c = float(coords[\"center_y\"])\n            w = float(coords[\"width\"])\n            h = float(coords[\"height\"])\n            \n            x1 = x_c - w/2\n            x2 = x_c + w/2\n            y1 = y_c - h/2\n            y2 = y_c + h/2\n            \n            conf = det[\"confidence\"]\n            \n            yolov4_boxes[id].append([x1, y1, x2, y2])\n            yolov4_scores[id].append(conf)\n            yolov4_labels[id].append(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## FRCNN","metadata":{}},{"cell_type":"code","source":"import torch, torchvision\nprint(torch.__version__, torch.cuda.is_available())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd ..","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/detectron2/omegaconf-2.0.6-py3-none-any.whl\n!pip install /kaggle/input/detectron2/iopath-0.1.8-py3-none-any.whl\n!pip install /kaggle/input/detectron2/fvcore-0.1.3.post20210317/fvcore-0.1.3.post20210317/\n!pip install /kaggle/input/detectron2/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar\n!pip install /kaggle/input/detectron2/detectron2-0.4cu110-cp37-cp37m-linux_x86_64.whl","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.data import DatasetCatalog, MetadataCatalog\nfrom detectron2.utils.visualizer import ColorMode","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x.yaml\"))\ncfg.MODEL.WEIGHTS = \"./models/model_0004199.pth\"  # path to the model we just trained\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.005  # set a custom testing threshold\ncfg.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS = [[0.383, 0.61, 1.0, 1.64, 2.61]]\ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 256\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 1 ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frcnn_boxes = {image_name[:-4] : list() for image_name in images_list}\nfrcnn_scores = {image_name[:-4] : list() for image_name in images_list}\nfrcnn_labels = {image_name[:-4] : list() for image_name in images_list}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictor = DefaultPredictor(cfg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image_name in images_list:\n    image_path = os.path.join(\"./dataset/test\", image_name)\n    image = cv2.imread(image_path)\n    \n    id = image_name[:-4]\n    #image_width, image_height = orig_shapes[id]\n    \n    dets = predictor(image)[\"instances\"]\n    \n    fields = dets.get_fields()\n    #pred_classes = np.array(fields[\"pred_classes\"])  # (n_boxes,)\n    pred_scores = np.array(fields[\"scores\"].cpu()) # shape (n_boxes, 4). (xmin, ymin, xmax, ymax)\n    pred_boxes = np.array(fields[\"pred_boxes\"].tensor.cpu()) / 832.\n    \n    if len(pred_boxes) > 0:\n        for i, box in enumerate(pred_boxes):\n            frcnn_boxes[id].append(list(box))\n            frcnn_scores[id].append(list(pred_scores)[i])\n            frcnn_labels[id].append(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble","metadata":{}},{"cell_type":"code","source":"from ensemble_boxes import *","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = [2, 2, 3, 2, 2, 3]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for id in list(yolov4_boxes.keys()):\n    boxes_list = [yolov4_boxes[id], yolov5_boxes[id], frcnn_boxes[id],\n                 yolov4_boxes[id], yolov5_boxes[id], frcnn_boxes[id]]\n    scores_list = [yolov4_scores[id], yolov5_scores[id], frcnn_scores[id],\n                  yolov4_scores[id], yolov5_scores[id], frcnn_scores[id]]\n    labels_list = [yolov4_labels[id], yolov5_labels[id], frcnn_labels[id],\n                  yolov4_labels[id], yolov5_labels[id], frcnn_labels[id]]\n    \n    boxes, scores, labels = weighted_boxes_fusion(\n        boxes_list, \n        scores_list, \n        labels_list, \n        weights=weights, \n        iou_thr=0.5)\n    \n    pred_str = \"\"\n    image_width, image_height = orig_shapes[id]\n    if len(boxes) > 1:\n        for i, box in enumerate(boxes):\n            xmin, ymin, xmax, ymax = np.array(box * np.array([image_width, image_height, image_width, image_height]), dtype = np.int)\n            score = scores[i]\n            pred_str += \"opacity {} {} {} {} {} \".format(score, xmin, ymin, xmax, ymax)\n        sub_df.loc[sub_df[\"id\"] == id + \"_image\", \"PredictionString\"] = pred_str","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Study Inference","metadata":{"execution":{"iopub.status.busy":"2021-07-31T20:11:48.592309Z","iopub.execute_input":"2021-07-31T20:11:48.592649Z","iopub.status.idle":"2021-07-31T20:11:48.597317Z","shell.execute_reply.started":"2021-07-31T20:11:48.592619Z","shell.execute_reply":"2021-07-31T20:11:48.595939Z"}}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras \nfrom tensorflow.keras import layers\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Flatten, Dense, Dropout\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization, GlobalAveragePooling2D\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test = os.listdir(\"./dataset/test\")\nx_test = list(map(lambda x : os.path.join(\"./dataset/test\", x), x_test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_mapping = { \n    0 : \"negative\",\n    1: \"typical\",\n    2: \"indeterminate\",\n    3: \"atypical\"}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Generator(Sequence) :\n    def __init__(self, image_filenames, batch_size, img_size): #, img_size) :\n        self.image_filenames = image_filenames\n        self.batch_size = batch_size\n        self.img_size = img_size\n\n    def __len__(self) :\n        return (np.ceil(len(self.image_filenames) / float(self.batch_size))).astype(np.int)\n\n    def __getitem__(self, idx) :\n        batch_x = self.image_filenames[idx * self.batch_size : (idx+1) * self.batch_size]\n        return np.array([self.get_image(file_name) for file_name in batch_x])\n    \n    def get_image(self, path):\n        img0 = cv2.imread(path)  # BGR\n        assert img0 is not None, 'Image Not Found ' + path\n        img = cv2.resize(np.copy(img0), (self.img_size, self.img_size))\n        # Convert\n        img = np.float32(img) / 255.\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)     \n        return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 8\ntest_gen = Generator(x_test, batch_size, 416)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#resnet_model = keras.models.load_model('./models/resnet50.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_metric(optimizer):\n    def lr(y_true, y_pred):\n        return optimizer.lr\n    return lr\n\noptimizer = keras.optimizers.Adam(0.001)\nlr_metric = get_lr_metric(optimizer)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\nfor i in range(4):\n    model = tf.keras.models.load_model(\"./models/model_{}.h5\".format(i),\n                                      custom_objects={\"lr\":lr_metric})\n    models.append(model)\n    del model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_preds = []\nfor i in range(4):\n    preds = models[i].predict(test_gen)\n    all_preds.append(preds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"weights = [0.8, 0.73, 0.82, 0.92]\ns = np.sum(weights)\nfor i in range(4):\n    weights[i] /= s\nnp.sum(weights)\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"predictions = np.zeros(shape=(all_preds[0].shape[0], 4))\n\nfor i in range(all_preds[0].shape[0]):\n    pos_pred = []\n    neg_pred = []\n    c_pred = []\n  \n    for j in range(4):\n        pos_pred.append(all_preds[j][i][-1])\n        neg_pred.append(all_preds[j][i][0])\n    \n    for j in range(4):\n        p = pos_pred[j] * np.sum(neg_pred[:j] + neg_pred[j+1:])\n        #print(np.sum(neg_pred[:j] + neg_pred[j+1:]))\n        c_pred.append(p)\n\n    final_p = []\n    for j in range(4):\n        #final_p.append(c_pred[j] / np.sum(c_pred))\n        final_p.append(weights[j] * c_pred[j] / np.sum(c_pred * np.array(weights)))\n\n    predictions[i] = final_p\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = np.zeros(shape=(all_preds[0].shape[0], 4))\n\nfor i in range(all_preds[0].shape[0]):\n    pos_pred = []\n    for j in range(4):\n        pos_pred.append(all_preds[j][i][-1])\n\n    final_p = []\n    for j in range(4):\n        final_p.append(pos_pred[j] / np.sum(pos_pred))\n        #final_p.append(weights[j] * pos_pred[j] / np.sum(pos_pred * np.array(weights)))\n    predictions[i] = final_p","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#predictions = resnet_model.predict(test_gen)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_dict = {}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k, pred in enumerate(predictions):\n    image_name = x_test[k].split(\"/\")[-1][:-4]\n    preds_dict[image_name] = pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_study_dict = {study_id: list() for study_id in set(study_mapping.values())}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image_id in study_mapping:\n    study_id = study_mapping[image_id]\n    \n    preds_study_dict[study_id].append(preds_dict[image_id])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for study_id in preds_study_dict:\n    if len(preds_study_dict[study_id]) > 1:\n        pred = np.mean(preds_study_dict[study_id], axis = 0)\n    else:\n        pred = preds_study_dict[study_id][0]\n    \n    pred_str = \"\"\n    \n    for i, score in enumerate(pred):\n        pred_str += \"{} {} 0 0 1 1 \".format(class_mapping[i], score)\n    sub_df.loc[sub_df[\"id\"] == study_id + \"_study\", \"PredictionString\"] = pred_str\n    #print(pred_str)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r gdcm\n!rm -r models\n!rm -r yolov5\n!rm -r yolov4\n!rm -r dataset\n!rm -r runs\n!rm gdcm.tar","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=False)\nsub_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}