{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is taken from https://www.kaggle.com/h053473666/siim-cov19-efnb7-yolov5-infer\n\nINSTALLING DEPENDENCIES","metadata":{}},{"cell_type":"code","source":"!conda install '/kaggle/input/pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-23T10:10:15.502381Z","iopub.execute_input":"2021-07-23T10:10:15.502763Z","iopub.status.idle":"2021-07-23T10:11:24.801510Z","shell.execute_reply.started":"2021-07-23T10:10:15.502678Z","shell.execute_reply":"2021-07-23T10:11:24.800574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"IMPORTS","metadata":{}},{"cell_type":"code","source":"import os\n\nfrom PIL import Image\nimport pandas as pd\nfrom tqdm.auto import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-23T10:12:14.331072Z","iopub.execute_input":"2021-07-23T10:12:14.331385Z","iopub.status.idle":"2021-07-23T10:12:14.335754Z","shell.execute_reply.started":"2021-07-23T10:12:14.331356Z","shell.execute_reply":"2021-07-23T10:12:14.334839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Making a seperate dataframe to make a faster submission as the original submission file takes time to process","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')\nif df.shape[0] == 2477:\n    fast_sub = True\n    fast_df = pd.DataFrame(([['00086460a852_study', 'negative 1 0 0 1 1'], \n                         ['000c9c05fd14_study', 'negative 1 0 0 1 1'], \n                         ['65761e66de9f_image', 'none 1 0 0 1 1'], \n                         ['51759b5579bc_image', 'none 1 0 0 1 1']]), \n                       columns=['id', 'PredictionString'])\nelse:\n    fast_sub = False\n","metadata":{"execution":{"iopub.status.busy":"2021-07-23T10:12:17.108543Z","iopub.execute_input":"2021-07-23T10:12:17.108898Z","iopub.status.idle":"2021-07-23T10:12:17.128587Z","shell.execute_reply.started":"2021-07-23T10:12:17.108863Z","shell.execute_reply":"2021-07-23T10:12:17.127860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Converting .dcm images into .png images\nA VOI LUT, if present, allows the transformation of the modality pixel values into pixel values that are meaningful for print or display. If not present, we simply convert into a pixel array. \n\nWe fix inversion in the images, if any, and then returned this transformed data. ","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-07-23T10:17:27.241530Z","iopub.execute_input":"2021-07-23T10:17:27.241898Z","iopub.status.idle":"2021-07-23T10:17:27.499447Z","shell.execute_reply.started":"2021-07-23T10:17:27.241863Z","shell.execute_reply":"2021-07-23T10:17:27.498493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Image resizing operation\nThe original is taken from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image","metadata":{}},{"cell_type":"code","source":"def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-07-23T10:20:00.871125Z","iopub.execute_input":"2021-07-23T10:20:00.871487Z","iopub.status.idle":"2021-07-23T10:20:00.877888Z","shell.execute_reply.started":"2021-07-23T10:20:00.871452Z","shell.execute_reply":"2021-07-23T10:20:00.876847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing study level test data\nNow we create a test directory '/kaggle/tmp/test/study/'\nIn case of a fast submission, manually transforming images and copying files into the new directory (names with '_study.png').\nOtherwise, copy all images in the test folder\n(size = 600)","metadata":{}},{"cell_type":"code","source":"\nsplit = 'test'\nsave_dir = f'/kaggle/tmp/{split}/'\n\nos.makedirs(save_dir, exist_ok=True)\n\nsave_dir = f'/kaggle/tmp/{split}/study/'\nos.makedirs(save_dir, exist_ok=True)\nif fast_sub:\n    xray = read_xray('../input/siim-covid19-detection/train/00086460a852/9e8302230c91/65761e66de9f.dcm')\n    im = resize(xray, size=600)  \n    study = '00086460a852' + '_study.png'\n    im.save(os.path.join(save_dir, study))\n    xray = read_xray('../input/siim-covid19-detection/train/000c9c05fd14/e555410bd2cd/51759b5579bc.dcm')\n    im = resize(xray, size=600)  \n    study = '000c9c05fd14' + '_study.png'\n    im.save(os.path.join(save_dir, study))\nelse:   \n    for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/{split}')):\n        for file in filenames:\n            # set keep_ratio=True to have original aspect ratio\n            xray = read_xray(os.path.join(dirname, file))\n            im = resize(xray, size=600)  \n            study = dirname.split('/')[-2] + '_study.png'\n            im.save(os.path.join(save_dir, study))\n","metadata":{"execution":{"iopub.status.busy":"2021-07-23T10:32:13.066439Z","iopub.execute_input":"2021-07-23T10:32:13.066855Z","iopub.status.idle":"2021-07-23T10:32:14.710759Z","shell.execute_reply.started":"2021-07-23T10:32:13.066816Z","shell.execute_reply":"2021-07-23T10:32:14.709887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing image level test data\n-We create three arrays - image_id, dim0 and dim1\n-Now we create an image level test directory '/kaggle/tmp/test/image/'\n-Images resized to 512\n-append image ID, shapes of the image to the three arrays","metadata":{}},{"cell_type":"code","source":"image_id = []\ndim0 = []\ndim1 = []\nsplits = []\nsave_dir = f'/kaggle/tmp/{split}/image/'\nos.makedirs(save_dir, exist_ok=True)\nif fast_sub:\n    xray = read_xray('../input/siim-covid19-detection/train/00086460a852/9e8302230c91/65761e66de9f.dcm')\n    im = resize(xray, size=512)  \n    im.save(os.path.join(save_dir,'65761e66de9f_image.png'))\n    image_id.append('65761e66de9f.dcm'.replace('.dcm', ''))\n    dim0.append(xray.shape[0])\n    dim1.append(xray.shape[1])\n    splits.append(split)\n    xray = read_xray('../input/siim-covid19-detection/train/000c9c05fd14/e555410bd2cd/51759b5579bc.dcm')\n    im = resize(xray, size=512)  \n    im.save(os.path.join(save_dir, '51759b5579bc_image.png'))\n    image_id.append('51759b5579bc.dcm'.replace('.dcm', ''))\n    dim0.append(xray.shape[0])\n    dim1.append(xray.shape[1])\n    splits.append(split)\nelse:\n    for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/{split}')):\n        for file in filenames:\n            # set keep_ratio=True to have original aspect ratio\n            xray = read_xray(os.path.join(dirname, file))\n            im = resize(xray, size=512)  \n            im.save(os.path.join(save_dir, file.replace('.dcm', '_image.png')))\n            image_id.append(file.replace('.dcm', ''))\n            dim0.append(xray.shape[0])\n            dim1.append(xray.shape[1])\n            splits.append(split)\nmeta = pd.DataFrame.from_dict({'image_id': image_id, 'dim0': dim0, 'dim1': dim1, 'split': splits})","metadata":{"execution":{"iopub.status.busy":"2021-07-23T10:52:11.087506Z","iopub.execute_input":"2021-07-23T10:52:11.087929Z","iopub.status.idle":"2021-07-23T10:52:11.478360Z","shell.execute_reply.started":"2021-07-23T10:52:11.087882Z","shell.execute_reply":"2021-07-23T10:52:11.477544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predicting results for Study-level\n","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nif fast_sub:\n    df = fast_df.copy()\nelse:\n    df = pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')\nid_laststr_list  = []\nfor i in range(df.shape[0]):\n    id_laststr_list.append(df.loc[i,'id'][-1])\ndf['id_last_str'] = id_laststr_list\n\nstudy_len = df[df['id_last_str'] == 'y'].shape[0]","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:03:08.156455Z","iopub.execute_input":"2021-07-23T11:03:08.156867Z","iopub.status.idle":"2021-07-23T11:03:08.174623Z","shell.execute_reply.started":"2021-07-23T11:03:08.156830Z","shell.execute_reply":"2021-07-23T11:03:08.172746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_laststr_list","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:03:36.519234Z","iopub.execute_input":"2021-07-23T11:03:36.519596Z","iopub.status.idle":"2021-07-23T11:03:36.525807Z","shell.execute_reply.started":"2021-07-23T11:03:36.519564Z","shell.execute_reply":"2021-07-23T11:03:36.524575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/kerasapplications -q\n!pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps\n\nimport os\n\nimport efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\ndef auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n\n    def decode_with_labels(path, label):\n        return decode(path), label\n\n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        return img\n\n    def augment_with_labels(img, label):\n        return augment(img), label\n\n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=32, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n\n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n\n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n\n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n\n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n\n    return dset\n\nstrategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 16\n\nIMSIZE = (224, 240, 260, 300, 380, 456, 528, 600, 512)\n\n#load_dir = f\"/kaggle/input/{COMPETITION_NAME}/\"\nif fast_sub:\n    sub_df = fast_df.copy()\nelse:\n    sub_df = pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')\nsub_df = sub_df[:study_len]\ntest_paths = f'/kaggle/tmp/{split}/study/' + sub_df['id'] +'.png'\n\nsub_df['negative'] = 0\nsub_df['typical'] = 0\nsub_df['indeterminate'] = 0\nsub_df['atypical'] = 0\n\n\nlabel_cols = sub_df.columns[2:]\n\ntest_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[7], IMSIZE[7]), ext='png')\ndtest = build_dataset(\n    test_paths, bsize=BATCH_SIZE, repeat=False, \n    shuffle=False, augment=False, cache=False,\n    decode_fn=test_decoder\n)\n\nwith strategy.scope():\n    \n    models = []\n    \n    models0 = tf.keras.models.load_model(\n        '../input/siim-covid19-efnb7-train-study/model0.h5'\n    )\n    models1 = tf.keras.models.load_model(\n        '../input/siim-covid19-efnb7-train-study/model1.h5'\n    )\n    models2 = tf.keras.models.load_model(\n        '../input/siim-covid19-efnb7-train-study/model2.h5'\n    )\n    models3 = tf.keras.models.load_model(\n        '../input/siim-covid19-efnb7-train-study/model3.h5'\n    )\n    models4 = tf.keras.models.load_model(\n        '../input/siim-covid19-efnb7-train-study/model4.h5'\n    )\n    \n    models.append(models0)\n    models.append(models1)\n    models.append(models2)\n    models.append(models3)\n    models.append(models4)\n\n    \n    \n    \nsub_df[label_cols] = sum([model.predict(dtest, verbose=1) for model in models]) / len(models)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:05:55.623769Z","iopub.execute_input":"2021-07-23T11:05:55.624117Z","iopub.status.idle":"2021-07-23T11:08:55.811331Z","shell.execute_reply.started":"2021-07-23T11:05:55.624086Z","shell.execute_reply":"2021-07-23T11:08:55.810441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.columns = ['id', 'PredictionString1', 'negative', 'typical', 'indeterminate', 'atypical']\ndf = pd.merge(df, sub_df, on = 'id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:11:22.508173Z","iopub.execute_input":"2021-07-23T11:11:22.508530Z","iopub.status.idle":"2021-07-23T11:11:22.520637Z","shell.execute_reply.started":"2021-07-23T11:11:22.508498Z","shell.execute_reply":"2021-07-23T11:11:22.519771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# study string","metadata":{}},{"cell_type":"code","source":"for i in range(study_len):\n    negative = df.loc[i,'negative']\n    typical = df.loc[i,'typical']\n    indeterminate = df.loc[i,'indeterminate']\n    atypical = df.loc[i,'atypical']\n    df.loc[i, 'PredictionString'] = f'negative {negative} 0 0 1 1 typical {typical} 0 0 1 1 indeterminate {indeterminate} 0 0 1 1 atypical {atypical} 0 0 1 1'","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:12:40.899464Z","iopub.execute_input":"2021-07-23T11:12:40.899798Z","iopub.status.idle":"2021-07-23T11:12:40.907363Z","shell.execute_reply.started":"2021-07-23T11:12:40.899752Z","shell.execute_reply":"2021-07-23T11:12:40.905609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_study = df[['id', 'PredictionString']]","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:12:46.102202Z","iopub.execute_input":"2021-07-23T11:12:46.102556Z","iopub.status.idle":"2021-07-23T11:12:46.108854Z","shell.execute_reply.started":"2021-07-23T11:12:46.102511Z","shell.execute_reply":"2021-07-23T11:12:46.107845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:13:22.696333Z","iopub.execute_input":"2021-07-23T11:13:22.696681Z","iopub.status.idle":"2021-07-23T11:13:22.713746Z","shell.execute_reply.started":"2021-07-23T11:13:22.696650Z","shell.execute_reply":"2021-07-23T11:13:22.712729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if fast_sub:\n    sub_df = fast_df.copy()\nelse:\n    sub_df = pd.read_csv('/kaggle/input/siim-covid19-detection/sample_submission.csv')\nsub_df = sub_df[study_len:]\ntest_paths = f'/kaggle/tmp/{split}/image/' + sub_df['id'] +'.png'\nsub_df['none'] = 0\n\nlabel_cols = sub_df.columns[2]\n\ntest_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[8], IMSIZE[8]), ext='png')\ndtest = build_dataset(\n    test_paths, bsize=BATCH_SIZE, repeat=False, \n    shuffle=False, augment=False, cache=False,\n    decode_fn=test_decoder\n)\n\nwith strategy.scope():\n    \n    models = []\n    \n    models0 = tf.keras.models.load_model(\n        '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model0.h5'\n    )\n    models1 = tf.keras.models.load_model(\n        '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model1.h5'\n    )\n    models2 = tf.keras.models.load_model(\n        '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model2.h5'\n    )\n    models3 = tf.keras.models.load_model(\n        '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model3.h5'\n    )\n    models4 = tf.keras.models.load_model(\n        '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model4.h5'\n    )\n    \n    models.append(models0)\n    models.append(models1)\n    models.append(models2)\n    models.append(models3)\n    models.append(models4)\n\nweights = {\n    0: 1,\n    1: 1,\n    2: 1,\n    3: 1,\n    4: 3\n}\n\nweights_sum = sum(weights.values())\nweights = {k: v/weights_sum for k, v in weights.items()}\n\npredictions = [model.predict(dtest, verbose=1) for model in models]\nfor i, pred in enumerate(predictions):\n    predictions[i] = weights[i] * pred\n    \nsub_df[label_cols] = sum(predictions)\n#sub_df[label_cols] = sum([model.predict(dtest, verbose=1) for model in models]) / len(models)\ndf_2class = sub_df.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:18:34.130427Z","iopub.execute_input":"2021-07-23T11:18:34.130754Z","iopub.status.idle":"2021-07-23T11:20:36.974070Z","shell.execute_reply.started":"2021-07-23T11:18:34.130723Z","shell.execute_reply":"2021-07-23T11:20:36.973249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del models\ndel models0, models1, models2, models3, models4","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:20:43.343137Z","iopub.execute_input":"2021-07-23T11:20:43.343474Z","iopub.status.idle":"2021-07-23T11:20:43.347402Z","shell.execute_reply.started":"2021-07-23T11:20:43.343444Z","shell.execute_reply":"2021-07-23T11:20:43.346187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numba import cuda\nimport torch\ncuda.select_device(0)\ncuda.close()\ncuda.select_device(0)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:20:45.869564Z","iopub.execute_input":"2021-07-23T11:20:45.869944Z","iopub.status.idle":"2021-07-23T11:20:48.149288Z","shell.execute_reply.started":"2021-07-23T11:20:45.869909Z","shell.execute_reply":"2021-07-23T11:20:48.148471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# yolov5 predict","metadata":{}},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns\nimport torch","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:20:52.561370Z","iopub.execute_input":"2021-07-23T11:20:52.561706Z","iopub.status.idle":"2021-07-23T11:20:52.753867Z","shell.execute_reply.started":"2021-07-23T11:20:52.561677Z","shell.execute_reply":"2021-07-23T11:20:52.752979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta = meta[meta['split'] == 'test']\nif fast_sub:\n    test_df = fast_df.copy()\nelse:\n    test_df = pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')\ntest_df = df[study_len:].reset_index(drop=True) \nmeta['image_id'] = meta['image_id'] + '_image'\nmeta.columns = ['id', 'dim0', 'dim1', 'split']\ntest_df = pd.merge(test_df, meta, on = 'id', how = 'left')\n","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:20:55.630182Z","iopub.execute_input":"2021-07-23T11:20:55.630636Z","iopub.status.idle":"2021-07-23T11:20:55.646671Z","shell.execute_reply.started":"2021-07-23T11:20:55.630563Z","shell.execute_reply":"2021-07-23T11:20:55.645710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dim = 512 #1024, 256, 'original'\ntest_dir = f'/kaggle/tmp/{split}/image'\n#weights_dir = '/kaggle/input/siim-cov19-yolov5-train/yolov5/runs/train/exp/weights/best.pt'\nweights_dir = '/kaggle/input/weights-of-yolov5-150-epochs/best.pt'\n\nshutil.copytree('/kaggle/input/yolov5-repo/yolov5-master', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5') # install dependencies\n!git pull\n\nimport torch\n#from IPython.display import Image, clear_output  # to display images\n\n#clear_output()\n#print('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))\n\n\n!python detect.py --weights $weights_dir\\\n--img 512\\\n--conf 0.001\\\n--iou 0.5\\\n--augment\\\n--max 50\\\n--source $test_dir\\\n--save-txt --save-conf --exist-ok\ndef yolo2voc(image_height, image_width, bboxes):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y1]\n\n    \"\"\" \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n\n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n\n    return bboxes\nimage_ids = []\nPredictionStrings = []\n\nfor file_path in tqdm(glob('runs/detect/exp/labels/*.txt')):\n    image_id = file_path.split('/')[-1].split('.')[0]\n    w, h = test_df.loc[test_df.id==image_id,['dim1', 'dim0']].values[0]\n    f = open(file_path, 'r')\n    data = np.array(f.read().replace('\\n', ' ').strip().split(' ')).astype(np.float32).reshape(-1, 6)\n    data = data[:, [0, 5, 1, 2, 3, 4]]\n    bboxes = list(np.round(np.concatenate((data[:, :2], np.round(yolo2voc(h, w, data[:, 2:]))), axis =1).reshape(-1), 12).astype(str))\n    for idx in range(len(bboxes)):\n        bboxes[idx] = str(int(float(bboxes[idx]))) if idx%6!=1 else bboxes[idx]\n    image_ids.append(image_id)\n    PredictionStrings.append(' '.join(bboxes))\n\n\npred_df = pd.DataFrame({'id':image_ids,\n                        'PredictionString':PredictionStrings})","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:25:16.257282Z","iopub.execute_input":"2021-07-23T11:25:16.257657Z","iopub.status.idle":"2021-07-23T11:25:32.183514Z","shell.execute_reply.started":"2021-07-23T11:25:16.257623Z","shell.execute_reply":"2021-07-23T11:25:32.182513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df.drop(['PredictionString'], axis=1)\nsub_df = pd.merge(test_df, pred_df, on = 'id', how = 'left').fillna(\"none 1 0 0 1 1\")\nsub_df = sub_df[['id', 'PredictionString']]\nfor i in range(sub_df.shape[0]):\n    if sub_df.loc[i,'PredictionString'] == \"none 1 0 0 1 1\":\n        continue\n    sub_df_split = sub_df.loc[i,'PredictionString'].split()\n    sub_df_list = []\n    for j in range(int(len(sub_df_split) / 6)):\n        sub_df_list.append('opacity')\n        sub_df_list.append(sub_df_split[6 * j + 1])\n        sub_df_list.append(sub_df_split[6 * j + 2])\n        sub_df_list.append(sub_df_split[6 * j + 3])\n        sub_df_list.append(sub_df_split[6 * j + 4])\n        sub_df_list.append(sub_df_split[6 * j + 5])\n    sub_df.loc[i,'PredictionString'] = ' '.join(sub_df_list)\nsub_df['none'] = df_2class['none'] \nfor i in range(sub_df.shape[0]):\n    if sub_df.loc[i,'PredictionString'] != 'none 1 0 0 1 1':\n        sub_df.loc[i,'PredictionString'] = sub_df.loc[i,'PredictionString'] + ' none ' + str(sub_df.loc[i,'none']) + ' 0 0 1 1'\nsub_df = sub_df[['id', 'PredictionString']]   \ndf_study = df_study[:study_len]\ndf_study = df_study.append(sub_df).reset_index(drop=True)\ndf_study.to_csv('/kaggle/working/submission.csv',index = False)  \nshutil.rmtree('/kaggle/working/yolov5')","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]}]}