{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Imports","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-23T06:59:42.679483Z","iopub.execute_input":"2022-02-23T06:59:42.679845Z","iopub.status.idle":"2022-02-23T07:00:04.139315Z","shell.execute_reply.started":"2022-02-23T06:59:42.679767Z","shell.execute_reply":"2022-02-23T07:00:04.138292Z"}}},{"cell_type":"code","source":"!cp ../input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:09:08.611568Z","iopub.execute_input":"2022-03-15T03:09:08.61187Z","iopub.status.idle":"2022-03-15T03:09:30.266524Z","shell.execute_reply.started":"2022-03-15T03:09:08.611793Z","shell.execute_reply":"2022-03-15T03:09:30.265532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \n!conda install '/kaggle/input/pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:09:30.268777Z","iopub.execute_input":"2022-03-15T03:09:30.269143Z","iopub.status.idle":"2022-03-15T03:10:24.29813Z","shell.execute_reply.started":"2022-03-15T03:09:30.269104Z","shell.execute_reply":"2022-03-15T03:10:24.296995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#2\nimport os\nimport glob\nfrom tqdm import tqdm\nfrom pathlib import Path\n\nimport pandas as pd\nimport numpy as np\n\n# from pydicom.pixel_data_handlers.util import apply_voi_lut\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom collections import Counter\n\nfrom PIL import Image\nimport gdcm\nimport pydicom\n\nimport cv2\nimport random","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:10:24.30102Z","iopub.execute_input":"2022-03-15T03:10:24.301648Z","iopub.status.idle":"2022-03-15T03:10:25.283661Z","shell.execute_reply.started":"2022-03-15T03:10:24.301601Z","shell.execute_reply":"2022-03-15T03:10:25.282751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Preparation","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')\nif df.shape[0] == 2477:\n    fast_sub = True\n    df.to_csv('submission.csv', index=False)\nelse:\n    fast_sub = False","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-15T03:10:25.285551Z","iopub.execute_input":"2022-03-15T03:10:25.286081Z","iopub.status.idle":"2022-03-15T03:10:25.437179Z","shell.execute_reply.started":"2022-03-15T03:10:25.286039Z","shell.execute_reply":"2022-03-15T03:10:25.436373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## set to True for faster inference (uses smaller dataset)\nfast_sub = False","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:10:25.438346Z","iopub.execute_input":"2022-03-15T03:10:25.438701Z","iopub.status.idle":"2022-03-15T03:10:25.443512Z","shell.execute_reply.started":"2022-03-15T03:10:25.438666Z","shell.execute_reply":"2022-03-15T03:10:25.442343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nfrom PIL import Image\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport numpy as np\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\ndef resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im\n\nimage_id = []\nstudy_id = []\ndim0 = []\ndim1 = []\nsplit = 'test'\nsave_dir = f'/kaggle/tmp/{split}/image/'\nos.makedirs(save_dir, exist_ok=True)\n\nfor dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/{split}')):\n    for file in filenames:\n        # set keep_ratio=True to have original aspect ratio\n        xray = read_xray(os.path.join(dirname, file))\n        im = resize(xray, size=512)  \n        im.save(os.path.join(save_dir, file.replace('.dcm', '.png')))\n        image_id.append(file.replace('.dcm', ''))\n        study_id.append(dirname.split('/')[-2])\n        dim0.append(xray.shape[0])\n        dim1.append(xray.shape[1])\n        \n        if len(dim0) > 3 and fast_sub:\n            break\n    if len(dim0) > 3 and fast_sub:\n            break\nmeta = pd.DataFrame.from_dict({'image_id': image_id, 'dim0': dim0, 'dim1': dim1, 'study_id': study_id})","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:10:25.445098Z","iopub.execute_input":"2022-03-15T03:10:25.445889Z","iopub.status.idle":"2022-03-15T03:21:11.64814Z","shell.execute_reply.started":"2022-03-15T03:10:25.445841Z","shell.execute_reply":"2022-03-15T03:21:11.647282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -a \"/kaggle/tmp/test/image\"","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:21:11.649438Z","iopub.execute_input":"2022-03-15T03:21:11.649926Z","iopub.status.idle":"2022-03-15T03:21:12.328808Z","shell.execute_reply.started":"2022-03-15T03:21:11.649888Z","shell.execute_reply":"2022-03-15T03:21:12.327809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !zip -r 'images_set.zip' '/kaggle/tmp/test/image'","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:21:12.332938Z","iopub.execute_input":"2022-03-15T03:21:12.333215Z","iopub.status.idle":"2022-03-15T03:21:12.337318Z","shell.execute_reply.started":"2022-03-15T03:21:12.333187Z","shell.execute_reply":"2022-03-15T03:21:12.336292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta.to_csv('test_meta.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:21:12.339773Z","iopub.execute_input":"2022-03-15T03:21:12.340194Z","iopub.status.idle":"2022-03-15T03:21:12.358303Z","shell.execute_reply.started":"2022-03-15T03:21:12.340157Z","shell.execute_reply":"2022-03-15T03:21:12.357493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:21:12.359659Z","iopub.execute_input":"2022-03-15T03:21:12.360011Z","iopub.status.idle":"2022-03-15T03:21:12.38353Z","shell.execute_reply.started":"2022-03-15T03:21:12.359973Z","shell.execute_reply":"2022-03-15T03:21:12.382537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image enhancement using Histogram equalization ","metadata":{}},{"cell_type":"code","source":"# def rescale_dcm(dcm_px):\n#     zero_one = (dcm_px - dcm_px.min())/((dcm_px.max() - dcm_px.min()))\n#     rescaled = (zero_one * 255).astype(np.uint8)\n#     return rescaled\n\n# def apply_hist_equalization(array):\n#     clahe = cv2.createCLAHE(clipLimit = 2, tileGridSize = (8,8))\n#     cl_array = clahe.apply(array)\n#     return cl_array\n\n# SAMPLE_MEAN = 134.0\n# SAMPLE_STD = 56.0","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:21:12.384999Z","iopub.execute_input":"2022-03-15T03:21:12.385348Z","iopub.status.idle":"2022-03-15T03:21:12.389376Z","shell.execute_reply.started":"2022-03-15T03:21:12.385314Z","shell.execute_reply":"2022-03-15T03:21:12.38845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_df = meta.head(6)\n# sample_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:21:12.390952Z","iopub.execute_input":"2022-03-15T03:21:12.391585Z","iopub.status.idle":"2022-03-15T03:21:12.400836Z","shell.execute_reply.started":"2022-03-15T03:21:12.391549Z","shell.execute_reply":"2022-03-15T03:21:12.399886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_df = meta.head(6)\n# TRAIN = '/kaggle/input/siim-covid19-detection/test/'\n\n# # Plot\n# fig, ax = plt.subplots(nrows = 6, ncols = 4, figsize = (25,30))\n# r = 0\n# c = 0\n\n# for idx, image in tqdm(sample_df.iterrows(), total = len(sample_df)):\n\n#     dcm_path = glob.glob(os.path.join(TRAIN, \n#                                       image.study_id, \n#                                       \"*\",\n#                                       image.image_id+\".dcm\"))[0]\n# #     print(dcm_path)\n    \n#     # Read the .dcm metadata\n#     dcm = pydicom.dcmread(dcm_path)\n      \n#     # Get the pixel array\n#     dcm_px = dcm.pixel_array\n\n#     if dcm.PhotometricInterpretation == \"MONOCHROME1\":\n#         dcm_px = np.amax(dcm_px) - dcm_px\n    \n#     # Rescale the values in the 0-255 range\n#     dcm_rescaled = rescale_dcm(dcm_px)\n    \n#     #Histogram equalization\n#     cl_array = apply_hist_equalization(dcm_rescaled)\n    \n# #     std_array = (cl_array - SAMPLE_MEAN)/ SAMPLE_STD\n# #     std_array = cv2.resize(std_array, (512,512))\n    \n    \n#     ax[r, c].imshow(dcm_rescaled)\n\n#     ax[r, c+1].hist(dcm_rescaled.flatten(), bins = 100)\n#     ax[r, c].set_ylabel(f\"image id: {image.image_id}\")\n    \n#     ax[r+1, c].imshow(cl_array)\n#     ax[r+1, c].set_ylabel(f\"enhanced image id:, {image.image_id}\")\n#     ax[r+1, c+1].hist(cl_array.flatten(), bins = 100)\n    \n    \n#     c = c + 2\n#     if c%4 == 0:\n#         c = 0\n#         r = r+2\n","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:21:12.40212Z","iopub.execute_input":"2022-03-15T03:21:12.402559Z","iopub.status.idle":"2022-03-15T03:21:12.409264Z","shell.execute_reply.started":"2022-03-15T03:21:12.402524Z","shell.execute_reply":"2022-03-15T03:21:12.408269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig.savefig(\"histogram.png\")","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:21:12.410735Z","iopub.execute_input":"2022-03-15T03:21:12.411152Z","iopub.status.idle":"2022-03-15T03:21:12.417504Z","shell.execute_reply.started":"2022-03-15T03:21:12.411119Z","shell.execute_reply":"2022-03-15T03:21:12.416574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fast_df = pd.read_csv('test_meta.csv')\n# fast_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:21:12.418866Z","iopub.execute_input":"2022-03-15T03:21:12.419252Z","iopub.status.idle":"2022-03-15T03:21:12.425822Z","shell.execute_reply.started":"2022-03-15T03:21:12.419218Z","shell.execute_reply":"2022-03-15T03:21:12.424867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image Level Object Detection","metadata":{}},{"cell_type":"code","source":"!mkdir det_txt\n!mkdir det\n! pip install ../input/siim-libs/addict-2.4.0-py3-none-any.whl >> /dev/null\n! pip install ../input/siim-libs/timm-0.4.12-py3-none-any.whl >> /dev/null\n! pip install ../input/siim-libs/ensemble_boxes-1.0.6-py3-none-any.whl >> /dev/null\n! pip install ../input/siim-libs/loguru-0.5.3-py3-none-any.whl >> /dev/null\n! pip install ../input/siim-libs/thop-0.0.31.post2005241907-py3-none-any.whl >> /dev/null\n! pip install ../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl >> /dev/null\n! pip install ../input/omegaconf/omegaconf-2.0.5-py3-none-any.whl >> /dev/null\n! pip install -q ../input/landmark-additional-packages/EfficientNet-PyTorch/EfficientNet-PyTorch-master \n! pip install -U ../input/landmark-additional-packages/timm-0.4.12-py3-none-any.whl # to fix \n! pip install ../input/siim-libs/segmentation_models_pytorch-0.1.3-py3-none-any.whl --no-deps  ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-15T03:21:12.427007Z","iopub.execute_input":"2022-03-15T03:21:12.427475Z","iopub.status.idle":"2022-03-15T03:25:33.454637Z","shell.execute_reply.started":"2022-03-15T03:21:12.427441Z","shell.execute_reply":"2022-03-15T03:25:33.453513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Yolov5 inference","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/siim-v5-mh/* .","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:25:33.456525Z","iopub.execute_input":"2022-03-15T03:25:33.456915Z","iopub.status.idle":"2022-03-15T03:25:34.441013Z","shell.execute_reply.started":"2022-03-15T03:25:33.456872Z","shell.execute_reply":"2022-03-15T03:25:34.439945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# our inference\n# !cp -r ../input/yolov5inference/* .","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:25:34.444446Z","iopub.execute_input":"2022-03-15T03:25:34.444719Z","iopub.status.idle":"2022-03-15T03:25:34.448121Z","shell.execute_reply.started":"2022-03-15T03:25:34.44469Z","shell.execute_reply":"2022-03-15T03:25:34.447304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ./inference.py --is_val 0 --output_path det_txt/ --input_path '/kaggle/tmp/test/image/*png' --weight_path '/kaggle/input/siim-v5-weights/*/*/*/best.pt'","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-15T03:25:34.449388Z","iopub.execute_input":"2022-03-15T03:25:34.449948Z","iopub.status.idle":"2022-03-15T03:52:53.513547Z","shell.execute_reply.started":"2022-03-15T03:25:34.449912Z","shell.execute_reply":"2022-03-15T03:52:53.512584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r ../input/yolox-inference/* .","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:52:53.51517Z","iopub.execute_input":"2022-03-15T03:52:53.515561Z","iopub.status.idle":"2022-03-15T03:52:54.441878Z","shell.execute_reply.started":"2022-03-15T03:52:53.515527Z","shell.execute_reply":"2022-03-15T03:52:54.440559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from kaggle_datasets import KaggleDatasets\n# dataset_name = '  ...   '\n# GCS_PATH = KaggleDatasets().get_gcs_path(dataset_name)\n# print(GCS_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:52:54.448542Z","iopub.execute_input":"2022-03-15T03:52:54.44885Z","iopub.status.idle":"2022-03-15T03:52:54.453446Z","shell.execute_reply.started":"2022-03-15T03:52:54.448818Z","shell.execute_reply":"2022-03-15T03:52:54.452068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Yolox Inference","metadata":{}},{"cell_type":"code","source":"!python inference.py -f ../input/yolox-weights/yolox_weights/yolox_siim_d_f0.py -c ss --path '/kaggle/tmp/test/image/*png' \\\n    --wei_dir ../input/yolox-weights/yolox_weights/ --conf 0.0001 --nms 0.5 --tsize 384 --save_result --device gpu","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:52:54.458964Z","iopub.execute_input":"2022-03-15T03:52:54.46075Z","iopub.status.idle":"2022-03-15T03:57:46.804901Z","shell.execute_reply.started":"2022-03-15T03:52:54.460713Z","shell.execute_reply":"2022-03-15T03:57:46.803789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### EfficientDet inference","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/siimeffdet/* .","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:57:46.812115Z","iopub.execute_input":"2022-03-15T03:57:46.812463Z","iopub.status.idle":"2022-03-15T03:57:47.88184Z","shell.execute_reply.started":"2022-03-15T03:57:46.812427Z","shell.execute_reply":"2022-03-15T03:57:47.88045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from kaggle_datasets import KaggleDatasets\n# dataset_name = 'siim-det-models'\n# GCS_PATH = KaggleDatasets().get_gcs_path(dataset_name)\n# print(GCS_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T03:57:47.885632Z","iopub.execute_input":"2022-03-15T03:57:47.885936Z","iopub.status.idle":"2022-03-15T03:57:47.889786Z","shell.execute_reply.started":"2022-03-15T03:57:47.885903Z","shell.execute_reply":"2022-03-15T03:57:47.888824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python predict_oof_det.py --is_val 0 --test_path '/kaggle/tmp/test/image/*png' --model_dir ../input/siim-det-models/","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-15T03:57:47.891648Z","iopub.execute_input":"2022-03-15T03:57:47.892049Z","iopub.status.idle":"2022-03-15T04:03:07.669172Z","shell.execute_reply.started":"2022-03-15T03:57:47.892012Z","shell.execute_reply":"2022-03-15T04:03:07.668211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! cp det/* det_txt","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:03:07.670843Z","iopub.execute_input":"2022-03-15T04:03:07.671239Z","iopub.status.idle":"2022-03-15T04:03:08.52857Z","shell.execute_reply.started":"2022-03-15T04:03:07.6712Z","shell.execute_reply":"2022-03-15T04:03:08.527455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Ensemble mechanism","metadata":{}},{"cell_type":"code","source":"! python ensemble1.py --input_path  'det_txt/*txt' --image_path '/kaggle/tmp/test/image/*png' --thr 0.001","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:03:08.530282Z","iopub.execute_input":"2022-03-15T04:03:08.530732Z","iopub.status.idle":"2022-03-15T04:10:54.899851Z","shell.execute_reply.started":"2022-03-15T04:03:08.53069Z","shell.execute_reply":"2022-03-15T04:10:54.898874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#mask_dir = f'/kaggle/tmp/{split}/mask/'\n#os.makedirs(mask_dir, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:10:54.902543Z","iopub.execute_input":"2022-03-15T04:10:54.902964Z","iopub.status.idle":"2022-03-15T04:10:54.906635Z","shell.execute_reply.started":"2022-03-15T04:10:54.902925Z","shell.execute_reply":"2022-03-15T04:10:54.905389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!cp -r ../input/siimsegs/* .","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:10:54.907995Z","iopub.execute_input":"2022-03-15T04:10:54.908736Z","iopub.status.idle":"2022-03-15T04:10:54.915745Z","shell.execute_reply.started":"2022-03-15T04:10:54.908698Z","shell.execute_reply":"2022-03-15T04:10:54.914953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!python train_seg.py --image_path '/kaggle/tmp/test/image/*png' --weight_path best_loss.pth","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-15T04:10:54.917085Z","iopub.execute_input":"2022-03-15T04:10:54.917528Z","iopub.status.idle":"2022-03-15T04:10:54.924395Z","shell.execute_reply.started":"2022-03-15T04:10:54.917494Z","shell.execute_reply":"2022-03-15T04:10:54.923439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Study Level Classification","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/siim-cls-code/* .","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:10:54.925682Z","iopub.execute_input":"2022-03-15T04:10:54.92626Z","iopub.status.idle":"2022-03-15T04:10:56.340557Z","shell.execute_reply.started":"2022-03-15T04:10:54.926225Z","shell.execute_reply":"2022-03-15T04:10:56.33948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###### NfNet","metadata":{}},{"cell_type":"code","source":"!python main.py -C n_cf11_6 -M test -W ../input/siim-cls-weights/n_cf11_6_f3/\n!python main.py -C n_cf11 -M test -W ../input/siim-cls-weights/n_cf11_l1/\n!python main.py -C n_cf11_7 -M test -W ../input/siim-cls-weights/n_cf11_7/\n!python main.py -C n_cf11_9 -M test -W ../input/siim-cls-weights/n_cf11_9/\n!python main.py -C n_cf11_10 -M test -W ../input/siim-cls-weights/n_cf11_10/\n!python main.py -C n_cf11_1 -M test -W ../input/siim-cls-weights/n_cf_11_1/\n!python main.py -C n_cf11_rot1 -M test -W ../input/siim-cls-weights/n_cf11_rot1/","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:10:56.342121Z","iopub.execute_input":"2022-03-15T04:10:56.342509Z","iopub.status.idle":"2022-03-15T04:43:02.556442Z","shell.execute_reply.started":"2022-03-15T04:10:56.34247Z","shell.execute_reply":"2022-03-15T04:43:02.555445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Chexnet","metadata":{}},{"cell_type":"code","source":"IMG_SIZE = 512\nimport tensorflow as tf\n\nbase = tf.keras.applications.densenet.DenseNet121(weights=None,\n                            include_top=True,\n                            input_shape=(IMG_SIZE,IMG_SIZE,3),\n                            classes=14\n                           )\n\nx = tf.keras.layers.Dense(5,activation='sigmoid')(base.output)\n# fine_tune = tf.keras.layers.Dense(NUM_CLASSES,activation='sigmoid')(x)\n\nbase = tf.keras.Model(inputs=base.input, outputs= x)\n\nprint(\"CheXNet loaded\")\nbase.trainable= False # freeze most layers\nbase.training= False\n\nbase.layers.pop()\n","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:43:02.558053Z","iopub.execute_input":"2022-03-15T04:43:02.558425Z","iopub.status.idle":"2022-03-15T04:43:07.472082Z","shell.execute_reply.started":"2022-03-15T04:43:02.558383Z","shell.execute_reply":"2022-03-15T04:43:07.471255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base.load_weights('../input/model-chexnet/model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:43:07.474946Z","iopub.execute_input":"2022-03-15T04:43:07.475204Z","iopub.status.idle":"2022-03-15T04:43:08.975217Z","shell.execute_reply.started":"2022-03-15T04:43:07.475178Z","shell.execute_reply":"2022-03-15T04:43:08.974353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255)\ntest_generator = test_datagen.flow_from_directory(\n        '/kaggle/tmp/test/',\n        classes=['image'],\n        target_size=(512, 512),\n        color_mode=\"rgb\",\n        shuffle = False,\n        class_mode=None,\n        batch_size=1)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:43:08.977043Z","iopub.execute_input":"2022-03-15T04:43:08.97739Z","iopub.status.idle":"2022-03-15T04:43:09.091989Z","shell.execute_reply.started":"2022-03-15T04:43:08.977339Z","shell.execute_reply":"2022-03-15T04:43:09.091096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = base.predict_generator(test_generator)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:43:09.093348Z","iopub.execute_input":"2022-03-15T04:43:09.093715Z","iopub.status.idle":"2022-03-15T04:43:37.028241Z","shell.execute_reply.started":"2022-03-15T04:43:09.093676Z","shell.execute_reply":"2022-03-15T04:43:37.027128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\nchexnet_pred = pd.DataFrame(predictions , columns = ['pred_cls1','pred_cls2','pred_cls3','pred_cls4','pred_cls5'])","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:43:37.032186Z","iopub.execute_input":"2022-03-15T04:43:37.032494Z","iopub.status.idle":"2022-03-15T04:43:37.03924Z","shell.execute_reply.started":"2022-03-15T04:43:37.032465Z","shell.execute_reply":"2022-03-15T04:43:37.038213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files = os.listdir('/kaggle/tmp/test/image')\nfor i in files:\n    files\nchexnet_pred['image_id'] = files\nchexnet_pred['image_id'] =chexnet_pred['image_id'].apply(lambda x: x.split('.')[0])\nchexnet_pred.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:43:37.040781Z","iopub.execute_input":"2022-03-15T04:43:37.041189Z","iopub.status.idle":"2022-03-15T04:43:37.066052Z","shell.execute_reply.started":"2022-03-15T04:43:37.041155Z","shell.execute_reply":"2022-03-15T04:43:37.065288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### EfficientNet","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/kerasapplications -q\n!pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps\n\nimport os\n\nimport efficientnet.tfkeras as efn\nimport tensorflow as tf\n\ndef auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n\n    def decode_with_labels(path, label):\n        return decode(path), label\n\n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        return img\n\n    def augment_with_labels(img, label):\n        return augment(img), label\n\n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=32, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n\n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n\n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n\n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n\n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n\n    return dset\n\nstrategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 16\n\nIMSIZE = (224, 240, 260, 300, 380, 456, 528, 600, 512)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:43:37.067269Z","iopub.execute_input":"2022-03-15T04:43:37.067653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### 4 class classification","metadata":{}},{"cell_type":"code","source":"sub_df = meta.copy()\ntest_paths = f'/kaggle/tmp/{split}/image/' + sub_df['image_id'] +'.png'\n\nsub_df['negative'] = 0\nsub_df['typical'] = 0\nsub_df['indeterminate'] = 0\nsub_df['atypical'] = 0\n\n\nlabel_cols = sub_df.columns[4:]\n\ntest_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[7], IMSIZE[7]), ext='png')\ndtest = build_dataset(\n    test_paths, bsize=BATCH_SIZE, repeat=False, \n    shuffle=False, augment=False, cache=False,\n    decode_fn=test_decoder\n)\n\n\nwith strategy.scope():\n    \n    models = []\n    \n    models0 = tf.keras.models.load_model(\n        '../input/siim-covid19-efnb7-train-study/model0.h5'\n    )\n    models1 = tf.keras.models.load_model(\n        '../input/siim-covid19-efnb7-train-study/model1.h5'\n    )\n    models2 = tf.keras.models.load_model(\n        '../input/siim-covid19-efnb7-train-study/model2.h5'\n    )\n    models3 = tf.keras.models.load_model(\n        '../input/siim-covid19-efnb7-train-study/model3.h5'\n    )\n    models4 = tf.keras.models.load_model(\n        '../input/siim-covid19-efnb7-train-study/model4.h5'\n    )\n    \n    models.append(models0)\n    models.append(models1)\n    models.append(models2)\n    models.append(models3)\n    models.append(models4)\n    \nsub_df[label_cols] = sum([model.predict(dtest, verbose=1) for model in models]) / len(models)\n","metadata":{"execution":{"iopub.status.idle":"2022-03-15T04:51:49.7377Z","shell.execute_reply.started":"2022-03-15T04:44:29.55492Z","shell.execute_reply":"2022-03-15T04:51:49.736813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('sheep_df.csv' , index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:51:49.740254Z","iopub.execute_input":"2022-03-15T04:51:49.741659Z","iopub.status.idle":"2022-03-15T04:51:49.778376Z","shell.execute_reply.started":"2022-03-15T04:51:49.741618Z","shell.execute_reply":"2022-03-15T04:51:49.777481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:51:49.780982Z","iopub.execute_input":"2022-03-15T04:51:49.782356Z","iopub.status.idle":"2022-03-15T04:51:49.801752Z","shell.execute_reply.started":"2022-03-15T04:51:49.782317Z","shell.execute_reply":"2022-03-15T04:51:49.800973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### 2 class classification","metadata":{}},{"cell_type":"code","source":"\nsub_df = meta.copy()\ntest_paths = f'/kaggle/tmp/{split}/image/' + sub_df['image_id'] +'.png'\n\nsub_df['none'] = 0\n\nlabel_cols = sub_df.columns[4]\n\ntest_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[8], IMSIZE[8]), ext='png')\ndtest = build_dataset(\n    test_paths, bsize=BATCH_SIZE, repeat=False, \n    shuffle=False, augment=False, cache=False,\n    decode_fn=test_decoder\n)\n\nwith strategy.scope():\n    \n    models = []\n    \n    models0 = tf.keras.models.load_model(\n        '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model0.h5'\n    )\n    models1 = tf.keras.models.load_model(\n        '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model1.h5'\n    )\n    models2 = tf.keras.models.load_model(\n        '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model2.h5'\n    )\n    models3 = tf.keras.models.load_model(\n        '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model3.h5'\n    )\n    models4 = tf.keras.models.load_model(\n        '/kaggle/input/siim-covid19-efnb7-train-fold0-5-2class/model4.h5'\n    )\n    \n    models.append(models0)\n    models.append(models1)\n    models.append(models2)\n    models.append(models3)\n    models.append(models4)\n\nweights = {\n    0: 1,\n    1: 1,\n    2: 1,\n    3: 1,\n    4: 3\n}\n\nweights_sum = sum(weights.values())\nweights = {k: v/weights_sum for k, v in weights.items()}\n\npredictions = [model.predict(dtest, verbose=1) for model in models]\nfor i, pred in enumerate(predictions):\\\n    predictions[i] = weights[i] * pred\n    \nsub_df[label_cols] = sum(predictions)\n#sub_df[label_cols] = sum([model.predict(dtest, verbose=1) for model in models]) / len(models)\ndf_2class = sub_df.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:51:49.80398Z","iopub.execute_input":"2022-03-15T04:51:49.805292Z","iopub.status.idle":"2022-03-15T04:57:29.655408Z","shell.execute_reply.started":"2022-03-15T04:51:49.805254Z","shell.execute_reply":"2022-03-15T04:57:29.654484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Results for classification and detection","metadata":{}},{"cell_type":"code","source":"df_2class.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:29.65677Z","iopub.execute_input":"2022-03-15T04:57:29.657315Z","iopub.status.idle":"2022-03-15T04:57:29.670176Z","shell.execute_reply.started":"2022-03-15T04:57:29.657272Z","shell.execute_reply":"2022-03-15T04:57:29.668874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2class.to_csv('public_test_sheep_predict_none.csv' , index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:29.671638Z","iopub.execute_input":"2022-03-15T04:57:29.671988Z","iopub.status.idle":"2022-03-15T04:57:29.689631Z","shell.execute_reply.started":"2022-03-15T04:57:29.671954Z","shell.execute_reply":"2022-03-15T04:57:29.688639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('test_meta.csv')[[\"image_id\", \"study_id\"]]\ntest = test.rename(columns={\"image_id\": \"ImageUID\", \"study_id\": \"StudyInstanceUID\"})\nfor e in ['Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']:\n    test[e] = 0","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:29.690936Z","iopub.execute_input":"2022-03-15T04:57:29.691297Z","iopub.status.idle":"2022-03-15T04:57:29.706978Z","shell.execute_reply.started":"2022-03-15T04:57:29.691263Z","shell.execute_reply":"2022-03-15T04:57:29.70614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head(25)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:29.70825Z","iopub.execute_input":"2022-03-15T04:57:29.708691Z","iopub.status.idle":"2022-03-15T04:57:29.729676Z","shell.execute_reply.started":"2022-03-15T04:57:29.708652Z","shell.execute_reply":"2022-03-15T04:57:29.728609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! python tocsv.py --input_path  'test_v5neg_2a.txt' --meta_path 'test_meta.csv'","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:29.731101Z","iopub.execute_input":"2022-03-15T04:57:29.731525Z","iopub.status.idle":"2022-03-15T04:57:37.136327Z","shell.execute_reply.started":"2022-03-15T04:57:29.731485Z","shell.execute_reply":"2022-03-15T04:57:37.135258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prob2str(row):\n    return f'negative {row.pred_cls1:.6f} 0 0 1 1 typical {row.pred_cls2:.6f} 0 0 1 1 indeterminate {row.pred_cls3:.6f} 0 0 1 1 atypical {row.pred_cls4:.6f} 0 0 1 1'\n    #return f''","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.137782Z","iopub.execute_input":"2022-03-15T04:57:37.138111Z","iopub.status.idle":"2022-03-15T04:57:37.143565Z","shell.execute_reply.started":"2022-03-15T04:57:37.138073Z","shell.execute_reply":"2022-03-15T04:57:37.142528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def combine_image(row):\n     return f'none {row.pred_cls5:.6f} 0 0 1 1 {row.PredictionString}'\n        #return f''","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.144979Z","iopub.execute_input":"2022-03-15T04:57:37.145589Z","iopub.status.idle":"2022-03-15T04:57:37.152205Z","shell.execute_reply.started":"2022-03-15T04:57:37.145552Z","shell.execute_reply":"2022-03-15T04:57:37.151237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tosub(df1):\n    df1_image = df1[['image_id', 'pred_cls1', 'pred_cls5', 'PredictionString']].copy()\n    df1_image.rename(columns={\"image_id\": \"id\"}, inplace=True)\n    df1_image['id'] = df1_image['id'].apply(lambda x: f'{x}_image')\n    df1_image['PredictionString'] = df1_image.apply(lambda r: combine_image(r), axis=1) \n    df1_image = df1_image[['id', 'PredictionString']]\n    \n    df1_study = df1[['study_id', 'pred_cls1', 'pred_cls2', 'pred_cls3', 'pred_cls4']].copy()\n    df1_study = df1_study.groupby('study_id').agg('mean').reset_index()\n    df1_study.rename(columns={\"study_id\": \"id\"}, inplace=True)\n    df1_study['id'] = df1_study['id'].apply(lambda x: f'{x}_study')\n    df1_study[\"PredictionString\"] = df1_study.apply(lambda r: prob2str(r), axis=1) \n    df1_study = df1_study[['id', 'PredictionString']]\n\n    df1_sub = pd.concat([df1_study, df1_image])\n\n    return df1_sub","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.153982Z","iopub.execute_input":"2022-03-15T04:57:37.154495Z","iopub.status.idle":"2022-03-15T04:57:37.167043Z","shell.execute_reply.started":"2022-03-15T04:57:37.15446Z","shell.execute_reply":"2022-03-15T04:57:37.16612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = pd.read_csv('n_cf11_9.csv') \ndf2 = pd.read_csv('n_cf11.csv')#.head(10)\ndf3 = pd.read_csv('n_cf11_6.csv')\ndf4 = pd.read_csv('n_cf11_7.csv')\ndf5 = pd.read_csv('n_cf11_10.csv')\ndf6 = pd.read_csv('n_cf11_rot1.csv')\ndf7 = pd.read_csv('n_cf11_1.csv')\n# df8 = pd.read_csv('n_cf11_8.csv')\n\ndf2 = df1[[\"image_id\"]].merge(df2, on=[\"image_id\"])\ndf3 = df1[[\"image_id\"]].merge(df3, on=[\"image_id\"])\ndf4 = df1[[\"image_id\"]].merge(df4, on=[\"image_id\"])\ndf5 = df1[[\"image_id\"]].merge(df5, on=[\"image_id\"])\ndf6 = df1[[\"image_id\"]].merge(df6, on=[\"image_id\"])\ndf7 = df1[[\"image_id\"]].merge(df7, on=[\"image_id\"])\n#df8 = df1[[\"image_id\"]].merge(df8, on=[\"image_id\"])","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.168398Z","iopub.execute_input":"2022-03-15T04:57:37.168896Z","iopub.status.idle":"2022-03-15T04:57:37.279896Z","shell.execute_reply.started":"2022-03-15T04:57:37.168858Z","shell.execute_reply":"2022-03-15T04:57:37.279125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.281009Z","iopub.execute_input":"2022-03-15T04:57:37.28148Z","iopub.status.idle":"2022-03-15T04:57:37.301277Z","shell.execute_reply.started":"2022-03-15T04:57:37.281446Z","shell.execute_reply":"2022-03-15T04:57:37.300314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chexnet_predictions = chexnet_pred.set_index('image_id')\nchexnet_predictions =chexnet_predictions.reindex(index = df1['image_id'])\nchexnet_predictions.reset_index()\nchexnet_predictions.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.302627Z","iopub.execute_input":"2022-03-15T04:57:37.303117Z","iopub.status.idle":"2022-03-15T04:57:37.324711Z","shell.execute_reply.started":"2022-03-15T04:57:37.303078Z","shell.execute_reply":"2022-03-15T04:57:37.323812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.326056Z","iopub.execute_input":"2022-03-15T04:57:37.32655Z","iopub.status.idle":"2022-03-15T04:57:37.340249Z","shell.execute_reply.started":"2022-03-15T04:57:37.326513Z","shell.execute_reply":"2022-03-15T04:57:37.339457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sheep_df = pd.read_csv('sheep_df.csv')\nsheep_df = sheep_df.rename(columns={\"image_id\": \"image_id\", \"negative\": \"pred_cls1\", \n                         \"typical\": \"pred_cls2\", \"indeterminate\": \"pred_cls3\",\n                        \"atypical\": \"pred_cls4\"})\n\nsheep_df = df2[[\"image_id\"]].merge(sheep_df, on=[\"image_id\"])\n\nfor col in ['pred_cls1', 'pred_cls2', 'pred_cls3', 'pred_cls4', 'pred_cls5']:\n    df1[col] = (df1[col] + df2[col] + df3[col] + df4[col] + df5[col] + df6[col] + df7[col])/7\n    \nfor col in ['pred_cls1', 'pred_cls2', 'pred_cls3', 'pred_cls4']:\n    df1[col] = (1*df1[col] + sheep_df[col])/2","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.341929Z","iopub.execute_input":"2022-03-15T04:57:37.342529Z","iopub.status.idle":"2022-03-15T04:57:37.391263Z","shell.execute_reply.started":"2022-03-15T04:57:37.342463Z","shell.execute_reply":"2022-03-15T04:57:37.390285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sheep_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.392697Z","iopub.execute_input":"2022-03-15T04:57:37.393269Z","iopub.status.idle":"2022-03-15T04:57:37.415275Z","shell.execute_reply.started":"2022-03-15T04:57:37.393215Z","shell.execute_reply":"2022-03-15T04:57:37.414341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df1.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.416603Z","iopub.execute_input":"2022-03-15T04:57:37.417118Z","iopub.status.idle":"2022-03-15T04:57:37.422048Z","shell.execute_reply.started":"2022-03-15T04:57:37.417079Z","shell.execute_reply":"2022-03-15T04:57:37.420314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sheep_none_df = pd.read_csv('public_test_sheep_predict_none.csv')\nsheep_none_df = df1[[\"image_id\"]].merge(sheep_none_df, on=[\"image_id\"])\n\n#df1['pred_cls5'] = sheep_none_df['Negative for Pneumonia']\ndf1['pred_cls5'] = 2*(1-df1['pred_cls5']) + 1*sheep_df['pred_cls1'] + 1*sheep_none_df['none']","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.425101Z","iopub.execute_input":"2022-03-15T04:57:37.426409Z","iopub.status.idle":"2022-03-15T04:57:37.450397Z","shell.execute_reply.started":"2022-03-15T04:57:37.426305Z","shell.execute_reply":"2022-03-15T04:57:37.449482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_sub = pd.read_csv('v5_50.csv')\nimage_sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.451762Z","iopub.execute_input":"2022-03-15T04:57:37.452289Z","iopub.status.idle":"2022-03-15T04:57:37.567218Z","shell.execute_reply.started":"2022-03-15T04:57:37.452251Z","shell.execute_reply":"2022-03-15T04:57:37.566404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_sub = pd.read_csv('v5_50.csv')\nimage_sub = df1[[\"image_id\"]].merge(image_sub, on=[\"image_id\"])\ndf1['PredictionString'] = image_sub['PredictionString']\ndf_sub  = tosub(df1)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.568665Z","iopub.execute_input":"2022-03-15T04:57:37.569025Z","iopub.status.idle":"2022-03-15T04:57:37.750034Z","shell.execute_reply.started":"2022-03-15T04:57:37.568987Z","shell.execute_reply":"2022-03-15T04:57:37.749165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.751385Z","iopub.execute_input":"2022-03-15T04:57:37.751748Z","iopub.status.idle":"2022-03-15T04:57:37.767673Z","shell.execute_reply.started":"2022-03-15T04:57:37.75171Z","shell.execute_reply":"2022-03-15T04:57:37.766484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from IPython.display import FileLink\n# FileLink(r'./required.zip')","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.769445Z","iopub.execute_input":"2022-03-15T04:57:37.769807Z","iopub.status.idle":"2022-03-15T04:57:37.775131Z","shell.execute_reply.started":"2022-03-15T04:57:37.76977Z","shell.execute_reply":"2022-03-15T04:57:37.774066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_sub = pd.read_csv('v5_50.csv')\nimage_sub = df1[[\"image_id\"]].merge(image_sub, on=[\"image_id\"])\ndf1['PredictionString'] = image_sub['PredictionString']\ndf_sub  = tosub(df1)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.77681Z","iopub.execute_input":"2022-03-15T04:57:37.77722Z","iopub.status.idle":"2022-03-15T04:57:37.968868Z","shell.execute_reply.started":"2022-03-15T04:57:37.777182Z","shell.execute_reply":"2022-03-15T04:57:37.96765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.tail()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.970507Z","iopub.execute_input":"2022-03-15T04:57:37.970967Z","iopub.status.idle":"2022-03-15T04:57:37.983268Z","shell.execute_reply.started":"2022-03-15T04:57:37.970923Z","shell.execute_reply":"2022-03-15T04:57:37.982278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:37.985008Z","iopub.execute_input":"2022-03-15T04:57:37.985659Z","iopub.status.idle":"2022-03-15T04:57:37.992102Z","shell.execute_reply.started":"2022-03-15T04:57:37.985618Z","shell.execute_reply":"2022-03-15T04:57:37.991102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.iloc[:,1]","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:38.001022Z","iopub.execute_input":"2022-03-15T04:57:38.00158Z","iopub.status.idle":"2022-03-15T04:57:38.011385Z","shell.execute_reply.started":"2022-03-15T04:57:38.001538Z","shell.execute_reply":"2022-03-15T04:57:38.0101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r ./*","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:38.013541Z","iopub.execute_input":"2022-03-15T04:57:38.013946Z","iopub.status.idle":"2022-03-15T04:57:38.844633Z","shell.execute_reply.started":"2022-03-15T04:57:38.013906Z","shell.execute_reply":"2022-03-15T04:57:38.843398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T05:01:26.117568Z","iopub.execute_input":"2022-03-15T05:01:26.117958Z","iopub.status.idle":"2022-03-15T05:01:26.132337Z","shell.execute_reply.started":"2022-03-15T05:01:26.117924Z","shell.execute_reply":"2022-03-15T05:01:26.131309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T04:57:38.865495Z","iopub.execute_input":"2022-03-15T04:57:38.866192Z","iopub.status.idle":"2022-03-15T04:57:39.186091Z","shell.execute_reply.started":"2022-03-15T04:57:38.866151Z","shell.execute_reply":"2022-03-15T04:57:39.185195Z"},"trusted":true},"execution_count":null,"outputs":[]}]}