{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Install dependencies","metadata":{}},{"cell_type":"code","source":"%%capture\n!conda install '/kaggle/input/pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-03T14:21:31.495837Z","iopub.execute_input":"2021-07-03T14:21:31.496429Z","iopub.status.idle":"2021-07-03T14:22:49.907547Z","shell.execute_reply.started":"2021-07-03T14:21:31.496307Z","shell.execute_reply":"2021-07-03T14:22:49.906295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install ../input/icevision08-kagcuda/loguru-0.5.3-py3-none-any.whl -f ./ --no-index --no-deps\n!pip install ../input/icevision08-kagcuda/timm-0.4.9-py3-none-any.whl -f ./ --no-index --no-deps\n!pip install ../input/icevision08-kagcuda/effdet-0.2.4-py3-none-any.whl -f ./ --no-index --no-deps\n!pip install ../input/icevision08-kagcuda/icevision-0.8.0-py3-none-any.whl -f ./ --no-index --no-deps\n!pip install ../input/icevision08-kagcuda/omegaconf-2.0.6-py3-none-any.whl -f ./ --no-index --no-deps","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:22:49.909140Z","iopub.execute_input":"2021-07-03T14:22:49.909417Z","iopub.status.idle":"2021-07-03T14:23:00.302617Z","shell.execute_reply.started":"2021-07-03T14:22:49.909389Z","shell.execute_reply":"2021-07-03T14:23:00.301335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n%mkdir -p /root/.icevision/fonts\n%cp /kaggle/input/icevision08-kagcuda/SpaceGrotesk-Medium.ttf /root/.icevision/fonts/","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:23:00.305081Z","iopub.execute_input":"2021-07-03T14:23:00.305390Z","iopub.status.idle":"2021-07-03T14:23:01.777598Z","shell.execute_reply.started":"2021-07-03T14:23:00.305360Z","shell.execute_reply":"2021-07-03T14:23:01.776306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install ../input/pycocotools202/pycocotools-2.0.2-cp37-cp37m-linux_x86_64.whl -f ./ --no-index --no-deps","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:23:01.779996Z","iopub.execute_input":"2021-07-03T14:23:01.780450Z","iopub.status.idle":"2021-07-03T14:23:03.644118Z","shell.execute_reply.started":"2021-07-03T14:23:01.780398Z","shell.execute_reply":"2021-07-03T14:23:03.642904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading dependencies","metadata":{}},{"cell_type":"code","source":"from fastai.vision.all import *\nfrom fastai.medical.imaging import *\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport albumentations as A\nimport cv2\nimport gdcm\nfrom tqdm.auto import tqdm\nfrom joblib import Parallel, delayed\nimport PIL","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:23:03.645774Z","iopub.execute_input":"2021-07-03T14:23:03.646111Z","iopub.status.idle":"2021-07-03T14:23:08.006648Z","shell.execute_reply.started":"2021-07-03T14:23:03.646077Z","shell.execute_reply":"2021-07-03T14:23:08.005420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Converting DICOM files to JPG","metadata":{}},{"cell_type":"code","source":"def Dicom2array(path):\n    im = pydicom.dcmread(path)\n    is_mono1 = (im.PhotometricInterpretation == \"MONOCHROME1\")\n    im = np.array(apply_voi_lut(im.pixel_array, im))\n    if is_mono1:\n        im = np.amax(im) - im\n    im = im - np.min(im)\n    im = im / np.max(im)\n    im = (im * 255).astype(np.uint8)\n    return im\n\ndef Dicom2jpg(file):\n    img = Dicom2array(file)        \n    out_path = Path('/kaggle/working')/Path(*file.parts[3:]).with_suffix('.jpg')\n    os.makedirs(out_path.parent, exist_ok=True)\n    cv2.imwrite(str(out_path), img)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:23:08.008113Z","iopub.execute_input":"2021-07-03T14:23:08.008412Z","iopub.status.idle":"2021-07-03T14:23:08.016118Z","shell.execute_reply.started":"2021-07-03T14:23:08.008385Z","shell.execute_reply":"2021-07-03T14:23:08.014976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom_file = get_dicom_files('../input/siim-covid19-detection/test')\no = Parallel(n_jobs=2)(delayed(Dicom2jpg)(f) for f in tqdm(dicom_file))","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:23:08.017673Z","iopub.execute_input":"2021-07-03T14:23:08.017992Z","iopub.status.idle":"2021-07-03T14:28:09.316124Z","shell.execute_reply.started":"2021-07-03T14:23:08.017962Z","shell.execute_reply":"2021-07-03T14:28:09.315138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading model","metadata":{}},{"cell_type":"code","source":"%mkdir -p /root/.cache/torch/hub/checkpoints\n!cp ../input/tf-efficientnet-b3-weights/tf_efficientnet_b3_aa-84b4657e.pth /root/.cache/torch/hub/checkpoints/tf_efficientnet_b3_aa-84b4657e.pth","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:28:09.319158Z","iopub.execute_input":"2021-07-03T14:28:09.319649Z","iopub.status.idle":"2021-07-03T14:28:11.515718Z","shell.execute_reply.started":"2021-07-03T14:28:09.319600Z","shell.execute_reply":"2021-07-03T14:28:11.514522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from icevision import models, show_preds, tfms, Dataset\nmax_size = 512\nextra_args = {\n    'img_size': max_size\n}\nweights = '../input/siimcovid19-models/efficientdet-tf_d3_BL2.pth'\narch = 'tf_efficientdet_d3'\nnum_classes = 2\nmodel_type = models.ross.efficientdet\nbackbone = model_type.backbones.tf_d3(pretrained=False)\nmodel = model_type.model(backbone=backbone, num_classes=num_classes, **extra_args) \nmodel.load_state_dict(torch.load(weights, map_location=torch.device('cpu')))","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:28:11.517646Z","iopub.execute_input":"2021-07-03T14:28:11.518190Z","iopub.status.idle":"2021-07-03T14:28:15.979775Z","shell.execute_reply.started":"2021-07-03T14:28:11.518153Z","shell.execute_reply":"2021-07-03T14:28:15.978652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !rm /root/.cache/torch/hub/checkpoints/tf_efficientnet_b3_aa-84b4657e.pth","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:28:15.981123Z","iopub.execute_input":"2021-07-03T14:28:15.981404Z","iopub.status.idle":"2021-07-03T14:28:15.989049Z","shell.execute_reply.started":"2021-07-03T14:28:15.981375Z","shell.execute_reply":"2021-07-03T14:28:15.986055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predicting","metadata":{}},{"cell_type":"code","source":"%run ../input/icevision-functions/icevision_functions.py","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:28:15.990352Z","iopub.execute_input":"2021-07-03T14:28:15.990862Z","iopub.status.idle":"2021-07-03T14:28:16.148459Z","shell.execute_reply.started":"2021-07-03T14:28:15.990829Z","shell.execute_reply":"2021-07-03T14:28:16.147296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_files = get_image_files('/kaggle/working')\ninfer_tfms = tfms.A.Adapter([A.Resize(height=max_size, width=max_size, p=1), tfms.A.Normalize()])","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:28:16.149908Z","iopub.execute_input":"2021-07-03T14:28:16.150234Z","iopub.status.idle":"2021-07-03T14:28:16.259306Z","shell.execute_reply.started":"2021-07-03T14:28:16.150203Z","shell.execute_reply":"2021-07-03T14:28:16.258286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nfor file in tqdm(image_files):\n    img = PIL.Image.open(file).convert('RGB')\n    pred = _end2end_detect(img, infer_tfms, model, ClassMap(['opacity']), 0.35, model_type.predict)\n    for d in pred:\n        d['id'] = file.stem\n    preds += pred","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:28:16.260588Z","iopub.execute_input":"2021-07-03T14:28:16.260900Z","iopub.status.idle":"2021-07-03T14:45:20.088984Z","shell.execute_reply.started":"2021-07-03T14:28:16.260852Z","shell.execute_reply":"2021-07-03T14:45:20.087825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_df = pd.DataFrame(preds)\npreds_df['PredictionString'] = preds_df.apply(lambda x: f\"{x['class']} {x['score']:.3f} {' '.join([str(b) for b in x['bbox']])}\", axis = 1)\npreds_df = preds_df[['id', 'PredictionString']].groupby('id').agg(lambda x: ' '.join(x)).reset_index()\npreds_df['id'] = preds_df['id'].apply(lambda x: x + '_image')\npreds_df","metadata":{"execution":{"iopub.status.busy":"2021-07-03T14:49:42.842730Z","iopub.execute_input":"2021-07-03T14:49:42.843124Z","iopub.status.idle":"2021-07-03T14:49:42.904769Z","shell.execute_reply.started":"2021-07-03T14:49:42.843088Z","shell.execute_reply":"2021-07-03T14:49:42.903562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Building the submission files","metadata":{}},{"cell_type":"code","source":"submission_image = pd.DataFrame({\n    'id': [f.stem + '_image' for f in image_files if (f.stem + '_image') not in preds_df['id'].values],\n    'PredictionString': 'none 1 0 0 1 1'\n})\nsubmission_image = pd.concat([preds_df, submission_image]).sort_values('id').reset_index(drop=True)\nsubmission_image","metadata":{"execution":{"iopub.status.busy":"2021-07-03T15:06:54.513006Z","iopub.execute_input":"2021-07-03T15:06:54.513667Z","iopub.status.idle":"2021-07-03T15:06:54.602254Z","shell.execute_reply.started":"2021-07-03T15:06:54.513629Z","shell.execute_reply":"2021-07-03T15:06:54.599978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study = list(set([f.parent.parent.name + '_study' for f in image_files]))\nprediction_string = ['negative 1 0 0 1 1'] * len(study)\nsubmission_study = pd.DataFrame({\n    'id': study,\n    'PredictionString': prediction_string\n}).sort_values('id').reset_index(drop=True)\nsubmission_study","metadata":{"execution":{"iopub.status.busy":"2021-07-03T15:06:58.342512Z","iopub.execute_input":"2021-07-03T15:06:58.342951Z","iopub.status.idle":"2021-07-03T15:06:58.368284Z","shell.execute_reply.started":"2021-07-03T15:06:58.342914Z","shell.execute_reply":"2021-07-03T15:06:58.367205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([submission_study, submission_image])\nsubmission.to_csv('submission.csv',index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2021-07-03T15:07:00.538601Z","iopub.execute_input":"2021-07-03T15:07:00.538988Z","iopub.status.idle":"2021-07-03T15:07:00.563740Z","shell.execute_reply.started":"2021-07-03T15:07:00.538957Z","shell.execute_reply":"2021-07-03T15:07:00.562578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.rmtree('./test')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}