{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!cp ../input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-04T17:08:47.634984Z","iopub.execute_input":"2021-06-04T17:08:47.635339Z","iopub.status.idle":"2021-06-04T17:09:15.308026Z","shell.execute_reply.started":"2021-06-04T17:08:47.635241Z","shell.execute_reply":"2021-06-04T17:09:15.306858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm gdcm.tar\n!rm -rf gdcm","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:15.309761Z","iopub.execute_input":"2021-06-04T17:09:15.310144Z","iopub.status.idle":"2021-06-04T17:09:16.694265Z","shell.execute_reply.started":"2021-06-04T17:09:15.310101Z","shell.execute_reply":"2021-06-04T17:09:16.693257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import fastai","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:16.697937Z","iopub.execute_input":"2021-06-04T17:09:16.698213Z","iopub.status.idle":"2021-06-04T17:09:16.708462Z","shell.execute_reply.started":"2021-06-04T17:09:16.698184Z","shell.execute_reply":"2021-06-04T17:09:16.707549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision.all import *\nfrom fastai.basics import *\nfrom fastai.vision.utils import *","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:16.711805Z","iopub.execute_input":"2021-06-04T17:09:16.712076Z","iopub.status.idle":"2021-06-04T17:09:19.248210Z","shell.execute_reply.started":"2021-06-04T17:09:16.712050Z","shell.execute_reply":"2021-06-04T17:09:19.247364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = Path(\"/kaggle/input/siim-covid19-detection/test/\")","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:19.249456Z","iopub.execute_input":"2021-06-04T17:09:19.249817Z","iopub.status.idle":"2021-06-04T17:09:19.257202Z","shell.execute_reply.started":"2021-06-04T17:09:19.249779Z","shell.execute_reply":"2021-06-04T17:09:19.256340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files = get_files(test_path, extensions=\".dcm\")","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:19.260377Z","iopub.execute_input":"2021-06-04T17:09:19.260646Z","iopub.status.idle":"2021-06-04T17:09:24.926997Z","shell.execute_reply.started":"2021-06-04T17:09:19.260620Z","shell.execute_reply":"2021-06-04T17:09:24.926134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(test_files)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:24.928233Z","iopub.execute_input":"2021-06-04T17:09:24.928607Z","iopub.status.idle":"2021-06-04T17:09:24.936784Z","shell.execute_reply.started":"2021-06-04T17:09:24.928571Z","shell.execute_reply":"2021-06-04T17:09:24.935781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_ids = []\nstudy_ids = []\nfor file in test_files:\n    img_ids.append(file.stem)\n    study_ids.append(file.parent.parent.name)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:24.939774Z","iopub.execute_input":"2021-06-04T17:09:24.940127Z","iopub.status.idle":"2021-06-04T17:09:24.956357Z","shell.execute_reply.started":"2021-06-04T17:09:24.940089Z","shell.execute_reply":"2021-06-04T17:09:24.955510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img2study = {}\nstudy2imgs = defaultdict(list)\nfor (img, study) in zip(img_ids, study_ids):\n    img2study[img] = study\n    study2imgs[study].append(img)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:24.959919Z","iopub.execute_input":"2021-06-04T17:09:24.960280Z","iopub.status.idle":"2021-06-04T17:09:24.969018Z","shell.execute_reply.started":"2021-06-04T17:09:24.960217Z","shell.execute_reply":"2021-06-04T17:09:24.968168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(study2imgs)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:24.970545Z","iopub.execute_input":"2021-06-04T17:09:24.970980Z","iopub.status.idle":"2021-06-04T17:09:24.982640Z","shell.execute_reply.started":"2021-06-04T17:09:24.970932Z","shell.execute_reply":"2021-06-04T17:09:24.981651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_png = Path(\"./test_converted_png\")\npath_png.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:24.984273Z","iopub.execute_input":"2021-06-04T17:09:24.984664Z","iopub.status.idle":"2021-06-04T17:09:24.992142Z","shell.execute_reply.started":"2021-06-04T17:09:24.984623Z","shell.execute_reply":"2021-06-04T17:09:24.991407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef convert_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    img = PILImage.create(data)\n    new_size = resize_to(img, targ_sz=800, use_min=True)\n    img = img.resize(new_size)\n    img.save(path_png/f\"{path.stem}.png\")\n","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:24.993523Z","iopub.execute_input":"2021-06-04T17:09:24.993973Z","iopub.status.idle":"2021-06-04T17:09:25.252690Z","shell.execute_reply.started":"2021-06-04T17:09:24.993928Z","shell.execute_reply":"2021-06-04T17:09:25.251828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.time()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:09:25.254050Z","iopub.execute_input":"2021-06-04T17:09:25.254427Z","iopub.status.idle":"2021-06-04T17:09:25.259105Z","shell.execute_reply.started":"2021-06-04T17:09:25.254387Z","shell.execute_reply":"2021-06-04T17:09:25.257965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with ThreadPoolExecutor(defaults.cpus) as e:\n    list(e.map(convert_xray, test_files))","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:16:44.562230Z","iopub.execute_input":"2021-06-04T17:16:44.562588Z","iopub.status.idle":"2021-06-04T17:24:22.497476Z","shell.execute_reply.started":"2021-06-04T17:16:44.562557Z","shell.execute_reply":"2021-06-04T17:24:22.496170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end=time.time()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:24:22.499525Z","iopub.execute_input":"2021-06-04T17:24:22.499847Z","iopub.status.idle":"2021-06-04T17:24:22.503944Z","shell.execute_reply.started":"2021-06-04T17:24:22.499806Z","shell.execute_reply":"2021-06-04T17:24:22.503166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(end - start)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:24:22.505725Z","iopub.execute_input":"2021-06-04T17:24:22.506211Z","iopub.status.idle":"2021-06-04T17:24:22.528911Z","shell.execute_reply.started":"2021-06-04T17:24:22.506173Z","shell.execute_reply":"2021-06-04T17:24:22.527829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = load_learner(\"../input/cov19models/softmax-100-epoch-xres50\", cpu=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:24:22.530572Z","iopub.execute_input":"2021-06-04T17:24:22.531208Z","iopub.status.idle":"2021-06-04T17:24:23.381016Z","shell.execute_reply.started":"2021-06-04T17:24:22.531167Z","shell.execute_reply":"2021-06-04T17:24:23.380186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"png_files = get_image_files(path_png)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:24:23.382458Z","iopub.execute_input":"2021-06-04T17:24:23.382792Z","iopub.status.idle":"2021-06-04T17:24:23.398921Z","shell.execute_reply.started":"2021-06-04T17:24:23.382758Z","shell.execute_reply":"2021-06-04T17:24:23.398221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dl = learn.dls.test_dl(png_files)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:24:23.401616Z","iopub.execute_input":"2021-06-04T17:24:23.401857Z","iopub.status.idle":"2021-06-04T17:24:23.409158Z","shell.execute_reply.started":"2021-06-04T17:24:23.401833Z","shell.execute_reply":"2021-06-04T17:24:23.408324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = learn.get_preds(dl=test_dl)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:24:23.410406Z","iopub.execute_input":"2021-06-04T17:24:23.410746Z","iopub.status.idle":"2021-06-04T17:24:54.887650Z","shell.execute_reply.started":"2021-06-04T17:24:23.410711Z","shell.execute_reply":"2021-06-04T17:24:54.886820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = preds[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:24:54.889761Z","iopub.execute_input":"2021-06-04T17:24:54.890028Z","iopub.status.idle":"2021-06-04T17:24:54.896851Z","shell.execute_reply.started":"2021-06-04T17:24:54.890000Z","shell.execute_reply":"2021-06-04T17:24:54.896040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ids = [x.stem for x in test_dl.dataset.items]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:24:54.898343Z","iopub.execute_input":"2021-06-04T17:24:54.898706Z","iopub.status.idle":"2021-06-04T17:24:54.907100Z","shell.execute_reply.started":"2021-06-04T17:24:54.898668Z","shell.execute_reply":"2021-06-04T17:24:54.906159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img2pred = {_id:p for _id,p in zip(test_ids, preds)}","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:24:54.909828Z","iopub.execute_input":"2021-06-04T17:24:54.910105Z","iopub.status.idle":"2021-06-04T17:24:54.921538Z","shell.execute_reply.started":"2021-06-04T17:24:54.910079Z","shell.execute_reply":"2021-06-04T17:24:54.920692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study2preds = defaultdict(list)\nfor study_id, img_ids in study2imgs.items():\n    for img_id in img_ids:\n        study2preds[study_id].append(img2pred[img_id])","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:50:18.340455Z","iopub.execute_input":"2021-06-04T17:50:18.340774Z","iopub.status.idle":"2021-06-04T17:50:18.348843Z","shell.execute_reply.started":"2021-06-04T17:50:18.340745Z","shell.execute_reply":"2021-06-04T17:50:18.347651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_round(x):\n    if x < 0.0001:\n        return 0\n    else:\n        return round(x, 5)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:50:18.691544Z","iopub.execute_input":"2021-06-04T17:50:18.691899Z","iopub.status.idle":"2021-06-04T17:50:18.696182Z","shell.execute_reply.started":"2021-06-04T17:50:18.691866Z","shell.execute_reply":"2021-06-04T17:50:18.695239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study2preds = {k:list(map(my_round, torch.stack(v, dim=0).mean(dim=0).tolist())) for k,v in study2preds.items()}","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:50:19.046755Z","iopub.execute_input":"2021-06-04T17:50:19.047089Z","iopub.status.idle":"2021-06-04T17:50:19.086064Z","shell.execute_reply.started":"2021-06-04T17:50:19.047058Z","shell.execute_reply":"2021-06-04T17:50:19.085297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_ids_col = []\nstudy_preds_col = []\nfor study, study_preds in study2preds.items():\n    study_ids_col.append(study + \"_study\")\n    pred_string = \" \".join([f\"{learn.dls.vocab[i]} {study_preds[i]} 0 0 1 1\" for i in range(len(learn.dls.vocab))])\n    study_preds_col.append(pred_string)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:53:18.065048Z","iopub.execute_input":"2021-06-04T17:53:18.065384Z","iopub.status.idle":"2021-06-04T17:53:18.460008Z","shell.execute_reply.started":"2021-06-04T17:53:18.065347Z","shell.execute_reply":"2021-06-04T17:53:18.459217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(study_ids_col)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:53:22.517637Z","iopub.execute_input":"2021-06-04T17:53:22.517952Z","iopub.status.idle":"2021-06-04T17:53:22.522574Z","shell.execute_reply.started":"2021-06-04T17:53:22.517926Z","shell.execute_reply":"2021-06-04T17:53:22.521784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = [x.stem + \"_image\" for x in get_files( Path(\"../input/siim-covid19-detection/test\"), extensions=\".dcm\")]\nimages_preds = [\"none 1 0 0 1 1\"] * len(images)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:53:25.145575Z","iopub.execute_input":"2021-06-04T17:53:25.145896Z","iopub.status.idle":"2021-06-04T17:53:26.831428Z","shell.execute_reply.started":"2021-06-04T17:53:25.145866Z","shell.execute_reply":"2021-06-04T17:53:26.830577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\"id\": study_ids_col + images, \"PredictionString\": study_preds_col + images_preds})","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:53:28.084869Z","iopub.execute_input":"2021-06-04T17:53:28.085225Z","iopub.status.idle":"2021-06-04T17:53:28.100639Z","shell.execute_reply.started":"2021-06-04T17:53:28.085193Z","shell.execute_reply":"2021-06-04T17:53:28.099615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2021-06-04T17:53:29.650635Z","iopub.execute_input":"2021-06-04T17:53:29.650953Z","iopub.status.idle":"2021-06-04T17:53:29.677722Z","shell.execute_reply.started":"2021-06-04T17:53:29.650923Z","shell.execute_reply":"2021-06-04T17:53:29.676586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf ./test_converted_png","metadata":{},"execution_count":null,"outputs":[]}]}