{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#!conda install '/kaggle/input/libs/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n#!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n#!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n#!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n#!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n#!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-23T12:44:18.853964Z","iopub.execute_input":"2021-06-23T12:44:18.854360Z","iopub.status.idle":"2021-06-23T12:44:18.859055Z","shell.execute_reply.started":"2021-06-23T12:44:18.854325Z","shell.execute_reply":"2021-06-23T12:44:18.857896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport pathlib\nimport itertools\nimport pydicom\nimport os\nimport cv2\nimport IPython\n#import gdcm","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:18.861500Z","iopub.execute_input":"2021-06-23T12:44:18.861979Z","iopub.status.idle":"2021-06-23T12:44:18.883859Z","shell.execute_reply.started":"2021-06-23T12:44:18.861932Z","shell.execute_reply":"2021-06-23T12:44:18.882283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pydicom.datadict import DicomDictionary, keyword_dict\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom pydicom.dataset import Dataset\nfrom tqdm.notebook import tqdm\nfrom IPython.display import Image\n\n#from fmi.fmi.explore import *\n#from fmi.fmi.preprocessing import *\n#from fmi.fmi.pipeline import *\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\n\nfrom torchvision.utils import save_image\nfrom skimage import exposure","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:18.886115Z","iopub.execute_input":"2021-06-23T12:44:18.886801Z","iopub.status.idle":"2021-06-23T12:44:18.899613Z","shell.execute_reply.started":"2021-06-23T12:44:18.886758Z","shell.execute_reply":"2021-06-23T12:44:18.898362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#system_info()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:18.901984Z","iopub.execute_input":"2021-06-23T12:44:18.902480Z","iopub.status.idle":"2021-06-23T12:44:18.912214Z","shell.execute_reply.started":"2021-06-23T12:44:18.902434Z","shell.execute_reply":"2021-06-23T12:44:18.911239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Constants & Paths & Configs","metadata":{}},{"cell_type":"code","source":"SIIM_COVID19_DETECTION_DIR = '/kaggle/input/siim-covid19-detection/'\nPART0_RESIZED_DIR = '../input/siim-covid19-resized-to-512px-jpg'\nMETA_DIR = SIIM_COVID19_DETECTION_DIR + '**/*.dcm'\nTEMP_DIR = '/kaggle/temp/'\n\nINPUT_DIR = PART0_RESIZED_DIR+'/train/'\nOUTPUT_DIR = DATASET_DIR = TEMP_DIR+'/train/'\nTRAIN_DIR = DATASET_DIR + 'train/'\nTA_DIR = TRAIN_DIR+'ta/'\nIA_DIR = TRAIN_DIR+'ia/'\nAA_DIR = TRAIN_DIR+'aa/'\nNP_DIR = TRAIN_DIR+'np/'\n\nWORKING_DIR = '/kaggle/working/'\n\nWANDB_PROJECT_NAME = 'project8-kaggle-covid19'\nWANDB_ENTITY_NAME = ''\n\nTRAIN_IMAGE_LEVEL_PATH = SIIM_COVID19_DETECTION_DIR+'train_image_level.csv'\nTRAIN_STUDY_LEVEL_PATH = SIIM_COVID19_DETECTION_DIR+'train_study_level.csv'\nMETA_PATH = PART0_RESIZED_DIR +'meta.csv'\n\n\nBATCH_SIZE = 32\nEPOCHS = 25\nIMG_SIZE = WIDTH = HEIGHT = 224\nLEARNING_RATE = 0.00008\n\nINTERPOLATION = cv2.INTER_LANCZOS4\n\nprint(os.listdir(SIIM_COVID19_DETECTION_DIR))\n\nroot = pathlib.Path('/kaggle/input/siim-covid19-detection')\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:18.914129Z","iopub.execute_input":"2021-06-23T12:44:18.914776Z","iopub.status.idle":"2021-06-23T12:44:18.931563Z","shell.execute_reply.started":"2021-06-23T12:44:18.914734Z","shell.execute_reply":"2021-06-23T12:44:18.930752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Objectives\n1. Learn how to handle DICOM (Digital Imaging and Communications in Medicine) via pydicom https://dicom.innolitics.com/ciods/segmentation/general-image/00080008\n1. Recycle another network https://keras.io/api/applications/densenet/\n1. Write a basic network yourself","metadata":{}},{"cell_type":"markdown","source":"# Pre-process study, image files","metadata":{}},{"cell_type":"code","source":"df_train_image_level = pd.read_csv(TRAIN_IMAGE_LEVEL_PATH)\ndf_train_study_level = pd.read_csv(TRAIN_STUDY_LEVEL_PATH)\n\ndf_train_image_level['id'] = df_train_image_level.apply(lambda row: row.id.split('_')[0], axis=1)\ndf_train_image_level['path'] = df_train_image_level.apply(lambda row: INPUT_DIR+row.id+'.jpg', axis=1)\ndf_train_image_level['image_level'] = df_train_image_level.apply(lambda row: row.label.split(' ')[0], axis=1)\n\ndf_train_study_level['id'] = df_train_study_level.apply(lambda row: row.id.split('_')[0], axis=1)\ndf_train_study_level.columns = ['StudyInstanceUID', 'Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']\ndf_train_study_level.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:18.933357Z","iopub.execute_input":"2021-06-23T12:44:18.933877Z","iopub.status.idle":"2021-06-23T12:44:19.438659Z","shell.execute_reply.started":"2021-06-23T12:44:18.933843Z","shell.execute_reply":"2021-06-23T12:44:19.437893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_image_level = df_train_image_level.merge(df_train_study_level, on='StudyInstanceUID',how=\"left\")\ndf_train_image_level = df_train_image_level[['id','StudyInstanceUID','path','Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance']]\ndf_train_image_level = df_train_image_level.dropna()\ndf_train_image_level = df_train_image_level[~df_train_image_level.duplicated(subset=['StudyInstanceUID'], keep='first')]\ndf_train_image_level = df_train_image_level.reset_index(drop=True)\ndf_train_image_level.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:19.439867Z","iopub.execute_input":"2021-06-23T12:44:19.440258Z","iopub.status.idle":"2021-06-23T12:44:19.474628Z","shell.execute_reply.started":"2021-06-23T12:44:19.440229Z","shell.execute_reply":"2021-06-23T12:44:19.473869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract metadata from DICOM files\n* https://www.kaggle.com/avirdee/understanding-dicoms","metadata":{}},{"cell_type":"code","source":"\nIPython.display.Image(url='https://asvcode.github.io/MedicalImaging/images/dicom_.PNG')","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:19.475634Z","iopub.execute_input":"2021-06-23T12:44:19.476060Z","iopub.status.idle":"2021-06-23T12:44:19.480724Z","shell.execute_reply.started":"2021-06-23T12:44:19.476031Z","shell.execute_reply":"2021-06-23T12:44:19.479991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# credit @raddar\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\ndef resize_and_save(file_path):\n    split = 'train' if 'train' in file_path else 'test'\n    base_dir = f'/kaggle/working/{split}'\n    img = read_xray(file_path)\n    h, w = img.shape[:2]  # orig hw\n    if aspect_ratio:\n        r = dim / max(h, w)  # resize image to img_size\n        interp = cv2.INTER_AREA if r < 1 else cv2.INTER_LINEAR\n        if r != 1:  # always resize down, only resize up if training with augmentation\n            img = cv2.resize(img, (int(w * r), int(h * r)), interpolation=interp)\n    else:\n        img = cv2.resize(img, (dim, dim), cv2.INTER_AREA)\n    filename = file_path.split('/')[-1].split('.')[0]\n    cv2.imwrite(os.path.join(base_dir, f'{filename}.jpg'), img)\n    return filename.replace('dcm','')+'_image',w, h","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:19.481674Z","iopub.execute_input":"2021-06-23T12:44:19.482053Z","iopub.status.idle":"2021-06-23T12:44:19.499420Z","shell.execute_reply.started":"2021-06-23T12:44:19.482026Z","shell.execute_reply":"2021-06-23T12:44:19.498044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#filepaths = df_train_image_level.filepath.iloc[:100 if debug else test_df.shape[0]]\n#metadata = []\n#for filepath in tqdm(filepaths):\n#    metadata.append(resize_and_save(filepath))","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:47.151238Z","iopub.execute_input":"2021-06-23T12:44:47.151593Z","iopub.status.idle":"2021-06-23T12:44:47.155916Z","shell.execute_reply.started":"2021-06-23T12:44:47.151563Z","shell.execute_reply":"2021-06-23T12:44:47.154636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = []\nfor file in tqdm(root.glob('**/*.dcm')):\n    dcm = pydicom.dcmread(file, stop_before_pixels=True)\n    elements = itertools.chain(dcm.iterall(), dcm.file_meta.iterall())\n    for elem in elements:\n        meta = {\n            'VM': elem.VM,\n            'VR': elem.VR,\n            'tag': elem.tag,\n            'name': elem.name,\n            'keyword': elem.keyword,\n            # SQ values are redundant with other elements\n            'data': elem.value if elem.VR != 'SQ' else None,\n            'value': elem.repval,\n            'filename': file.name,\n        }\n        metadata.append(meta)\nmetadata = pd.DataFrame(metadata)\nmetadata.head(10)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:49.195041Z","iopub.execute_input":"2021-06-23T12:44:49.195442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dcm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:19.693512Z","iopub.status.idle":"2021-06-23T12:44:19.694041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:19.695310Z","iopub.status.idle":"2021-06-23T12:44:19.695798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandas_profiling import ProfileReport\nmetadtaProfile = metadata.profile_report(title = 'Metadata Report', infer_dtypes=True)\n#metadtaProfile.to_file(\"profileMarchant.html\")\nmetadtaProfile.to_notebook_iframe()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:19.697293Z","iopub.status.idle":"2021-06-23T12:44:19.697746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IPython.display.Image(\"https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F1723677%2F3f6e6f2e073f25f6db8f77b5a3409b43%2Fimg5.png?generation=1602506639545491&alt=media\")","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:19.698795Z","iopub.status.idle":"2021-06-23T12:44:19.699220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transfer Learning with Dense Net 169\n","metadata":{}},{"cell_type":"markdown","source":"## References\n* https://towardsdatascience.com/understanding-and-visualizing-densenets-7f688092391a\n* https://www.kaggle.com/pytorch/densenet169\n## Segmentation models\n* https://github.com/qubvel/segmentation_models.pytorch\n* https://www.tensorflow.org/datasets/catalog/cassava","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IPython.display.Image(url='https://imgur.com/wWHWbQt.jpg')","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:19.700565Z","iopub.status.idle":"2021-06-23T12:44:19.701036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IPython.display.Image(url=\"https://imgur.com/oiTdqJL.jpg\")","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:19.702413Z","iopub.status.idle":"2021-06-23T12:44:19.702954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\nUtilities to create submission file\n* https://www.kaggle.com/farhanhaikhan/random-easy-submission-demo/notebook","metadata":{}},{"cell_type":"code","source":"def CreateSub(testid, tst_preds):\n    sub_df = pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')\n    for i in range(len(test)):\n        for j in range(len(test)):\n            a = test.loc[i,'id'].split('.')[0]\n            b = sub_df.loc[j,'id']\n            if a==b:\n                negative, typical, indeterminate, atypical = str(tst_preds[i][0]),str(tst_preds[i][1]),str(tst_preds[i][2]),str(tst_preds[i][3]),\n                sub_df.loc[j,'PredictionString'] = f'negative {negative} 0 0 1 1 typical {typical} 0 0 1 1 indeterminate {indeterminate} 0 0 1 1 atypical {atypical} 0 0 1 1'\n    return sub_df\n\nsumfile = CreateSub(test, tst_preds)\nsumfile.to_csv('./submission.csv',index=False)\n\n!rm -r ./test/study/","metadata":{"execution":{"iopub.status.busy":"2021-06-23T12:44:19.704043Z","iopub.status.idle":"2021-06-23T12:44:19.704472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References\n* https://www.kaggle.com/freaxmind/extract-metadata-from-dicom-files-efficiently\n* https://github.com/pydicom/pydicom/blob/master/examples/input_output/plot_read_fileset.py","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}