{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom fastai.imports import *\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nfrom matplotlib.patches import Rectangle\nimport os\nimport seaborn as sns\nimport pydicom\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%matplotlib inline\n%reload_ext autoreload\n%autoreload 2","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"import pandas as pd\nstage_2_detailed_class_info = pd.read_csv(\"../input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv\")\nstage_2_train_labels = pd.read_csv(\"../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stage_2_detailed_class_info.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_class_df = stage_2_train_labels.merge(stage_2_detailed_class_info, left_on='patientId', right_on='patientId', how='inner')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.backends.cudnn.enabled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Looking at the CSV\nstage_2_train_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setting the path\npath = \"/kaggle/input/rsna-pneumonia-detection-challenge/\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA Basic\nAny area in the chest radiograph that is more white than it should be is referred to as 'Opacity'\n\nUsually the lungs are full of air. When someone has pneumonia, the air in the lungs is replaced by other material - fluids, bacteria, immune system cells, etc. That's why areas of opacities are areas that are grey but should be more black. When we see them we understand that the lung tissue in that area is probably not healthy."},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(1,1, figsize=(6,4))\ntotal = float(len(stage_2_detailed_class_info))\nsns.countplot(stage_2_detailed_class_info['class'],order = stage_2_detailed_class_info['class'].value_counts().index, palette='Set3').set_title('Class Distribution')\nfor p in ax.patches:\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height + 3,\n            '{:1.2f}%'.format(100*height/total),\n            ha=\"center\") \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_dicom_images_with_boxes(data):\n    img_data = list(data.T.to_dict().values())\n    f, ax = plt.subplots(3,3, figsize=(16,18))\n    for i,data_row in enumerate(img_data):\n        patientImage = data_row['patientId']+'.dcm'\n        imagePath = os.path.join(path,\"stage_2_train_images/\",patientImage)\n        data_row_img_data = pydicom.read_file(imagePath)\n        modality = data_row_img_data.Modality\n        age = data_row_img_data.PatientAge\n        sex = data_row_img_data.PatientSex\n        data_row_img = pydicom.dcmread(imagePath)\n        ax[i//3, i%3].imshow(data_row_img.pixel_array, cmap=plt.cm.bone) \n        ax[i//3, i%3].axis('off')\n        ax[i//3, i%3].set_title('ID: {}\\nModality: {} Age: {} Sex: {} Target: {}\\nClass: {}'.format(\n                data_row['patientId'],modality, age, sex, data_row['Target'], data_row['class']))\n        rows = train_class_df[train_class_df['patientId']==data_row['patientId']]\n        box_data = list(rows.T.to_dict().values())\n        for j, row in enumerate(box_data):\n            ax[i//3, i%3].add_patch(Rectangle(xy=(row['x'], row['y']),\n                        width=row['width'],height=row['height'], \n                        color=\"yellow\",alpha = 0.1))   \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_dicom_images_with_boxes(train_class_df[train_class_df['Target']==1].sample(9))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are different kinds of opacities . Some are related to pneumonia and some are not. It is marked by the bounding boxes.\n\nPneumonia is a lung infection that can be caused by bacteria, viruses, or fungi. Because of the infection and the body's immune response, the sacks in the lungs (termed alveoli) are filled with fluids instead of air.\n\nThe reason that pneumonia associated lung opacities look diffuse on the chest radiograph is because the infection and fluid that accumulate spread within the normal tree of airways in the lung. There is no clear border where the infection stops. That is different from other diseases like tumors, which are totally different from the normal lung, and do not maintain the normal structure of the airways inside the lung."},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision import *\nfrom fastai.basics import *\nfrom fastai.metrics import error_rate\nimport pydicom\nimport imageio\nimport PIL\nimport json, pdb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def open_dcm_image(fn:PathOrStr,convert_mode:str='RGB',after_open:Callable=None)->Image:\n    \"Return `Image` object created from image in file `fn`.\"\n    array = pydicom.dcmread(fn).pixel_array\n    x = PIL.Image.fromarray(array).convert('RGB')\n    return Image(pil2tensor(x,np.float32).div_(255))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Updating the default method to the custom method\nvision.data.open_image = open_dcm_image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setting Transforms\ntfms = get_transforms(flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0)\n# Creating Image DataBunch\ndata = ImageDataBunch.from_csv(path,folder='stage_2_train_images',csv_labels='stage_2_train_labels.csv',ds_tfms=tfms,fn_col='patientId',label_col='Target',suffix='.dcm',seed=47,size=224)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(rows=3, figsize=(9,9))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = cnn_learner(data, models.resnet34, metrics=[accuracy,error_rate])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.model_dir = '/kaggle/working'\nlearn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.load('stage-1-34')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.unfreeze()\ndefaults.device = torch.device('cuda')\nlearn.fit_one_cycle(4, max_lr=slice(1e-6,1e-2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save('stage-2-34')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\n\nlosses,idxs = interp.top_losses()\n\nlen(data.valid_ds)==len(losses)==len(idxs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"interp.plot_top_losses(9, figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"interp.plot_confusion_matrix(figsize=(12,12), dpi=60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn2 = cnn_learner(data, models.resnet50, metrics=[accuracy,error_rate])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn2.model_dir = '/kaggle/working'\nlearn2.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn2.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn2.unfreeze()\n# It's mostly overfitting after 4 epochs\nlearn2.fit_one_cycle(6, max_lr=slice(1e-4,4*1e-2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn2.save('stage1-50')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn2)\n\nlosses,idxs = interp.top_losses()\n\nlen(data.valid_ds)==len(losses)==len(idxs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"interp.plot_top_losses(9, figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"interp.plot_confusion_matrix(figsize=(12,12), dpi=60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}