{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10338,"databundleVersionId":862042,"sourceType":"competition"}],"dockerImageVersionId":9432,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA Pneumonia Detection Challenge with ChexNet\n![](https://i.pinimg.com/564x/ff/cd/c4/ffcdc4d74eed036d029a84c381604a10.jpg)\n\n# Why to detetct this\n- Pneumonia accounts for over **15% of all deaths of children under 5 years old** internationally. In 2015, **920,000 children under the age of 5 died from the disease. **\n- In the United States, **pneumonia accounts for over 500,000** visits to emergency departments [[1]](http://www.cdc.gov/nchs/data/nhamcs/web_tables/2015_ed_web_tables.pdf) and over **50,000 deaths in 2015** [[2]](http://www.cdc.gov/nchs/data/nvsr/nvsr66/nvsr66_06_tables.pdf), keeping the ailment on the list of top **10** causes of death in the country.\n\n# Symptoms to detect Pneumonia\n- The Diagnosis of pneumonia on CXR( is complicated because of a number of other **nditions in the lungs**uch as **uid overload (pulmonary edema), bleeding, volume loss (atelectasis or collapse), lung cancer, or post-radiation or surgical changes.**\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
CharacteristicsPneumonia
HistoryUnderlying lung disease, contact with individuals having upper or lower respiratory infection, contact with birds/animals.
CausesBacteria, virus, fungi; aspiration.
Body systems Respiratory system—lungs.
Clinical symptomsHigh fever (sometimes with chills and rigors), cough, wheezing, breathing difficulty, chest pain.
InvestigationsBlood investigations—complete blood count, ESR, sputum examination and culture, chest X-ray, CT scan, bronchoscopy, thoracocentesis, pleural fluid aspiration and culture.
TreatmentsAppropriate antimicrobial therapy, expectorant, antipyretics and analgesics, oxygen therapy (if required), fluids.
\n\n[Table References](http://www.differencebetween.net/science/health/difference-between-pneumonia-and-typhoid/)\n\n# Goal of This competition\n- In this competition, the primary endpoint will be the **detection of bounding boxes** corresponding to the **diagnosis of pneumonia** (e.g. **lung infection**) on chest radiographs, a **special 2D high resolution grayscale medical image**. \n- **Note** that pnuemonia is just one of many possible disease processes that can **occur on a chest radiograph**, and that any given **single image** may contain **0, 1** or **many boxes corresponding** to possible pneumonia locations.\n\nThanks to this kernel\n- https://www.kaggle.com/robikscube/eda-lets-detect-pneumonia-explore-lung-images\n\n
\n
\n

Data Summary

Stage 1 Images - stage_1_train_images.zip and stage_1_test_images.zip

\n

Stage 1 Labels - stage_1_train_labels.csv and Stage 1 Sample Submission stage_1_sample_submission.csv

\n

Stage 1 Detailed Info - stage_1_detailed_class_info.csv

\n\n
\n
\n\n

File descriptions

\n\n

Data fields

\n","metadata":{"_uuid":"14fc1ef31fe24097d8844ecade8d726a961d77fd"}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Required packages\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pylab as plt\nimport pydicom\nimport os\nfrom os import listdir\nfrom os.path import isfile, join\n\n# Any results you write to the current directory are saved as output.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-09T12:35:42.64888Z","iopub.execute_input":"2024-05-09T12:35:42.6494Z","iopub.status.idle":"2024-05-09T12:35:42.819034Z","shell.execute_reply.started":"2024-05-09T12:35:42.649282Z","shell.execute_reply":"2024-05-09T12:35:42.818235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Images Example\ntrain_images_dir = '../input/stage_1_train_images/'\ntrain_images = [f for f in listdir(train_images_dir) if isfile(join(train_images_dir, f))]\ntest_images_dir = '../input/stage_1_test_images/'\ntest_images = [f for f in listdir(test_images_dir) if isfile(join(test_images_dir, f))]\nprint('5 Training images', train_images[:5]) # Print the first 5","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-09T12:35:42.82329Z","iopub.execute_input":"2024-05-09T12:35:42.823673Z","iopub.status.idle":"2024-05-09T12:35:43.224065Z","shell.execute_reply.started":"2024-05-09T12:35:42.823608Z","shell.execute_reply":"2024-05-09T12:35:43.16277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of train images:', len(train_images))\nprint('Number of test images:', len(test_images))","metadata":{"_uuid":"896a2bf4b39e3a57a0e190282f00c2e128aa00d9","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-09T12:35:43.164164Z","iopub.status.idle":"2024-05-09T12:35:43.165912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ChexNet\n[Stanford ChexNet](https://arxiv.org/abs/1711.05225)\n\n### Radiology \n* The ***ChexNet*** paper reviews performance of AI versus 4 trained radiologists in diagnosing pneumonia. \n* **Pneumonia is a clinical diagnosis** — a **patient** will present with **fever and cough** , and can get a **chest Xray(CXR) to identify complications of pneumonia.** Patients will usually get **blood cultures** to **supplement diagnosis. Pneumonia on a CXR** is not easily distinguishable from other findings that fill the **alevolar spaces  —  specifically pus , blood , fluid or collapsed lung called atelectasis.** \n* The radiologists interpreting these studies can therefore use terms like infiltrates , consolidation and atelectasis interchangeably.\n\n### Architeture of ChexNet( from [Stanford ChexNet](https://arxiv.org/abs/1711.05225))\n* The **CheXNet algorithm** is a **121-layer deep 2D Convolutional Neural Network;** a **Densenet** after **Huang & Liu**. **The Densenet’s multiple residual connections reduce parameters and training time**, allowing a deeper, more powerful model. The model accepts a vectorized ***two-dimensional image of size 224 pixels by 224 pixels.***\n\n* To improve trust in CheXNet’s output, a Class Activation Mapping (GRAD-CAM) heatmap was utilized after [Zhou et al](https://people.csail.mit.edu/bzhou/publication/Zhou_Learning_Deep_Features_CVPR_2016_paper.pdf). This allows the human user to “see” what areas of the radiograph provide the strongest activation of the Densenet for the highest probability label. \n* CheXNet is a 121-layer Dense Convolutional Network (DenseNet) (Huang et al., 2016) trained on the ChestX-ray 14 dataset. DenseNets improve flow of information and gradients through the network, making the optimization of very deep networks tractable. We replace the final fully connected layer with one that has a single output, after which we apply a sigmoid nonlinearity. The weights of the network are initialized with weights from a model pretrained on ImageNet (Deng et al., 2009). The network is trained end-to-end using Adam with standard parameters (ß1 = 0.9 and ß2 = 0.999) (Kingma & Ba, 2014). We train the model using minibatches of size 16. We use an initial learning rate of 0.001 that is decayed by a factor of 10 each time the validation loss plateaus after an epoch, and pick the model with the lowest validation loss.\n\n","metadata":{"_uuid":"c16adc9b91919249fc92fc075bb97588d36566a5"}},{"cell_type":"markdown","source":"![](https://cloud.githubusercontent.com/assets/8370623/17981494/f838717a-6ad1-11e6-9391-f0906c80bc1d.jpg)\n![](https://cloud.githubusercontent.com/assets/8370623/17981496/fa648b32-6ad1-11e6-9625-02fdd72fdcd3.jpg)","metadata":{"_uuid":"31fd70be24ff5d46dbdff6b710ca1e456a1428b0"}},{"cell_type":"markdown","source":"# ChexNet Keras \n\n
\nimport importlib\nfrom keras.layers import Input\nfrom keras.layers.core import Dense\nfrom keras.models import Model\n\n\nclass ModelFactory:\n    \"\"\"\n    Model facotry for Keras default models\n    \"\"\"\n\n    def __init__(self):\n        self.models_ = dict(\n            VGG16=dict(\n                input_shape=(224, 224, 3),\n                module_name=\"vgg16\",\n                last_conv_layer=\"block5_conv3\",\n            ),\n            VGG19=dict(\n                input_shape=(224, 224, 3),\n                module_name=\"vgg19\",\n                last_conv_layer=\"block5_conv4\",\n            ),\n            DenseNet121=dict(\n                input_shape=(224, 224, 3),\n                module_name=\"densenet\",\n                last_conv_layer=\"bn\",\n            ),\n            ResNet50=dict(\n                input_shape=(224, 224, 3),\n                module_name=\"resnet50\",\n                last_conv_layer=\"activation_49\",\n            ),\n            InceptionV3=dict(\n                input_shape=(299, 299, 3),\n                module_name=\"inception_v3\",\n                last_conv_layer=\"mixed10\",\n            ),\n            InceptionResNetV2=dict(\n                input_shape=(299, 299, 3),\n                module_name=\"inception_resnet_v2\",\n                last_conv_layer=\"conv_7b_ac\",\n            ),\n            NASNetMobile=dict(\n                input_shape=(224, 224, 3),\n                module_name=\"nasnet\",\n                last_conv_layer=\"activation_188\",\n            ),\n            NASNetLarge=dict(\n                input_shape=(331, 331, 3),\n                module_name=\"nasnet\",\n                last_conv_layer=\"activation_260\",\n            ),\n        )\n\n    def get_last_conv_layer(self, model_name):\n        return self.models_[model_name][\"last_conv_layer\"]\n\n    def get_input_size(self, model_name):\n        return self.models_[model_name][\"input_shape\"][:2]\n\n    def get_model(self, class_names, model_name=\"DenseNet121\", use_base_weights=True,\n                  weights_path=None, input_shape=None):\n\n        if use_base_weights is True:\n            base_weights = \"imagenet\"\n        else:\n            base_weights = None\n\n        base_model_class = getattr(\n            importlib.import_module(\n                f\"keras.applications.{self.models_[model_name]['module_name']}\"\n            ),\n            model_name)\n\n        if input_shape is None:\n            input_shape = self.models_[model_name][\"input_shape\"]\n\n        img_input = Input(shape=input_shape)\n\n        base_model = base_model_class(\n            include_top=False,\n            input_tensor=img_input,\n            input_shape=input_shape,\n            weights=base_weights,\n            pooling=\"avg\")\n        x = base_model.output\n        predictions = Dense(len(class_names), activation=\"sigmoid\", name=\"predictions\")(x)\n        model = Model(inputs=img_input, outputs=predictions)\n\n        if weights_path == \"\":\n            weights_path = None\n\n        if weights_path is not None:\n            print(f\"load model weights_path: {weights_path}\")\n            model.load_weights(weights_path)\n        return model\n
","metadata":{"_uuid":"9e7979c98777525059304a853a3ad37d034304c7"}},{"cell_type":"markdown","source":"# Display Images for review","metadata":{"_uuid":"3656ea70d0a6b41c0eccfe7e743098addf38582c"}},{"cell_type":"code","source":"plt.style.use('default')\nfig=plt.figure(figsize=(20, 10))\ncolumns = 9; rows = 4\nfor i in range(1, columns*rows +1):\n ds = pydicom.dcmread(train_images_dir + train_images[i])\n fig.add_subplot(rows, columns, i)\n plt.imshow(ds.pixel_array, cmap=plt.cm.bone)\n fig.add_subplot","metadata":{"_uuid":"58dc39fc9746f92485eb51aa6e4972c3bdf3f119","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-09T12:35:43.166658Z","iopub.status.idle":"2024-05-09T12:35:43.167038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Distribution of Positive Case","metadata":{"_uuid":"27c4fff11aaf6071de35e6ae8e5f3fef3249617a"}},{"cell_type":"code","source":"from plotly.offline import init_notebook_mode, iplot\ninit_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.plotly as py\nimport cufflinks as cf\n\n# Number of positive targets\ntrain_labels = pd.read_csv('../input/stage_1_train_labels.csv')\nprint(round((8964 / (8964 + 20025)) * 100, 2), '% of the examples are positive')\ntrained_label = pd.DataFrame(train_labels.groupby('Target')['patientId'].count())\ntrace1 = go.Bar(x=train_labels.index, y=trained_label.patientId,\n marker=dict(color='rgb(158,120,200)',line=dict(color='rgb(8,48,107)',width=1.5)))\n\ndata = [trace1]\nlayout = dict(title = 'Classes by Patient',\n xaxis= dict(title= 'Patient ID',ticklen= 5,zeroline= False),\n yaxis = dict(title=\"Target\",ticklen= 5,zeroline= False))\n\nfig = dict(data = data, layout = layout)\niplot(fig)\ntrained_label","metadata":{"_kg_hide-input":true,"_uuid":"5cd2918e275423eb90c14293aff3d1260003fc67","execution":{"iopub.status.busy":"2024-05-09T12:35:43.16759Z","iopub.status.idle":"2024-05-09T12:35:43.167953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Size of the Area that Impact","metadata":{"_uuid":"2faddd0d4b3fe4db4f9270e7587b51a177933149"}},{"cell_type":"code","source":"cf.set_config_file(offline=False, world_readable=True, theme='ggplot')\ntrain_labels['area'] = train_labels['width'] * train_labels['height']\ntrace1 = go.Histogram(x=train_labels['area'],opacity=0.75,name = \"name\",marker=dict(color='rgba(256, 120, 236, 0.9)'))\nlayout = go.Layout(barmode='overlay',\n title='Distribution of Area within Image idenfitying a positive target',\n xaxis=dict(title='Area'),\n yaxis=dict( title='Count'),\n)\ndata = [trace1]\nfig = go.Figure(data=data, layout=layout)\niplot(fig)","metadata":{"_kg_hide-input":true,"_uuid":"2f78cf22e8a83a12d0032072b91ea4f4babb9187","execution":{"iopub.status.busy":"2024-05-09T12:35:43.169096Z","iopub.status.idle":"2024-05-09T12:35:43.169492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ploting Boxes Images","metadata":{"_uuid":"9ed54c53de6bc29c9589ca54c6f89eab8b217c26"}},{"cell_type":"code","source":"# Forked from `https://www.kaggle.com/peterchang77/exploratory-data-analysis`\ndef parse_data(df):\n \"\"\"\n Method to read a CSV file (Pandas dataframe) and parse the \n data into the following nested dictionary:\n\n parsed = {\n \n 'patientId-00': {\n 'dicom': path/to/dicom/file,\n 'label': either 0 or 1 for normal or pnuemonia, \n 'boxes': list of box(es)\n },\n 'patientId-01': {\n 'dicom': path/to/dicom/file,\n 'label': either 0 or 1 for normal or pnuemonia, \n 'boxes': list of box(es)\n }, ...\n\n }\n\n \"\"\"\n # --- Define lambda to extract coords in list [y, x, height, width]\n extract_box = lambda row: [row['y'], row['x'], row['height'], row['width']]\n\n parsed = {}\n for n, row in df.iterrows():\n # --- Initialize patient entry into parsed \n pid = row['patientId']\n if pid not in parsed:\n parsed[pid] = {\n 'dicom': '../input/stage_1_train_images/%s.dcm' % pid,\n 'label': row['Target'],\n 'boxes': []}\n\n # --- Add box if opacity is present\n if parsed[pid]['label'] == 1:\n parsed[pid]['boxes'].append(extract_box(row))\n\n return parsed\n\nparsed = parse_data(train_labels)\n\ndef draw(data):\n \"\"\"\n Method to draw single patient with bounding box(es) if present \n\n \"\"\"\n # --- Open DICOM file\n d = pydicom.read_file(data['dicom'])\n im = d.pixel_array\n\n # --- Convert from single-channel grayscale to 3-channel RGB\n im = np.stack([im] * 3, axis=2)\n\n # --- Add boxes with random color if present\n for box in data['boxes']:\n #rgb = np.floor(np.random.rand(3) * 256).astype('int')\n rgb = [255, 251, 204] # Just use yellow\n im = overlay_box(im=im, box=box, rgb=rgb, stroke=15)\n\n plt.imshow(im, cmap=plt.cm.gist_gray)\n plt.axis('off')\n\ndef overlay_box(im, box, rgb, stroke=2):\n \"\"\"\n Method to overlay single box on image\n\n \"\"\"\n # --- Convert coordinates to integers\n box = [int(b) for b in box]\n \n # --- Extract coordinates\n y1, x1, height, width = box\n y2 = y1 + height\n x2 = x1 + width\n\n im[y1:y1 + stroke, x1:x2] = rgb\n im[y2:y2 + stroke, x1:x2] = rgb\n im[y1:y2, x1:x1 + stroke] = rgb\n im[y1:y2, x2:x2 + stroke] = rgb\n\n return im","metadata":{"_kg_hide-input":true,"_uuid":"a42ec8cd5af2cfe329d354b47ef128aaa0c3c0ed","execution":{"iopub.status.busy":"2024-05-09T12:35:43.170024Z","iopub.status.idle":"2024-05-09T12:35:43.170361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use('default')\nfig=plt.figure(figsize=(20, 10))\ncolumns = 8; rows = 4\nfor i in range(1, columns*rows +1):\n fig.add_subplot(rows, columns, i)\n draw(parsed[train_labels['patientId'].unique()[i]])\n fig.add_subplot","metadata":{"_kg_hide-input":true,"_uuid":"a31ca3f3ba5fb89086e8d82972c1f43d5b60e843","execution":{"iopub.status.busy":"2024-05-09T12:35:43.170838Z","iopub.status.idle":"2024-05-09T12:35:43.171149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Label","metadata":{"_uuid":"ad01238bf9f9db0c44316d43234fe0851eba2264"}},{"cell_type":"code","source":"detailed_class_info = pd.read_csv('../input/stage_1_detailed_class_info.csv')\ndf_label = detailed_class_info.groupby('class').count()\n\ntrace1 = go.Bar(x=df_label.index, y=df_label.patientId,\n marker=dict(color='rgb(120,240,160)',line=dict(color='rgb(8,48,107)',width=1.5)))\n\ndata = [trace1]\nlayout = dict(title = 'Count of Class Label',\n xaxis= dict(title= 'Patient ID',ticklen= 5,zeroline= False),\n yaxis = dict(title=\"Target\",ticklen= 5,zeroline= False))\n\nfig = dict(data = data, layout = layout)\niplot(fig)\ndf_label","metadata":{"_kg_hide-input":true,"_uuid":"42f85381c633105a1a2ef6063c54702779cec786","execution":{"iopub.status.busy":"2024-05-09T12:35:43.171676Z","iopub.status.idle":"2024-05-09T12:35:43.171998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"2f5a123fc312d119afcda73f3b04cc2cd45fdbab","trusted":true},"execution_count":null,"outputs":[]}]}