{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-29T21:38:32.367023Z","iopub.execute_input":"2022-07-29T21:38:32.367449Z","iopub.status.idle":"2022-07-29T21:38:55.904278Z","shell.execute_reply.started":"2022-07-29T21:38:32.367415Z","shell.execute_reply":"2022-07-29T21:38:55.902474Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob, pylab, pandas as pd\nimport pydicom, numpy as np","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:41:10.011929Z","iopub.execute_input":"2022-07-29T21:41:10.012624Z","iopub.status.idle":"2022-07-29T21:41:10.019276Z","shell.execute_reply.started":"2022-07-29T21:41:10.012591Z","shell.execute_reply":"2022-07-29T21:41:10.017766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:46:13.396426Z","iopub.execute_input":"2022-07-29T21:46:13.396907Z","iopub.status.idle":"2022-07-29T21:46:14.184147Z","shell.execute_reply.started":"2022-07-29T21:46:13.396871Z","shell.execute_reply":"2022-07-29T21:46:14.18247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('stage_2_train_labels.csv')\nprint(df.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:46:25.28763Z","iopub.execute_input":"2022-07-29T21:46:25.288138Z","iopub.status.idle":"2022-07-29T21:46:25.370929Z","shell.execute_reply.started":"2022-07-29T21:46:25.288102Z","shell.execute_reply":"2022-07-29T21:46:25.369564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patientId = df['patientId'][0]\ndcm_file = 'stage_2_train_images/%s.dcm' % patientId\ndcm_data = pydicom.read_file(dcm_file)\n\nprint(dcm_data)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:47:16.066369Z","iopub.execute_input":"2022-07-29T21:47:16.06679Z","iopub.status.idle":"2022-07-29T21:47:16.088801Z","shell.execute_reply.started":"2022-07-29T21:47:16.066756Z","shell.execute_reply":"2022-07-29T21:47:16.087393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"im = dcm_data.pixel_array\nprint(type(im))\nprint(im.dtype)\nprint(im.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:47:39.563114Z","iopub.execute_input":"2022-07-29T21:47:39.563576Z","iopub.status.idle":"2022-07-29T21:47:39.601673Z","shell.execute_reply.started":"2022-07-29T21:47:39.563542Z","shell.execute_reply":"2022-07-29T21:47:39.60017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pylab.imshow(im, cmap=pylab.cm.gist_gray)\npylab.axis('off')","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:47:52.999542Z","iopub.execute_input":"2022-07-29T21:47:53.000003Z","iopub.status.idle":"2022-07-29T21:47:53.478522Z","shell.execute_reply.started":"2022-07-29T21:47:52.999969Z","shell.execute_reply":"2022-07-29T21:47:53.477269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def parse_data(df):\n    \"\"\"\n    Method to read a CSV file (Pandas dataframe) and parse the \n    data into the following nested dictionary:\n\n      parsed = {\n        \n        'patientId-00': {\n            'dicom': path/to/dicom/file,\n            'label': either 0 or 1 for normal or pnuemonia, \n            'boxes': list of box(es)\n        },\n        'patientId-01': {\n            'dicom': path/to/dicom/file,\n            'label': either 0 or 1 for normal or pnuemonia, \n            'boxes': list of box(es)\n        }, ...\n\n      }\n\n    \"\"\"\n    # --- Define lambda to extract coords in list [y, x, height, width]\n    extract_box = lambda row: [row['y'], row['x'], row['height'], row['width']]\n\n    parsed = {}\n    for n, row in df.iterrows():\n        # --- Initialize patient entry into parsed \n        pid = row['patientId']\n        if pid not in parsed:\n            parsed[pid] = {\n                'dicom': 'stage_2_train_images/%s.dcm' % pid,\n                'label': row['Target'],\n                'boxes': []}\n\n        # --- Add box if opacity is present\n        if parsed[pid]['label'] == 1:\n            parsed[pid]['boxes'].append(extract_box(row))\n\n    return parsed","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:50:51.497973Z","iopub.execute_input":"2022-07-29T21:50:51.498443Z","iopub.status.idle":"2022-07-29T21:50:51.507461Z","shell.execute_reply.started":"2022-07-29T21:50:51.498408Z","shell.execute_reply":"2022-07-29T21:50:51.506251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parsed = parse_data(df)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:50:52.945968Z","iopub.execute_input":"2022-07-29T21:50:52.946623Z","iopub.status.idle":"2022-07-29T21:50:55.280045Z","shell.execute_reply.started":"2022-07-29T21:50:52.946572Z","shell.execute_reply":"2022-07-29T21:50:55.278627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(parsed['00436515-870c-4b36-a041-de91049b9ab4'])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:50:55.282437Z","iopub.execute_input":"2022-07-29T21:50:55.282828Z","iopub.status.idle":"2022-07-29T21:50:55.28989Z","shell.execute_reply.started":"2022-07-29T21:50:55.282794Z","shell.execute_reply":"2022-07-29T21:50:55.288315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw(data):\n    \"\"\"\n    Method to draw single patient with bounding box(es) if present \n\n    \"\"\"\n    # --- Open DICOM file\n    d = pydicom.read_file(data['dicom'])\n    im = d.pixel_array\n\n    # --- Convert from single-channel grayscale to 3-channel RGB\n    im = np.stack([im] * 3, axis=2)\n\n    # --- Add boxes with random color if present\n    for box in data['boxes']:\n        rgb = np.floor(np.random.rand(3) * 256).astype('int')\n        im = overlay_box(im=im, box=box, rgb=rgb, stroke=6)\n\n    pylab.imshow(im, cmap=pylab.cm.gist_gray)\n    pylab.axis('off')\n\ndef overlay_box(im, box, rgb, stroke=1):\n    \"\"\"\n    Method to overlay single box on image\n\n    \"\"\"\n    # --- Convert coordinates to integers\n    box = [int(b) for b in box]\n    \n    # --- Extract coordinates\n    y1, x1, height, width = box\n    y2 = y1 + height\n    x2 = x1 + width\n\n    im[y1:y1 + stroke, x1:x2] = rgb\n    im[y2:y2 + stroke, x1:x2] = rgb\n    im[y1:y2, x1:x1 + stroke] = rgb\n    im[y1:y2, x2:x2 + stroke] = rgb\n\n    return im","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:50:56.802352Z","iopub.execute_input":"2022-07-29T21:50:56.802795Z","iopub.status.idle":"2022-07-29T21:50:56.814912Z","shell.execute_reply.started":"2022-07-29T21:50:56.80276Z","shell.execute_reply":"2022-07-29T21:50:56.8137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"draw(parsed['00436515-870c-4b36-a041-de91049b9ab4'])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:51:02.928464Z","iopub.execute_input":"2022-07-29T21:51:02.928914Z","iopub.status.idle":"2022-07-29T21:51:03.213418Z","shell.execute_reply.started":"2022-07-29T21:51:02.92888Z","shell.execute_reply":"2022-07-29T21:51:03.21224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_detailed = pd.read_csv('stage_2_detailed_class_info.csv')\nprint(df_detailed.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:51:26.923883Z","iopub.execute_input":"2022-07-29T21:51:26.92441Z","iopub.status.idle":"2022-07-29T21:51:26.9895Z","shell.execute_reply.started":"2022-07-29T21:51:26.92437Z","shell.execute_reply":"2022-07-29T21:51:26.987669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patientId = df_detailed['patientId'][0]\ndraw(parsed[patientId])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:51:37.208876Z","iopub.execute_input":"2022-07-29T21:51:37.210225Z","iopub.status.idle":"2022-07-29T21:51:37.516231Z","shell.execute_reply.started":"2022-07-29T21:51:37.210118Z","shell.execute_reply":"2022-07-29T21:51:37.514337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary = {}\nfor n, row in df_detailed.iterrows():\n    if row['class'] not in summary:\n        summary[row['class']] = 0\n    summary[row['class']] += 1\n    \nprint(summary)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:51:49.153245Z","iopub.execute_input":"2022-07-29T21:51:49.15376Z","iopub.status.idle":"2022-07-29T21:51:51.098547Z","shell.execute_reply.started":"2022-07-29T21:51:49.153723Z","shell.execute_reply":"2022-07-29T21:51:51.096356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom glob import glob\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport seaborn as sns\nimport pydicom as dcm\nimport math\nfrom tensorflow.keras.layers import Layer, Convolution2D, Flatten, Dense\nfrom tensorflow.keras.layers import Concatenate, UpSampling2D, Conv2D, Reshape, GlobalAveragePooling2D\nfrom tensorflow.keras.models import Model\n\nimport cv2\n\nfrom tensorflow.keras.applications.mobilenet import MobileNet\nfrom tensorflow.keras.applications.mobilenet import preprocess_input \n\nimport tensorflow.keras.utils as pltUtil\nfrom tensorflow.keras.utils import Sequence\n\nimport math\n\nfrom tensorflow.keras.applications.resnet import ResNet50\nfrom tensorflow.keras.applications.resnet import preprocess_input as resnetProcess_input","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:57:03.778233Z","iopub.execute_input":"2022-07-29T21:57:03.778685Z","iopub.status.idle":"2022-07-29T21:57:12.973595Z","shell.execute_reply.started":"2022-07-29T21:57:03.778653Z","shell.execute_reply":"2022-07-29T21:57:12.972241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv(\"stage_2_train_labels.csv\")\nlabels.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:58:14.638368Z","iopub.execute_input":"2022-07-29T21:58:14.638768Z","iopub.status.idle":"2022-07-29T21:58:14.703353Z","shell.execute_reply.started":"2022-07-29T21:58:14.638737Z","shell.execute_reply":"2022-07-29T21:58:14.701691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.shape\n## There are 30227 records and 6 rows","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:58:27.43691Z","iopub.execute_input":"2022-07-29T21:58:27.437337Z","iopub.status.idle":"2022-07-29T21:58:27.44611Z","shell.execute_reply.started":"2022-07-29T21:58:27.437301Z","shell.execute_reply":"2022-07-29T21:58:27.444538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.info()\n## There are 30227 rows \n## there are 30277 target as well\n## x ,y, WIdth and height count is 9555 , the others are null","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:59:29.815109Z","iopub.execute_input":"2022-07-29T21:59:29.816166Z","iopub.status.idle":"2022-07-29T21:59:29.850383Z","shell.execute_reply.started":"2022-07-29T21:59:29.816122Z","shell.execute_reply":"2022-07-29T21:59:29.84948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels[labels.isnull().any(axis=1)].Target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:59:42.733656Z","iopub.execute_input":"2022-07-29T21:59:42.734045Z","iopub.status.idle":"2022-07-29T21:59:42.754217Z","shell.execute_reply.started":"2022-07-29T21:59:42.734014Z","shell.execute_reply":"2022-07-29T21:59:42.753274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## we can see that all the non null column values are with Target 1 indicating that those patients have pneumonia\nlabels[~labels.isnull().any(axis=1)].Target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T21:59:56.916135Z","iopub.execute_input":"2022-07-29T21:59:56.916546Z","iopub.status.idle":"2022-07-29T21:59:56.93504Z","shell.execute_reply.started":"2022-07-29T21:59:56.916514Z","shell.execute_reply":"2022-07-29T21:59:56.934145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Distubution of Targets , there are 20672 records with no pneumonia and 9555 with pneumonia\nlabels.Target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:00:08.023234Z","iopub.execute_input":"2022-07-29T22:00:08.024167Z","iopub.status.idle":"2022-07-29T22:00:08.033737Z","shell.execute_reply.started":"2022-07-29T22:00:08.024117Z","shell.execute_reply":"2022-07-29T22:00:08.032413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Disturbution of Target, there are 31% of patients with pneumonia and the remaining are no pneumonia\n## There is a class imbalance issue\nlabel_count=labels['Target'].value_counts()\nexplode = (0.01,0.01)  \n\nfig1, ax1 = plt.subplots(figsize=(5,5))\nax1.pie(label_count.values, explode=explode, labels=label_count.index, autopct='%1.1f%%',\n        shadow=True, startangle=90)\nax1.axis('equal') \nplt.title('Target Distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:00:19.755925Z","iopub.execute_input":"2022-07-29T22:00:19.756365Z","iopub.status.idle":"2022-07-29T22:00:19.924559Z","shell.execute_reply.started":"2022-07-29T22:00:19.75633Z","shell.execute_reply":"2022-07-29T22:00:19.922642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicateRowsDF = labels[labels.duplicated(['patientId'])]\nduplicateRowsDF.shape\n\n## There are 3543 duplicates","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:01:02.881014Z","iopub.execute_input":"2022-07-29T22:01:02.881498Z","iopub.status.idle":"2022-07-29T22:01:02.898868Z","shell.execute_reply.started":"2022-07-29T22:01:02.881461Z","shell.execute_reply":"2022-07-29T22:01:02.897654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicateRowsDF.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:01:11.665479Z","iopub.execute_input":"2022-07-29T22:01:11.665913Z","iopub.status.idle":"2022-07-29T22:01:11.682456Z","shell.execute_reply.started":"2022-07-29T22:01:11.665874Z","shell.execute_reply":"2022-07-29T22:01:11.680698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Examining one of the patient id which is duplicate , we can see that the x,y, widht and height is not the same\n## This indicates that the same patient has two bounding boxes in the same dicom image\nlabels[labels.patientId=='00436515-870c-4b36-a041-de91049b9ab4']","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:01:21.871324Z","iopub.execute_input":"2022-07-29T22:01:21.87178Z","iopub.status.idle":"2022-07-29T22:01:21.89386Z","shell.execute_reply.started":"2022-07-29T22:01:21.871745Z","shell.execute_reply":"2022-07-29T22:01:21.892965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels[labels.patientId=='00704310-78a8-4b38-8475-49f4573b2dbb']","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:01:33.036677Z","iopub.execute_input":"2022-07-29T22:01:33.037105Z","iopub.status.idle":"2022-07-29T22:01:33.058271Z","shell.execute_reply.started":"2022-07-29T22:01:33.03707Z","shell.execute_reply":"2022-07-29T22:01:33.057357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Reading the classes label , \nclass_labels = pd.read_csv('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv')\nclass_labels.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:01:47.487417Z","iopub.execute_input":"2022-07-29T22:01:47.487776Z","iopub.status.idle":"2022-07-29T22:01:47.53717Z","shell.execute_reply.started":"2022-07-29T22:01:47.487747Z","shell.execute_reply":"2022-07-29T22:01:47.536288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_labels.shape\n## There are 30277 rows and two columns , 30277 rows same as the labels data set","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:02:00.206968Z","iopub.execute_input":"2022-07-29T22:02:00.207396Z","iopub.status.idle":"2022-07-29T22:02:00.214921Z","shell.execute_reply.started":"2022-07-29T22:02:00.20736Z","shell.execute_reply":"2022-07-29T22:02:00.213467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_labels.info()\n## There are no null columns ","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:02:08.889557Z","iopub.execute_input":"2022-07-29T22:02:08.889932Z","iopub.status.idle":"2022-07-29T22:02:08.913316Z","shell.execute_reply.started":"2022-07-29T22:02:08.889902Z","shell.execute_reply":"2022-07-29T22:02:08.912332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_labels['class'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:02:24.070437Z","iopub.execute_input":"2022-07-29T22:02:24.070807Z","iopub.status.idle":"2022-07-29T22:02:24.083937Z","shell.execute_reply.started":"2022-07-29T22:02:24.070777Z","shell.execute_reply":"2022-07-29T22:02:24.082527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Disturbution of Classes, there are 39% of patients with No Lung opacity , 29.3% Normal \n## and the remaining are with Lung Opacity\nlabel_count=class_labels['class'].value_counts()\nexplode = (0.01,0.01,0.01)  \n\nfig1, ax1 = plt.subplots(figsize=(5,5))\nax1.pie(label_count.values, explode=explode, labels=label_count.index, autopct='%1.1f%%',\n        shadow=True, startangle=90)\nax1.axis('equal') \nplt.title('Class Distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:02:32.612347Z","iopub.execute_input":"2022-07-29T22:02:32.612737Z","iopub.status.idle":"2022-07-29T22:02:32.758762Z","shell.execute_reply.started":"2022-07-29T22:02:32.612707Z","shell.execute_reply":"2022-07-29T22:02:32.75742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#labels.loc[labels.index.repeat(labels.patientId)]\nduplicateClassRowsDF = class_labels[class_labels.duplicated(['patientId'])]\nduplicateClassRowsDF.shape\n\n## There are 3543 duplicates similar to the labels dataset","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:02:44.939399Z","iopub.execute_input":"2022-07-29T22:02:44.940383Z","iopub.status.idle":"2022-07-29T22:02:44.959278Z","shell.execute_reply.started":"2022-07-29T22:02:44.940347Z","shell.execute_reply":"2022-07-29T22:02:44.958156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicateClassRowsDF.head(2)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:02:54.877271Z","iopub.execute_input":"2022-07-29T22:02:54.878565Z","iopub.status.idle":"2022-07-29T22:02:54.893266Z","shell.execute_reply.started":"2022-07-29T22:02:54.878512Z","shell.execute_reply":"2022-07-29T22:02:54.89162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## The same patient id has the same class even though they are duplicate\nclass_labels[class_labels.patientId=='00704310-78a8-4b38-8475-49f4573b2dbb']","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:03:06.239286Z","iopub.execute_input":"2022-07-29T22:03:06.239721Z","iopub.status.idle":"2022-07-29T22:03:06.258718Z","shell.execute_reply.started":"2022-07-29T22:03:06.239678Z","shell.execute_reply":"2022-07-29T22:03:06.257685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Conctinating the two dataset - 'labels' and 'class_labels':\ntraining_data = pd.concat([labels, class_labels['class']], axis = 1)\n\ntraining_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:03:17.34235Z","iopub.execute_input":"2022-07-29T22:03:17.342799Z","iopub.status.idle":"2022-07-29T22:03:17.367678Z","shell.execute_reply.started":"2022-07-29T22:03:17.342764Z","shell.execute_reply":"2022-07-29T22:03:17.366713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows = 1, figsize = (12, 6))\ntemp = training_data.groupby('Target')['class'].value_counts()\ndata_target_class = pd.DataFrame(data = {'Values': temp.values}, index = temp.index).reset_index()\nsns.barplot(ax = ax, x = 'Target', y = 'Values', hue = 'class', data = data_target_class, palette = 'Set3')\nplt.title('Class and Target  Distrubution')\n\n## it shows that class distrubution grouped by Target \n## Target 0 has only Normal or No Lung Opacity class\n## Target 1 has only Lung Opacity class","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:03:26.686976Z","iopub.execute_input":"2022-07-29T22:03:26.687396Z","iopub.status.idle":"2022-07-29T22:03:27.222409Z","shell.execute_reply.started":"2022-07-29T22:03:26.687362Z","shell.execute_reply":"2022-07-29T22:03:27.220994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## ANalysing the dicom image\nimport matplotlib.patches as patches\n\ndef inspectImages(data):\n    img_data = list(data.T.to_dict().values())\n    f, ax = plt.subplots(3,3, figsize=(16,18))\n    for i,data_row in enumerate(img_data):\n        patientImage = data_row['patientId']\n        dcm_file = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/'+'{}.dcm'.format(patientImage)\n        data_row_img_data = dcm.read_file(dcm_file)\n        modality = data_row_img_data.Modality\n        age = data_row_img_data.PatientAge\n        sex = data_row_img_data.PatientSex\n        data_row_img = dcm.dcmread(dcm_file)\n        ax[i//3, i%3].imshow(data_row_img.pixel_array, cmap=plt.cm.bone) \n        ax[i//3, i%3].axis('off')\n        ax[i//3, i%3].set_title('ID: {}\\nModality: {} Age: {} Sex: {} Target: {}\\nClass: {}\\Bounds: {}:{}:{}:{}'.format(\n                data_row['patientId'],\n                modality, age, sex, data_row['Target'], data_row['class'], \n                data_row['x'],data_row['y'],data_row['width'],data_row['height']))\n        label = data_row[\"class\"]\n        if not math.isnan(data_row['x']):\n            x, y, width, height  =  data_row['x'],data_row['y'],data_row['width'],data_row['height']\n            rect = patches.Rectangle((x, y),width, height,\n                                 linewidth = 2,\n                                 edgecolor = 'r',\n                                 facecolor = 'none')\n\n        # Draw the bounding box on top of the image\n            ax[i//3, i%3].add_patch(rect)\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:04:03.992582Z","iopub.execute_input":"2022-07-29T22:04:03.993627Z","iopub.status.idle":"2022-07-29T22:04:04.012428Z","shell.execute_reply.started":"2022-07-29T22:04:03.993575Z","shell.execute_reply":"2022-07-29T22:04:04.011053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## DCIM image contain the meta data alon with it, \n## Function to read the dcim data and appending to the resultset\ndef readDCIMData(rowData):\n    dcm_file = 'stage_2_train_images/'+'{}.dcm'.format(rowData.patientId)\n    dcm_data = dcm.read_file(dcm_file)\n    img = dcm_data.pixel_array\n    return dcm_data.PatientSex,dcm_data.PatientAge","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:17:55.720374Z","iopub.execute_input":"2022-07-29T22:17:55.720757Z","iopub.status.idle":"2022-07-29T22:17:55.727165Z","shell.execute_reply.started":"2022-07-29T22:17:55.720728Z","shell.execute_reply":"2022-07-29T22:17:55.725412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Reading the image data and append it to the training_data dataset\ntraining_data['sex'], training_data['age'] = zip(*training_data.apply(readDCIMData, axis=1))","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:18:06.67735Z","iopub.execute_input":"2022-07-29T22:18:06.677732Z","iopub.status.idle":"2022-07-29T22:23:17.843565Z","shell.execute_reply.started":"2022-07-29T22:18:06.677702Z","shell.execute_reply":"2022-07-29T22:23:17.842179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data.info()\n## There are 30227  records\n## sex and age also have the smae count indicating that there are no images missing every patient has an dicom image \n## Age should be a numeric , currently it is an objetc","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:24:09.408324Z","iopub.execute_input":"2022-07-29T22:24:09.409001Z","iopub.status.idle":"2022-07-29T22:24:09.443458Z","shell.execute_reply.started":"2022-07-29T22:24:09.408962Z","shell.execute_reply":"2022-07-29T22:24:09.44128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Converting age to Numeric as the current data type is a String\ntraining_data['age'] = training_data.age.astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:24:10.589565Z","iopub.execute_input":"2022-07-29T22:24:10.589971Z","iopub.status.idle":"2022-07-29T22:24:10.601887Z","shell.execute_reply.started":"2022-07-29T22:24:10.589941Z","shell.execute_reply":"2022-07-29T22:24:10.600715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data.describe(include=\"all\").T\n## The mean age is 46 years , where as minimum age is 1 year and the max age is 155 which seems to be an outlier\n## 50% of the patiens are of aroudn 49 age , the std deviation is 16 which sugges that age is not normally distubuted","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:24:14.965826Z","iopub.execute_input":"2022-07-29T22:24:14.966269Z","iopub.status.idle":"2022-07-29T22:24:15.052866Z","shell.execute_reply.started":"2022-07-29T22:24:14.966233Z","shell.execute_reply":"2022-07-29T22:24:15.052098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data.sex.value_counts()\n## there are only two genders ","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:24:16.159604Z","iopub.execute_input":"2022-07-29T22:24:16.160324Z","iopub.status.idle":"2022-07-29T22:24:16.173801Z","shell.execute_reply.started":"2022-07-29T22:24:16.160262Z","shell.execute_reply":"2022-07-29T22:24:16.17253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,5))  # setting the figure size\nax = sns.barplot(x='class', y='age', data=training_data, palette='muted')  # barplot'\n## This is the distubution of Age with class, maximum age of person with pneuomina is arund 45","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:24:20.342943Z","iopub.execute_input":"2022-07-29T22:24:20.343377Z","iopub.status.idle":"2022-07-29T22:24:20.961887Z","shell.execute_reply.started":"2022-07-29T22:24:20.343342Z","shell.execute_reply":"2022-07-29T22:24:20.960469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,7))\nsns.boxplot(x='class', y='age', data= training_data)\nplt.show()\n\n## The  class which has no pneuomia has few outliers , theie age is somewhere aroun 150 years","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:24:22.937745Z","iopub.execute_input":"2022-07-29T22:24:22.938455Z","iopub.status.idle":"2022-07-29T22:24:23.187815Z","shell.execute_reply.started":"2022-07-29T22:24:22.938417Z","shell.execute_reply":"2022-07-29T22:24:23.186679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Distribution of `Age`: Overall and Target = 1')\nfig = plt.figure(figsize = (10, 6))\n\nax = fig.add_subplot(121)\ng = (sns.distplot(training_data['age']).set_title('Distribution of PatientAge'))\n\nax = fig.add_subplot(122)\ng = (sns.distplot(training_data.loc[training_data['Target'] == 1, 'age']).set_title('Distribution of PatientAge who have pneumonia'))\n\n## Overall Distrubution of Age looks normal with very little skwe\n## Distubution of Patients afe who have penuomonia are a left skewed ","metadata":{"execution":{"iopub.status.busy":"2022-07-29T22:24:24.047714Z","iopub.execute_input":"2022-07-29T22:24:24.048133Z","iopub.status.idle":"2022-07-29T22:24:24.845235Z","shell.execute_reply.started":"2022-07-29T22:24:24.048099Z","shell.execute_reply":"2022-07-29T22:24:24.843914Z"},"trusted":true},"execution_count":null,"outputs":[]}]}