{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"main_dir = '/kaggle/input/rsna-pneumonia-detection-challenge'\ntraining_image_dir = main_dir + '/' + 'stage_2_train_images'\ntest_image_dir = main_dir + '/' + 'stage_2_test_images'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"os.getcwd()\n\nnum_of_training_samples = len([name for name in os.listdir(training_image_dir) if os.path.isfile(os.path.join(training_image_dir, name)) ])\nnum_of_test_samples = len([name for name in os.listdir(test_image_dir) if os.path.isfile(os.path.join(test_image_dir, name)) ])\n\nprint('Number of training samples : ', num_of_training_samples  )\nprint('Number of test samples     : ', num_of_test_samples )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\n\ntrain_class_info = pd.read_csv(main_dir + \"/stage_2_detailed_class_info.csv\")\ndisplay(train_class_info)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = pd.read_csv(main_dir +  \"/stage_2_train_labels.csv\")\ndisplay(train_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_labels.shape)\nprint(train_class_info.shape)\n\ntrain_label_class_info_merged =  pd.merge(train_class_info, train_labels, how='inner',left_index = True, right_index = True )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_label_class_info_merged.drop('patientId_y',axis = 1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_label_class_info_merged.rename(columns={\"patientId_x\": \"patientId\"},inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_label_class_info_merged.loc [:, ['Target','class'] ].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_label_class_info_merged.loc [ train_label_class_info_merged.Target == 1 , ['x','y','width','height'] ].isna().count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_label_class_info_merged.groupby(['patientId'], sort=False).count().max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_label_class_info_merged","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_label_class_info_merged.sort_values(by=['patientId'],inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\n\ndef fn_image_info_plot_image( image_dir, image_file_name, bPlot = True ) :\n    filename = image_dir + '/' + image_file_name\n    dataset = pydicom.dcmread(filename)\n\n    print()\n    print(\"Filename.........:\", filename)\n    print(\"Storage type.....:\", dataset.SOPClassUID)\n    print()\n\n    pat_name = dataset.PatientName\n    display_name = pat_name.family_name + \", \" + pat_name.given_name\n    print(\"Patient's name...:\", display_name)\n    print(\"Patient id.......:\", dataset.PatientID)\n    print(\"Modality.........:\", dataset.Modality)\n    print(\"Study Date.......:\", dataset.StudyDate)\n    print(\"Patient Age .......:\", dataset.PatientAge)\n    print(\"Patient Sex .......:\", dataset.PatientSex)\n    print(\"Body Part Examined .......:\", dataset.BodyPartExamined)\n    print(\"View Position .......:\", dataset.ViewPosition)\n    \n    if 'PixelData' in dataset:\n        rows = int(dataset.Rows)\n        cols = int(dataset.Columns)\n        print(\"Image size.......: {rows:d} x {cols:d}, {size:d} bytes\".format(rows=rows, cols=cols, size=len(dataset.PixelData)))\n    \n    if 'PixelSpacing' in dataset:\n        print(\"Pixel spacing....:\", dataset.PixelSpacing)\n\n    print(\"Slice location...:\", dataset.get('SliceLocation', \"(missing)\"))\n    \n    if bPlot == True :\n        f, ax = plt.subplots(1,1, figsize=(8,9))\n        \n        plt.imshow(dataset.pixel_array, cmap=plt.cm.bone)\n        \n        rows = train_label_class_info_merged[train_label_class_info_merged['patientId']==dataset.PatientID]\n        box_data = list(rows.T.to_dict().values())\n        for j, row in enumerate(box_data):\n            ax.add_patch(Rectangle(xy=(row['x'], row['y']),\n                        width=row['width'],height=row['height'], \n                        color=\"yellow\",alpha = 0.1))   \n        \n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fn_image_info_plot_image(training_image_dir,'000db696-cf54-4385-b10b-6b16fbb3f985.dcm', True )\n\nfn_image_info_plot_image(training_image_dir,'c1ec14ff-f6d7-4b38-b0cb-fe07041cbdc8.dcm',True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_WIDTH = 224\nIMAGE_HEIGHT = 224\nALPHA = 1.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nfrom tensorflow.keras.applications.mobilenet import preprocess_input\nfrom tensorflow.keras.preprocessing.image import img_to_array\n\ndef prepare_image_data(imagedata):\n    img = cv2.resize(imagedata, (IMAGE_WIDTH,IMAGE_HEIGHT), interpolation=cv2.INTER_CUBIC)\n    if len(img.shape) != 3 or img.shape[2] != 3:\n            img = np.stack((img,) * 3, -1)\n    img_array = img_to_array(img)\n    img_array_expanded_dims = np.expand_dims(img_array, axis = 0)\n    return preprocess_input(img_array_expanded_dims)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"masks = np.zeros((num_of_training_samples, IMAGE_HEIGHT, IMAGE_WIDTH))\nX_train = np.zeros((num_of_training_samples, IMAGE_HEIGHT, IMAGE_WIDTH, 3))\n\ntraining_image_meta_info = pd.DataFrame(columns = ['PatientID', 'Modality', 'PatientAge','PatientSex','BodyPartExamined','ViewPosition','Rows','Columns','PixelSpacing','SliceLocation','FileName'])\n\nindex = 0\n\nfor filename in os.listdir(training_image_dir) :\n    dataset = pydicom.dcmread(os.path.join(training_image_dir, filename))\n    training_image_meta_info = training_image_meta_info.append({'PatientID' : dataset.PatientID, 'Modality': dataset.Modality, 'PatientAge': dataset.PatientAge, 'PatientSex' : dataset.PatientSex, 'BodyPartExamined': dataset.BodyPartExamined, 'ViewPosition' : dataset.ViewPosition, 'Rows' : int(dataset.Rows), 'Columns' : int(dataset.Columns), 'PixelSpacing' : dataset.PixelSpacing, 'SliceLocation' : dataset.get('SliceLocation', \"-1\"), 'FileName' : filename},\n                                                               ignore_index = True)\n               \n    #try:\n    #    img = img[:, :, :3]\n    #except:\n    #    continue\n            \n    # X_train[index] = preprocess_input(np.array(img, dtype=np.float32))\n    X_train[index] = prepare_image_data(dataset.pixel_array)\n    \n    mask_df = train_labels.loc[ ( train_labels.patientId == dataset.PatientID ) &  ( train_labels.Target == 1 ) ,['x','y','width','height'] ]\n        \n    for mask_df_index, mask_df_row in mask_df.iterrows():\n        x1 = int(mask_df_row['x'] * IMAGE_WIDTH)\n        x2 = int(( mask_df_row['x'] + mask_df_row['width'] )  * IMAGE_WIDTH)\n        y1 = int(mask_df_row['y'] * IMAGE_HEIGHT)\n        y2 = int(( mask_df_row['y'] + mask_df_row['height'] ) * IMAGE_HEIGHT)\n        masks[index][y1:y2, x1:x2] = 1\n        \n    index += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image_meta_info = pd.DataFrame(columns = ['PatientID', 'Modality', 'PatientAge','PatientSex','BodyPartExamined','ViewPosition','Rows','Columns','PixelSpacing','SliceLocation','FileName'])\n\nfor filename in os.listdir(test_image_dir) :\n    dataset = pydicom.dcmread(os.path.join(test_image_dir, filename))\n    test_image_meta_info = test_image_meta_info.append({'PatientID' : dataset.PatientID, 'Modality': dataset.Modality, 'PatientAge': dataset.PatientAge, 'PatientSex' : dataset.PatientSex, 'BodyPartExamined': dataset.BodyPartExamined, 'ViewPosition' : dataset.ViewPosition, 'Rows' : int(dataset.Rows), 'Columns' : int(dataset.Columns), 'PixelSpacing' : dataset.PixelSpacing, 'SliceLocation' : dataset.get('SliceLocation', \"-1\"), 'FileName' : filename},\n                           ignore_index = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_image_meta_info","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image_meta_info","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_class = train_label_class_info_merged.groupby('patientId')['class','Target'].max()\npatient_class","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_image_meta_info_merged = pd.merge( training_image_meta_info, patient_class, how='inner', left_on = 'PatientID', right_on = 'patientId')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_info = patient_image_meta_info_merged.drop(['Modality','BodyPartExamined','Rows','Columns','PixelSpacing','SliceLocation','FileName'],axis = 1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_info.loc[patient_info.PatientID == '000db696-cf54-4385-b10b-6b16fbb3f985',:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_info.loc[patient_info.Target == 1, 'class'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_info.loc[patient_info.Target == 1, 'ViewPosition'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_info.loc[patient_info.Target == 1, 'PatientSex'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\n\nsns.distplot(patient_info.loc[patient_info.Target == 0, 'PatientAge'], hist=False, rug=True)\nsns.distplot(patient_info.loc[patient_info.Target == 1, 'PatientAge'], hist=False, rug=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Distribution of age follows almost a uniform distribution both with patients diagnosed with Pnemonia and those who are not diagnosed with Pnemonia","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(patient_info['PatientSex'], hue=patient_info[ 'Target']) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Proportion of female patients diagnosed with Pnemonia vs total number of female pateints is almost similiar compared to male patients diagnosed","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(patient_info['ViewPosition'], hue=patient_info[ 'Target']) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generally X-Rays are taken in 'AP' view position for the patients who are already bed ridden. So we have large number of patients whose X-Rays taken in AP position, were diagnosed with Pneumonia.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = patient_info.groupby(['class', 'PatientSex'])['PatientID'].count()\ndf1 = pd.DataFrame(data={'Count': tmp.values}, index=tmp.index).reset_index()\ntmp = df1.groupby(['Count','class', 'PatientSex']).count()\ndf3 = pd.DataFrame(data=tmp.values, index=tmp.index).reset_index()\nfig, (ax) = plt.subplots(nrows=1,figsize=(6,6))\nsns.barplot(ax=ax, x = 'PatientSex', y='Count', hue='class',data=df3)\nplt.title(\"Train set: Patient Sex and class\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The number of classes - Normal and \"No Long Opacity/Not Normal\" is similar in case of Male and female patients","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import Concatenate, UpSampling2D, Conv2D, Reshape\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.applications.mobilenet import MobileNet\n\ndef create_model(trainable=True):\n    model = MobileNet(input_shape=(IMAGE_HEIGHT, IMAGE_WIDTH, 3), include_top=False, alpha=ALPHA, weights=\"imagenet\")\n\n    for layer in model.layers:\n        layer.trainable = trainable\n\n    # Add all the UNET layers here\n    block  = model.get_layer(\"conv_pw_1_relu\").output\n    block1 = model.get_layer(\"conv_pw_3_relu\").output\n    block2 = model.get_layer(\"conv_pw_5_relu\").output\n    block3 = model.get_layer(\"conv_pw_11_relu\").output\n    block4 = model.get_layer(\"conv_pw_13_relu\").output\n\n    x = Concatenate()([UpSampling2D()(block4), block3])\n    x = Concatenate()([UpSampling2D()(x), block2])\n    x = Concatenate()([UpSampling2D()(x), block1])\n    x = Concatenate()([UpSampling2D()(x), block])\n    x = UpSampling2D()(x)\n    x = Conv2D(1, kernel_size=1, activation=\"sigmoid\")(x)\n\n    x = Reshape((IMAGE_HEIGHT, IMAGE_HEIGHT))(x)\n\n    return Model(inputs=model.input, outputs=x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_model(trainable=False)\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import reduce_sum\nfrom tensorflow.keras.backend import epsilon\n\ndef dice_coefficient(y_true, y_pred):\n  numerator = 2 * reduce_sum(y_true * y_pred)\n  denominator = reduce_sum(y_true + y_pred)\n\n  return numerator / (denominator + epsilon())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.losses import binary_crossentropy\nfrom tensorflow.keras.backend import log, epsilon\ndef loss(y_true, y_pred):\n    return binary_crossentropy(y_true, y_pred) - log(dice_coefficient(y_true, y_pred) + epsilon())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss=loss,\n              metrics=[dice_coefficient])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\ncheckpoint = ModelCheckpoint(\"model-{loss:.2f}.h5\", monitor=\"loss\", verbose=1, save_best_only=True,\n                             save_weights_only=True, mode=\"min\",  save_freq = 1)\nstop = EarlyStopping(monitor=\"loss\", patience=5, mode=\"min\")\nreduce_lr = ReduceLROnPlateau(monitor=\"loss\", factor=0.2, patience=5, min_lr=1e-6, verbose=1, mode=\"min\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#batch_size=1\n#epochs=1\n\n#model.fit(\n#  X_train, masks, batch_size=batch_size, epochs=epochs, callbacks=[checkpoint,reduce_lr,stop]\n#)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}