{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import random\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure\nfigure(figsize=(15, 12), dpi=120)\nimport seaborn as sns\nsns.set(style='whitegrid') #set seaborn plotting aesthetics\n%matplotlib inline\n\nimport pydicom as dcm\nfrom pathlib import Path\nimport os\nfrom tqdm.notebook import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Built on the work of;\n# https://www.kaggle.com/code/rahultheogre/pneumonia-detection-with-cnn-basic\n\ntrain_class = pd.read_csv('../input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv')\n\n# [patient ID, x, y, width, height, target]\n# (x, y, width, height) --> Bounding boxes for localizing Pneumonia\n# target --> Tells whether person has Pneumonia or not\ntrain_labels = pd.read_csv('../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')\n\ntrain_path = Path('../input/rsna-pneumonia-detection-challenge/stage_2_train_images')\n\ntest_path = Path('../input/rsna-pneumonia-detection-challenge/stage_2_test_images')\n\ntrain_meta = pd.concat([train_labels, \n                        train_class.drop(columns=['patientId'])], axis=1)\n\nbox_df = train_meta.groupby('patientId').size().reset_index(name='boxes')\n\ntrain_ds = pd.merge(train_meta, box_df, on='patientId')\n\nbox_df = box_df.groupby('boxes').size().reset_index(name='patients')\n\n\n# List of information we needs with us\nvars = ['PatientAge','PatientSex','ImagePath']\n\n\ndef process_dicom_data(df, path):\n    \n    # adding new columns to the imported DataFrame with Null values\n    for var in vars:\n        df[var] = None\n        \n    images = os.listdir(path)\n    \n    #looping through each dicom image, extract the information from it, and \n    # add it to the DataFrame\n    \n    for i, img_name in tqdm(enumerate(images)):\n        \n        imagePath = os.path.join(path,img_name)\n        img_data = dcm.read_file(imagePath)\n        \n        idx = (df['patientId']==img_data.PatientID)\n        df.loc[idx,'PatientAge'] = pd.to_numeric(img_data.PatientAge)\n        df.loc[idx,'PatientSex'] = img_data.PatientSex\n        df.loc[idx, 'ImagePath'] = str.format(imagePath)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_class.head())\nprint(train_labels.head())\nprint(train_ds.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process_dicom_data(train_ds,'../input/rsna-pneumonia-detection-challenge/stage_2_train_images')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# process_dicom_data(test_ds,'../input/rsna-pneumonia-detection-challenge/stage_2_test_images')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds.to_csv('pneumonia_ds', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **New code added**","metadata":{}},{"cell_type":"code","source":"print(train_ds.head())\n\nprint(len(set(train_ds[\"patientId\"])))\nprint(len(train_ds[\"patientId\"]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimg_path = train_ds.iloc[0][\"ImagePath\"]\ndcm_data = dcm.read_file(img_path)\nimg = dcm_data.pixel_array\nplt.imshow(img)\n\nprint(img.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Splitting the train_ds into Pneumonia (target=1) and Non-Pneumonia (target=0) dataset\n\nP_df = train_ds.loc[train_ds['Target'] == 1]\nNP_df = train_ds.loc[train_ds['Target'] == 0]\n\nprint(len(P_df), len(NP_df))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"P_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NP_df[:9000]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\n\nADJUSTED_IMAGE_SIZE = 224\n\ndef readAndReshapeImage(image):\n    img = np.array(image).astype(np.uint8)\n    res = cv2.resize(img,(ADJUSTED_IMAGE_SIZE,ADJUSTED_IMAGE_SIZE), interpolation = cv2.INTER_LINEAR)\n    return res\n\ndef populateImage(df):\n    imageList = []\n    for i in range(len(df)):\n        dcm_file = df.iloc[i][\"ImagePath\"] # Reading DCM file from Image path\n        dcm_data = dcm.read_file(dcm_file)\n        img = dcm_data.pixel_array\n        imageList.append(readAndReshapeImage(img))\n        \n    tmpImages = np.array(imageList)\n    return tmpImages\n\nP_images = populateImage(P_df[:9000])\nprint(P_images.shape)\n\nNP_images = populateImage(NP_df[:9000])\nprint(NP_images.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Storing images to numpy files**","metadata":{}},{"cell_type":"code","source":"# save numpy array as npy file\nfrom numpy import asarray\nfrom numpy import save\n\n# save to npy file\nsave('P_images.npy', P_images)\n\n# save to npy file\nsave('NP_images.npy', NP_images)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Reading images from Stored numpy files**","metadata":{}},{"cell_type":"code","source":"# load numpy array from npy file\nfrom numpy import load\n\n# load array\nP_img_data = load('P_images.npy')\nNP_img_data = load('NP_images.npy')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img1 = P_img_data[0]\nplt.imshow(img1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img2 = NP_img_data[0]\nplt.imshow(img2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Models","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.python.keras.layers import Dense, Flatten\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nimport tensorflow as tf\nfrom sklearn.metrics import accuracy_score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Making 3-channel images from 1-channel image\n# 1) Add extra dimension\n# 2) Repeat intensity in channel-1 across other remaining channels\n\ndef expand_greyscale_image_channels(grey_image):\n    grey_image_arr = np.array(grey_image)\n    grey_image_arr = np.expand_dims(grey_image_arr, -1)\n    grey_image_arr_3_channel = grey_image_arr.repeat(3, axis=-1)\n    return grey_image_arr_3_channel\n\nP_img_data3 = []\nNP_img_data3 = []\nfor grey_img in P_img_data:\n    P_img_data3.append(expand_greyscale_image_channels(grey_img)) \nfor grey_img in NP_img_data:\n    NP_img_data3.append(expand_greyscale_image_channels(grey_img)) \n\nP_img_data3 = np.array(P_img_data3)\nNP_img_data3 = np.array(NP_img_data3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Append P_img_data and NP_img_data together\nX_train = np.concatenate((P_img_data3, NP_img_data3))\n\n# Add Labels\nP_labels = np.zeros(9000)\nNP_labels = np.ones(9000)\nY_train = np.concatenate((P_labels, NP_labels))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ResNet50V2","metadata":{}},{"cell_type":"code","source":"# detect and init the TPU\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n\n# instantiate a distribution strategy\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"org_resnet50v2_model = tf.keras.applications.ResNet50V2()\norg_resnet50v2_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrained_model = tf.keras.applications.ResNet50V2(\n    include_top=False,\n    weights=\"imagenet\",\n    input_shape=(224,224,3),\n)\n\n# Freeze the layers from the pretrained model \nfor layer in pretrained_model.layers:\n    layer.trainable = False\n    \npretrained_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can't directly add new layer to inbuilt model\n# But we can create a new Sequential object\n# Add the inbuilt model which is considered as a layer\n# and then keep on adding more layers\n\nwith tpu_strategy.scope():\n    resnet50v2_model = Sequential()\n\n    resnet50v2_model.add(pretrained_model)\n    resnet50v2_model.add(Flatten())\n    resnet50v2_model.add(layers.Dense(512, activation=\"relu\"))\n    resnet50v2_model.add(layers.Dense(512, activation=\"relu\"))\n    resnet50v2_model.add(layers.Dense(256, activation=\"relu\"))\n    resnet50v2_model.add(layers.Dense(10, activation='softmax'))\n    \n    resnet50v2_model.compile(optimizer=Adam(learning_rate=0.001),loss='sparse_categorical_crossentropy',metrics=['accuracy'])\n\nresnet50v2_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[0].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = resnet50v2_model.fit(x=X_train, y=Y_train, epochs=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnet50v2_model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Accuracy on Train data\nval_loss, acc = resnet50v2_model.evaluate(X_train, Y_train, verbose=0)\nprint('Accuracy: %.3f' % acc)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}