{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **Imports**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os,shutil\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom PIL import Image\nimport pydicom as dicom\nfrom skimage.transform import resize\nimport cv2\nimport seaborn as sns\nsns.set_style('darkgrid')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"os.listdir('/kaggle/input/rsna-pneumonia-detection-challenge')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Reading The Metadata**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df=pd.read_csv('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['path']='/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/'+df['patientId'].astype(str)+'.dcm'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['path'][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"negative=df[df['Target']==0]\nprint(len(negative))\nnegative.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"positive=df[df['Target']==1]\nunique_positive=positive[['path','patientId']]\npath=unique_positive['path'].unique()\npatientId=unique_positive['patientId'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_positive=pd.DataFrame({'path':path,'patientId':patientId})\nlen(unique_positive)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Heatmap of the abnormalities**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(30,20))\n\nheatmap=np.zeros((1024,1024))\n\nfor _,row in df.iterrows():\n    if row['Target']==1:\n        x=int(row['x'])\n        y=int(row['y'])\n        w=int(row['width'])\n        h=int(row['height'])\n        heatmap[y:y+h,x:x+w]+=1\n\nplt.axis('off')\n\nplt.imshow(heatmap,cmap='hot')\n    \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Storing jpg files**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"os.mkdir('/kaggle/working/data')\n\nos.mkdir('/kaggle/working/data/positive')\n\nos.mkdir('/kaggle/working/data/negative')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('/kaggle/working')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for _,row in tqdm(unique_positive.iterrows()):\n    img=dicom.read_file(row['path']).pixel_array\n    img=resize(img,(256,256))\n    plt.imsave('data/positive/'+row['patientId']+'.jpg',img,cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for _,row in tqdm(negative.iterrows()):\n    img=dicom.read_file(row['path']).pixel_array\n    img=resize(img,(256,256))\n    plt.imsave('data/negative/'+row['patientId']+'.jpg',img,cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Random Samples**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(30,20))\nfor j,img in enumerate(os.listdir('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images')):\n    path=os.path.join('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images',img)\n    tar=df[df['path']==path]['Target'].values[0]\n    img=dicom.read_file(path).pixel_array\n    plt.subplot(4,4,j+1)\n    plt.axis('off')\n    if tar==0:\n        plt.title('Negative')\n    else:\n        plt.title('Positive')\n        \n        s=df[df['path']==path]\n        \n        for _,row in s.iterrows():\n            x=int(row['x'])\n            y=int(row['y'])\n            w=int(row['width'])\n            h=int(row['height'])\n            cv2.rectangle(img,(x,y),(x+h,y+h),(255,255,0),5)\n    plt.imshow(img,cmap='gray')\n    if(j==15):\n        break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Training**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.applications.vgg19 import VGG19,preprocess_input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen=ImageDataGenerator(samplewise_center=True,samplewise_std_normalization=True,horizontal_flip=True,\n                          width_shift_range=0.05,rescale=1/255,fill_mode='nearest',height_shift_range=0.05,\n                           preprocessing_function=preprocess_input,validation_split=0.1,\n                          )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train=datagen.flow_from_directory('data',color_mode='rgb',batch_size=128,class_mode='binary',subset='training')\ntest=datagen.flow_from_directory('data',color_mode='rgb',batch_size=32,class_mode='binary',subset='validation')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.class_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pre_trained_model = VGG19(input_shape = (256,256,3), \n                                include_top = False, \n                                weights = 'imagenet')\n\nfor layer in pre_trained_model.layers:\n  layer.trainable = False\n\n# pre_trained_model.summary()\n\nlast_layer = pre_trained_model.get_layer('block5_pool')\nprint('last layer output shape: ', last_layer.output_shape)\nlast_output = last_layer.output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import Flatten,Dense,Dropout,BatchNormalization,LeakyReLU,GaussianDropout\n\nmodel = Flatten()(last_output)\nmodel = Dense(1024)(model)\nmodel=LeakyReLU(0.1)(model)\nmodel=Dropout(0.25)(model)\nmodel=BatchNormalization()(model)\nmodel = Dense(1024)(model)\nmodel=LeakyReLU(0.1)(model)\nmodel=Dropout(0.25)(model)\nmodel=BatchNormalization()(model)\nmodel = Dense(1, activation='sigmoid')(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Model\n\n\nfmodel = Model( pre_trained_model.input, model) \n\nfmodel.compile(optimizer = 'adam', \n              loss = 'binary_crossentropy', \n              metrics = ['accuracy'])\n# fmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping,ReduceLROnPlateau\n\n\nearly=EarlyStopping(monitor='accuracy',patience=3,mode='auto')\nreduce_lr = ReduceLROnPlateau(monitor='accuracy', factor=0.5, patience=2, verbose=1,cooldown=0, mode='auto',min_delta=0.0001, min_lr=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weight={0:1,1:3.3}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fmodel.fit(train,epochs=13,callbacks=[reduce_lr],steps_per_epoch=100,validation_data=test,class_weight=class_weight)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Evaluation**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fmodel.save('/kaggle/working/model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(30,20))\nval_acc=np.asarray(fmodel.history.history['val_accuracy'])*100\nacc=np.asarray(fmodel.history.history['accuracy'])*100\nacc=pd.DataFrame({'val_acc':val_acc,'acc':acc})\nacc.plot(figsize=(20,10),yticks=range(50,100,5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss=fmodel.history.history['loss']\nval_loss=fmodel.history.history['val_loss']\nloss=pd.DataFrame({'val_loss':val_loss,'loss':loss})\nloss.plot(figsize=(20,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y=[]\n\ntest.reset()\n\nfor i in tqdm(range(84)):\n    _,tar=test.__getitem__(i)\n    for j in tar:\n        y.append(j)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.reset()\ny_pred=fmodel.predict(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred=[]\nfor i in y_pred:\n    if i[0]>=0.5:\n        pred.append(1)\n    else:\n        pred.append(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_curve,auc,precision_recall_curve,classification_report","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(y,pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(30,20))\nfpr,tpr,_=roc_curve(y,y_pred)\narea_under_curve=auc(fpr,tpr)\nprint('The area under the curve is:',area_under_curve)\nplt.plot(fpr,tpr,'b.-')\nplt.xlabel('false positive rate')\nplt.ylabel('true positive rate')\nplt.plot(fpr,fpr,linestyle='--',color='black')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}