{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"from glob import glob \nimport numpy as np\nimport pandas as pd\nimport keras,cv2,os\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, BatchNormalization, Activation\nfrom keras.layers import Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop, Adam\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.set()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = \"../input/histopathologic-cancer-detection/\" #adapt this path, when running locally\n#path = \"/histopathologic-cancer-detection/\"\ntrain_path = path + 'train/'\ntest_path = path + 'test/'\n\nprint(os.path.join(train_path,'*.tif'))\n#将tif的图片的路径与系统默认的路径合并在一起\ntiff_path=glob(os.path.join(train_path,'*.tif'))\n#glob函数的作用是查找该路径下符合.tif的文件\n\ntiff_path=pd.DataFrame({'path':tiff_path})\ntiff_path['id']= [x.split('/')[4].split('.')[0] for x in tiff_path.path] \n#利用列表表达式在path中提取出id\nprint(tiff_path.head())\ndata=pd.read_csv(path+'train_labels.csv')\n\ndf_data=tiff_path.merge(data,on='id')\ndf_data.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#异常图像\ndf_data = df_data[df_data['id'] != 'dd6dfed324f9fcb6f93f46f32fc800f2ec196be2']\ndf_data = df_data[df_data['id'] != '9369c7278ec8bcc6c880d99194de09fc2bd4efbe']\ndf_data = df_data[df_data['id'] !='0883a7e019a80bac0d0e61e21320f7fd30b37a46']\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"#a=cv2.imread(df['path'].iloc[0])\n#plt.imshow(a)\nfig=plt.figure(figsize=(5,4),dpi=150)\nfor i in range(12):\n    ax=fig.add_subplot(3,4,i+1,xticks=[],yticks=[])\n    img=cv2.imread(df_data['path'].iloc[i])\n    #读取图像的路径\n    plt.imshow(img)\n    plt.subplots_adjust(hspace=0.5)\n    ax.set_title('Label:'+str(df_data['label'][i]))\nplt.savefig('picture.png',dpi=300)    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split as ts\nfrom sklearn.utils import shuffle\n\nSAMPLE_SIZE = 60000 # load 80k negative examples\n\n# take a random sample of class 0 with size equal to num samples in class 1\ndf_0 = df_data[df_data['label'] == 0].sample(SAMPLE_SIZE, random_state = 1234)\n# filter out class 1\ndf_1 = df_data[df_data['label'] == 1].sample(SAMPLE_SIZE, random_state = 1234)\n\n# concat the dataframes\n\ndf_data = shuffle(pd.concat([df_0, df_1], axis=0).reset_index(drop=True))\n#pd.concat([df_0,df_1],axis=0).reset_index()\n#reset_index：drop是删除新的索引列\n\ny = df_data['label']\ndf_train, df_test = ts(df_data, test_size=0.1, random_state=101, stratify=y)\n#按照label进行分成抽样\n\ndf_test,df_val=ts(df_test,test_size=0.1,random_state=101)\n\n\ntrain_path = 'base_dir/train'\nvalid_path = 'base_dir/valid'\ntest_path='base_dir/test'\n#建立新的路径\n\n\nfor fold in [train_path, valid_path,test_path]:\n    for i in ['0','1']:\n        os.makedirs(os.path.join(fold,i))\n#将训练集和训练集分别建立新的文件，并且生成两种不同类别的子文件\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('训练集shape',df_train.shape)\nprint('验证集shape',df_val.shape)\nprint('测试集shape',df_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir('base_dir'))\nprint(os.listdir(train_path))\nprint(os.listdir(valid_path))\nprint(os.listdir(test_path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data.set_index('id',inplace=True)\n#s=df_data.loc['99dbe7a2b1341eeab4dd5e89578099be96e5800a','label']\n#s","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\n#导入copy模块\nfor image in df_train['id'].values:\n    # the id in the csv file does not have the .tif extension therefore we add it here\n    fname = image + '.tif'\n    if df_data.loc[image,'label']==0:\n        label='0'\n    else:\n        label='1'\n   #将数据分别分到0和1的文件夹里面     \n    src = os.path.join('../input/histopathologic-cancer-detection/train', fname)\n    dst = os.path.join(train_path,label,fname)\n    shutil.copyfile(src, dst)\n\n    \nfor image in df_val['id'].values:\n    fname = image + '.tif'\n    if df_data.loc[image,'label']==0:\n        label='0'\n    else:\n        label='1'\n    src = os.path.join('../input/histopathologic-cancer-detection/train', fname)\n    dst = os.path.join(valid_path,label,fname)\n    shutil.copyfile(src, dst)\n\n    \nfor image in df_test['id'].values:\n    fname = image + '.tif'\n    if df_data.loc[image,'label']==0:\n        label='0'\n    else:\n        label='1'\n    src = os.path.join('../input/histopathologic-cancer-detection/train', fname)\n    dst = os.path.join(test_path,label,fname)\n    shutil.copyfile(src, dst)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(os.listdir('base_dir/train/0')))\nprint(len(os.listdir('base_dir/valid/1')))\nprint(len(os.listdir('base_dir/test/0')))\nprint(os.listdir('base_dir/test'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#500张样本做色彩特征分析\nn=500\ndef read_img(n,df):\n    x=np.zeros([n,96,96,3],dtype=np.uint8)\n    for i in range(n):\n        x[i]=cv2.imread(df['path'].iloc[i])\n    return x\n\nx_nor=read_img(n,df_0)\nx_abnor=read_img(n,df_1)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#观测每个rgb通道中不同色彩的分布情况\n\nbright_bins = 256 #表示的是亮度，0是最暗即是黑色，256是最亮即是白色\nfig,axs = plt.subplots(4,2,sharey=True,figsize=(8,8),dpi=150)\n\n#RGB channels\n#x_nor[:,:,:,0] 想象该物体是正方体，拥有三个参数，这个三个参数是颜色参数。RGB的提取在最后的一个参数\n#array.flatten展开为一维数据\naxs[0,0].hist(x_nor[:,:,:,0].flatten(),bins=bright_bins,density=True)\naxs[0,1].hist(x_abnor[:,:,:,0].flatten(),density=True,bins=bright_bins)\n\n\naxs[1,0].hist(x_nor[:,:,:,1].flatten(),density=True,bins=bright_bins)\naxs[1,1].hist(x_abnor[:,:,:,1].flatten(),density=True,bins=bright_bins)\n\naxs[2,0].hist(x_nor[:,:,:,2].flatten(),density=True,bins=bright_bins)\naxs[2,1].hist(x_abnor[:,:,:,2].flatten(),density=True,bins=bright_bins)\n\n\naxs[3,0].hist(x_nor[:,:,:].flatten(),density=True,bins=bright_bins)\naxs[3,1].hist(x_abnor[:,:,:].flatten(),density=True,bins=bright_bins)\n\naxs[0,0].set_ylabel(\"Red\",horizontalalignment='left',labelpad=15,fontsize=12)\naxs[1,0].set_ylabel(\"Green\",horizontalalignment='left',labelpad=15,fontsize=12)\naxs[2,0].set_ylabel(\"Blue\",horizontalalignment='left',labelpad=15,fontsize=12)\naxs[3,0].set_ylabel(\"RGB\",horizontalalignment='left',labelpad=15,fontsize=12)\n\naxs[0,0].set_title('normal')\naxs[0,1].set_title('abnormal')\n\nplt.savefig('channel.png',dpi=300)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"将所有图像的RGB通道分离，可以看出normal和abnormal的图像在颜色的亮度方面有所不同。\n在R色彩的通道上可以得知红色channel上的normal的亮度较高，230-250之间；异常值的亮度集中在200之间。\nB,G channel的分析类似如此，至于为什么会有部分的图片的亮度到255，可能是实验人员在制作切片时留下的白光斑。"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,axs=plt.subplots(1,2,figsize=(8,2),dpi=150)\naxs[0].hist(np.mean(x_nor,axis=(1,2,3)),bins=64,density=True)\naxs[1].hist(np.mean(x_abnor,axis=(1,2,3)),bins=64,density=True)\naxs[0].set_title('normal')\naxs[1].set_title('abnormal')\n# #三个不同维度下平均亮度分布有所不同\nplt.savefig('averange_bright.png',dpi=300)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ndel x_nor\ndel x_abnor\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\nIMAGE_SIZE = 96\n\nnum_train_samples = len(df_train)\nnum_val_samples = len(df_val)\nnum_test_samples = len(df_test)\n\ntrain_batch_size = 32\nval_batch_size = 32\n#test_batch_size =32\n\ntrain_steps = np.ceil(num_train_samples / train_batch_size)\nval_steps = np.ceil(num_val_samples / val_batch_size)\n#test_steps=np.ceil(num_test_samples/test_batch_size)\n\nprint('train_steps',train_steps)\nprint('val_steps',val_steps)\n#print('test_steps',test_steps)\n\ndatagen = ImageDataGenerator(rescale=1/255,\n                             #rescale：缩放因子有助于模型的收敛\n                            horizontal_flip=True,\n                            vertical_flip=True)\n\ntrain_gen = datagen.flow_from_directory(directory=train_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=train_batch_size,\n                                        class_mode='binary')\n\nval_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=val_batch_size,\n                                        class_mode='binary')\n\ntest_gen = datagen.flow_from_directory(test_path,\n                                       batch_size=1,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        class_mode='binary',shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#设置模型训练的参数\nkernel_size=(3,3)\n#内核\npool_size=(2,2)\n#池化层\nfirst_filters=32\n#第一层32个过滤器\nsecond_filters=64\n#第二层64个过滤器\nthird_filters=128\n\ndropout_conv=0.3\n#卷积正则化使用，保持神经元输出的可能性\ndropout_dense=0.3\n#全连接网络的正则化使用，保持神经元的输出的可能性\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=Sequential()\n\n#第一层卷积\nmodel.add(Conv2D(first_filters,kernel_size,input_shape=(96,96,3),activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(first_filters,kernel_size,use_bias=False,activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool2D(pool_size=pool_size))\nmodel.add(Dropout(dropout_conv))\n\n#第二层卷积\nmodel.add(Conv2D(second_filters,kernel_size,use_bias=False,activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(second_filters,kernel_size,use_bias=False,activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool2D(pool_size=pool_size))\nmodel.add(Dropout(dropout_conv))\n\n#第三层卷积\nmodel.add(Conv2D(third_filters,kernel_size,use_bias=False,activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(third_filters,kernel_size,use_bias=False,activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool2D(pool_size=pool_size))\nmodel.add(Dropout(dropout_conv))\n\n#两层全链接网络\nmodel.add(Flatten())\nmodel.add(Dense(256,use_bias=False,activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(dropout_dense))\nmodel.add(Dense(1,activation='sigmoid'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#模型编译\nmodel.compile(Adam(0.001),\n              loss='binary_crossentropy',\n             metrics=['accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau\nearlystopper = EarlyStopping(patience=3,restore_best_weights=True)\n#monitor:val_loss,  patience:如果连续两个epoch,val_loss都没有下降即停止训练 restore_best_weigths:停止后保存最佳权重=True\n\nreducel = ReduceLROnPlateau(monitor='val_loss', patience=2, verbose=1)\n#减少学习速率函数\n# #参数有：keras.callbacks.ReduceLROnPlateau(monitor='val_loss',\n#                                             factor=0.1, 每次减少的学习速率的因子\n#                                             patience=10, \n#                                             verbose=0, \n#                                            mode='auto', \n#                                         epsilon=0.0001, 阈值，用来确定是都进入检测值的平原区\n#                                        cooldown=0, 学习速率减少后，会经过多少个cooldown个epoch才重新进行操作\n#                                        min_lr=0)  学习率的下限\n\nhistory=model.fit_generator(train_gen, steps_per_epoch=train_steps, \n                    validation_data=val_gen,\n                    validation_steps=val_steps,\n                    epochs=13,\n                   callbacks=[reducel, earlystopper])\n#steps_per_epoch:当生成器返回steps_per_epoch次数据时计一个epoch，执行下一个epoch\n#validation_steps:指定验证集的生成器返回次数\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acc=history.history['accuracy']\nloss=history.history['loss']\nval_loss=history.history['val_loss']\nval_accuracy=history.history['val_accuracy']\n\n\nsns.lineplot(x=np.arange(1,len(acc)+1),y=acc,marker='o')\nsns.lineplot(x=np.arange(1,len(acc)+1),y=val_accuracy,marker='o')\nplt.title('accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend(['train_a','val_a'])\n\nplt.savefig('acc.png',dpi=100)\nplt.show()\n\nsns.lineplot(x=np.arange(1,len(acc)+1),y=loss,marker='o')\nsns.lineplot(x=np.arange(1,len(acc)+1),y=val_loss,marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('loss')\nplt.legend(['train_loss','val_loss'])\nplt.savefig('loss.png',dpi=300)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils import plot_model\nplot_model(model, to_file='model.png',dpi=200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc, roc_auc_score\nimport matplotlib.pyplot as plt\n\n# make a prediction\ny_pred_keras = model.predict_generator(test_gen, \n                                       steps=len(df_test),\n                                       verbose=1)\nfpr_keras, tpr_keras, thresholds_keras = roc_curve(test_gen.classes, y_pred_keras)\nauc_keras = auc(fpr_keras, tpr_keras)\nauc_keras\n#y_pred_keras.shape\n#test_gen.classes.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(1)\nplt.plot([0, 1], [0, 1], 'k--')\nplt.plot(fpr_keras, tpr_keras, label='area = {:.3f}'.format(auc_keras))\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='best')\n\nplt.savefig('roc.png',dpi=300)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_keras[y_pred_keras>0.5]=1\ny_pred_keras[y_pred_keras<=0.5]=0\na=y_pred_keras.flatten()\npd.Series(a).value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom sklearn import metrics as ms\n\n\nprecision=ms.precision_score(test_gen.classes,a)*100\nprint('precison:%.2f%%'%precision)\nrecall=ms.recall_score(test_gen.classes,a)*100 \nprint('recall:%.2f%%'%recall)\n\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}