{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import matplotlib.patches as patches\nimport random\nfrom sklearn.utils import shuffle\nfrom tqdm import tqdm\nfrom tqdm import tqdm_notebook\n\nfrom numpy.random import seed\nseed(101)\nfrom tensorflow import set_random_seed\nset_random_seed(101)\n\nimport pandas as pd\nimport numpy as np\n\nfrom shutil import copyfile, move\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import initializers\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Activation, BatchNormalization\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.applications.vgg16 import VGG16\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.layers import BatchNormalization\n\nimport os\nimport cv2\n\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nimport itertools\nimport shutil\nimport matplotlib.pyplot as plt\n%matplotlib inline\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 读入表格"},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/train_labels.csv')\ntrain_path = '/kaggle/input/train/'\ntest_path = '/kaggle/input/test/'\n# 为flow from dataframe做准备\ndata[\"filename\"] = [item.id+\".tif\" for idx, item in data.iterrows()]\n# data[\"class\"] = [\"b_has tumor\" if item.label==1 else \"a_no tumor\" for idx, item in data.iterrows()]\ndata[\"class\"] = [\"b_has tumor\" if item.label==1 else \"a_no tumor\" for idx, item in data.iterrows()]\nbaseline_data = data[:10000]\nprint(data['label'].value_counts())\nprint(data.head())\n\n# data = data.sample(10000, random_state = 101)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"130908+89117","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 数据清洗"},{"metadata":{"trusted":true},"cell_type":"code","source":"def readImage(path):\n    bgr_img = cv2.imread(path)\n    b,g,r = cv2.split(bgr_img)\n    rgb_img = cv2.merge([r,g,b])\n    return rgb_img\n\nshuffled_data = shuffle(data)\n# path = os.path.join(train_path, idx)\n\ndark_th = 10      # 黑色图片\nbright_th = 245   # 白色图片\ntoo_dark_idx = []\ntoo_bright_idx = []\n\nx_tot = np.zeros(3)\nx2_tot = np.zeros(3)\ncounted_ones = 0\nfor i, idx in tqdm_notebook(enumerate(shuffled_data['filename']), 'computing statistics...(220025 it total)'):\n    path = os.path.join(train_path, idx)\n    imagearray = readImage(path).reshape(-1,3)\n#     imagearray = readImage(path + '.tif')\n    # is this too dark\n    if(imagearray.max() < dark_th):\n        too_dark_idx.append(idx)\n        continue # do not include in statistics\n    # is this too bright\n    if(imagearray.min() > bright_th):\n        too_bright_idx.append(idx)\n        continue # do not include in statistics\n    x_tot += imagearray.mean(axis=0)\n    x2_tot += (imagearray**2).mean(axis=0)\n    counted_ones += 1\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"channel_avr = (x_tot/counted_ones)/255\nchannel_std = (np.sqrt(x2_tot/counted_ones - channel_avr**2))/255\nprint(channel_avr,channel_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There was {0} extremely dark image'.format(len(too_dark_idx)))\nprint('and {0} extremely bright images'.format(len(too_bright_idx)))\nprint('Dark one:')\nprint(too_dark_idx)\nprint('Bright ones:')\nprint(too_bright_idx)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 训练验证集分解"},{"metadata":{},"cell_type":"markdown","source":"## baseline的数据"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n# train_df = baseline_data\n\n# #If removing outliers, uncomment the four lines below\n# print('Before removing outliers we had {0} training samples.'.format(len(train_df)))\n\n# for i in too_dark_idx:\n#     train_df =  train_df[train_df['filename'] != i]\n    \n# for j in too_bright_idx:\n#     train_df =  train_df[train_df['filename'] != j]\n\n# print('After removing outliers we have {0} training samples.'.format(len(train_df)))\n\n# train_names = train_df.id.values\n# train_labels = np.asarray(train_df['label'].values)\n\n# # split, this function returns more than we need as we only need the validation indexes for fastai\n# df_train, df_val= train_test_split(train_df, test_size=0.1, stratify=train_labels, random_state=101)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_train = df_train.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 全数据"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df = data\n\n#If removing outliers, uncomment the four lines below\nprint('Before removing outliers we had {0} training samples.'.format(len(train_df)))\n# train_df[(~train_df['id'].isin(too_dark_idx))]\n# train_df = train_df.drop(labels=too_dark_idx, axis=0)\n# train_df = train_df.drop(labels=too_bright_idx, axis=0)\n# train_df = train_df[train_df.id != too_dark_idx]\nfor i in too_dark_idx:\n    train_df =  train_df[train_df['filename'] != i]\n    \nfor j in too_bright_idx:\n    train_df =  train_df[train_df['filename'] != j]\n\nprint('After removing outliers we have {0} training samples.'.format(len(train_df)))\ntrain_df = train_df.reset_index(drop=True)\ntrain_names = train_df.id.values\ntrain_labels = np.asarray(train_df['label'].values)\n\n# split, this function returns more than we need as we only need the validation indexes for fastai\ndf_train, df_val= train_test_split(train_df, test_size=0.1, stratify=train_labels, random_state=101)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 读入数据集"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_train_samples = len(df_train)\nnum_val_samples = len(df_val)\ntrain_batch_size = 10\nval_batch_size = 10\n\n\ntrain_steps = np.ceil(num_train_samples / train_batch_size)\nval_steps = np.ceil(num_val_samples / val_batch_size)\n\ntarget_size = (96,96)\n\ntrain_datagen = ImageDataGenerator(\n    rescale=1. / 255,\n    horizontal_flip=True,\n    vertical_flip=True,\n    rotation_range=40,\n    zoom_range=0.2, \n#     width_shift_range=0.1,\n#     height_shift_range=0.1\n)\n\n# train_datagen = ImageDataGenerator(\n#         rescale=1./255\n# )\n\ntrain_generator = train_datagen.flow_from_dataframe(\n    dataframe = df_train,\n    x_col='filename',\n    y_col='class',\n    directory='../input/train/',\n    target_size=target_size,\n    batch_size=train_batch_size,\n    shuffle=True,\n    class_mode='binary')\n\n\nval_datagen = ImageDataGenerator(rescale=1. / 255)\nval_generator = val_datagen.flow_from_dataframe(\n    dataframe = df_val,\n    x_col='filename',\n    y_col='class',\n    directory='../input/train/',\n    target_size=target_size,\n    shuffle=False,\n    batch_size=val_batch_size,\n    class_mode='binary')\n\ntest_datagen = ImageDataGenerator(rescale=1. / 255)\ntest_generator = val_datagen.flow_from_dataframe(\n    dataframe = df_val,\n    x_col='filename',\n    y_col='class',\n    directory='../input/train/',\n    target_size=target_size,\n    shuffle=False,\n    batch_size=val_batch_size,\n    class_mode='binary')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_generator.class_indices","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 看下图片"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_random_samples(generator):\n    generator_size = len(generator)\n    index=random.randint(0,generator_size-1)\n    image,label = generator.__getitem__(index)\n\n    sample_number = 10\n    fig = plt.figure(figsize = (20,sample_number))\n    for i in range(0,sample_number):\n        ax = fig.add_subplot(2, 5, i+1)\n        ax.imshow(image[i])\n        if label[i]==0:\n            ax.set_title(\"has tumor\")\n        elif label[i]==1:\n            ax.set_title(\"no tumor\")\n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_random_samples(val_generator)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 创建模型"},{"metadata":{},"cell_type":"markdown","source":"## 创建VGG16"},{"metadata":{"trusted":true},"cell_type":"code","source":"dropout_dense=0.3\nIMG_SIZE = (96, 96)\nIN_SHAPE = (96,96, 3)\nbase_model = VGG16( \n    weights='imagenet',\n    include_top=False,\n    input_shape=IN_SHAPE)\nbase_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for layer in base_model.layers[:-6]:\n    layer.trainable = False\n\nfor layer in base_model.layers:\n    print(layer, layer.trainable)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VGG16_model = Sequential()\nVGG16_model.add(base_model)\nVGG16_model.add(Flatten())\nVGG16_model.add(Dense(256, use_bias=False))\nVGG16_model.add(BatchNormalization())\nVGG16_model.add(Activation(\"relu\"))\nVGG16_model.add(Dropout(dropout_dense))\nVGG16_model.add(Dense(1, activation = \"sigmoid\"))\nVGG16_model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 创建Resnet50"},{"metadata":{"trusted":true},"cell_type":"code","source":"# dropout_dense=0.3\n# IMG_SIZE = (96, 96)\n# IN_SHAPE = (96,96, 3)\n# base_model = ResNet50(\n#     weights='imagenet',\n#     include_top=False,\n#     input_shape=IN_SHAPE\n# )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# base_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for layer in base_model.layers[:-6]:\n#     layer.trainable = False\n\n# for layer in base_model.layers:\n#     print(layer, layer.trainable)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# resnet50_model = Sequential()\n# resnet50_model.add(base_model)\n# resnet50_model.add(Flatten())\n# resnet50_model.add(Dense(256, use_bias=False))\n# resnet50_model.add(BatchNormalization())\n# resnet50_model.add(Activation(\"relu\"))\n# resnet50_model.add(Dropout(dropout_dense))\n# resnet50_model.add(Dense(1, activation = \"sigmoid\"))\n# resnet50_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# resnet50_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(val_generator.class_indices)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('/kaggle/working')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 生成resnet50的模型"},{"metadata":{"trusted":true},"cell_type":"code","source":"# resnet50_model.compile(Adam(lr=0.0001), loss='binary_crossentropy', \n#               metrics=['accuracy'])\n\n# sgd = keras.optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)\n# model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# filepath = \"CNN3_model.h5\"\n# filepath = \"CNN6_model.h5\"\n# filepath = \"CNN9_model.h5\"\n# filepath = \"VGG16_model.h5\"\n# filepath = \"ResNet50_model.h5\"\n# checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, \n#                              save_best_only=True, mode='max')\n\n# reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=2, \n#                                    verbose=1, mode='max', min_lr=0.00001)\n                              \n                              \n# callbacks_list = [checkpoint, reduce_lr]\n\n# history = resnet50_model.fit_generator(train_generator, steps_per_epoch=train_steps, \n#                     validation_data=val_generator,\n#                     validation_steps=val_steps,\n#                     epochs=10, verbose=1,\n#                    callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# resnet50_model.load_weights('ResNet50_model.h5')\n\n# val_loss, val_acc = \\\n# resnet50_model.evaluate_generator(test_generator, \n#                         steps=len(df_val))\n\n# print('val_loss:', val_loss)\n# print('val_acc:', val_acc)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 准确率画图"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plt.plot(history.history['acc'])\n# plt.plot(history.history['val_acc'])\n# plt.title('Accuracy over epochs')\n# plt.ylabel('Acc')\n# plt.xlabel('Epoch')\n# plt.legend(['Train', 'Validation'], loc='best')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### loss画图"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plt.plot(history.history['loss'])\n# plt.plot(history.history['val_loss'])\n# plt.title('Loss over epochs')\n# plt.ylabel('Loss')\n# plt.xlabel('Epoch')\n# plt.legend(['Train', 'Validation'], loc='best')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 生成VGG16的模型"},{"metadata":{"trusted":true},"cell_type":"code","source":"VGG16_model.compile(Adam(lr=0.0001), loss='binary_crossentropy', \n              metrics=['accuracy'])\n\n# sgd = keras.optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)\n# model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filepath = \"VGG16_model.h5\"\n\ncheckpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, \n                             save_best_only=True, mode='max')\n\nreduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=2, \n                                   verbose=1, mode='max', min_lr=0.00001)\n                              \n                              \ncallbacks_list = [checkpoint, reduce_lr]\n\nVGG16_history = VGG16_model.fit_generator(train_generator, steps_per_epoch=train_steps, \n                    validation_data=val_generator,\n                    validation_steps=val_steps,\n                    epochs=60, verbose=1,\n                   callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VGG16_model.load_weights('VGG16_model.h5')\n\nval_loss, val_acc = \\\nVGG16_model.evaluate_generator(test_generator, \n                        steps=len(df_val))\n\nprint('val_loss:', val_loss)\nprint('val_acc:', val_acc)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 准确率画图"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(VGG16_history.history['acc'])\nplt.plot(VGG16_history.history['val_acc'])\nplt.title('Accuracy over epochs')\nplt.ylabel('Acc')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### loss画图"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(VGG16_history.history['loss'])\nplt.plot(VGG16_history.history['val_loss'])\nplt.title('Loss over epochs')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 预测"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = VGG16_model.predict_generator(test_generator, steps=len(df_val), verbose=1)\npredictions.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 查看不同类的索引\ntest_generator.class_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_preds = pd.DataFrame(predictions, columns=['b_has tumor'])\ndf_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_generator.classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_true = test_generator.classes\ny_pred = df_preds['b_has tumor']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# RUC score"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc\n# 概率\nprobs = np.exp(y_pred[:])\n# 计算ROC曲线\nfpr, tpr, thresholds = roc_curve(y_true, probs, pos_label=1)\n\n# 计算ROC面积\nroc_auc = auc(fpr, tpr)\nprint('ROC area is {0}'.format(roc_auc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure()\nplt.plot(fpr, tpr, color='darkorange', label='ROC curve (area = %0.2f)' % roc_auc)\nplt.plot([0, 1], [0, 1], color='navy', linestyle='--')\nplt.xlim([-0.01, 1.0])\nplt.ylim([0.0, 1.01])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver operating characteristic')\nplt.legend(loc=\"lower right\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# confusion matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    画出混淆矩阵\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = predictions.flatten()\nindex = 0\n\nfor i in range(len(predictions)):\n    if predictions[i]>=0.5:\n        predictions[i]=1\n    else:\n        predictions[i]=0\npredictions = predictions.astype(int)\n\ntest_labels = test_generator.classes\ntest_labels = np.array(test_labels)\ncm = confusion_matrix(test_labels, predictions)\ntest_generator.class_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 定义类别的索引\ncm_plot_labels = ['a_no tumor', 'b_has tumor']\n\nplot_confusion_matrix(cm, cm_plot_labels, title='Confusion Matrix')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 分类报告"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report\n\n# Generate a classification report\n\n# For this to work we need y_pred as binary labels not as probabilities\ny_pred_binary = predictions\n\nreport = classification_report(y_true, y_pred_binary, target_names=cm_plot_labels)\n\nprint(report)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 提交"},{"metadata":{"trusted":true},"cell_type":"code","source":"src=\"../input/test\"\n\ntest_folder=\"../test_folder\"\ndst = test_folder+\"/test\"\nos.mkdir(test_folder)\nos.mkdir(dst)\n\nfile_list =  os.listdir(src)\nwith tqdm(total=len(file_list)) as pbar:\n    for filename in file_list:\n        pbar.update(1)\n        copyfile(src+\"/\"+filename,dst+\"/\"+filename)\n        \ntest_datagen = ImageDataGenerator(\n    rescale=1. / 255)\n\ntest_generator = test_datagen.flow_from_directory(\n    directory=test_folder,\n    target_size=target_size,\n    batch_size=1,\n    shuffle=False,\n    class_mode='binary'\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = VGG16_model.predict_generator(test_generator,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"csv_file = open(\"sample_submission.csv\",\"w\")\ncsv_file.write(\"id,label\\n\")\nfor filename, prediction in zip(test_generator.filenames,pred):\n    name = filename.split(\"/\")[1].replace(\".tif\",\"\")\n    csv_file.write(str(name)+\",\"+str(prediction[0])+\"\\n\")\ncsv_file.close()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}