{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from numpy.random import seed\nseed(101)\nfrom tensorflow import set_random_seed\nset_random_seed(101)\n\nimport pandas as pd\nimport numpy as np\n\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Activation\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.optimizers import Adam\n\nimport os\nimport cv2\nimport cv2 as cv\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nimport itertools\nimport shutil\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"IMAGE_SIZE = 96\nIMAGE_CHANNELS = 3\n\nSAMPLE_SIZE = 70000 # the number of images we use from each of the two classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../input')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data = pd.read_csv('../input/train_labels.csv')\ndf_data[df_data['id'] != 'dd6dfed324f9fcb6f93f46f32fc800f2ec196be2']\ndf_data[df_data['id'] != '9369c7278ec8bcc6c880d99194de09fc2bd4efbe']\nprint(df_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_0 = df_data[df_data['label'] == 0].sample(SAMPLE_SIZE, random_state = 101)\ndf_1 = df_data[df_data['label'] == 1].sample(SAMPLE_SIZE, random_state = 101)\ndf_data = pd.concat([df_0, df_1], axis=0).reset_index(drop=True)\ndf_data = shuffle(df_data)\ndf_data['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12,6))\nfor i in range(10):\n    name = df_data.iloc[i,0]\n    im = cv.imread(\"../input/train/\" + name + \".tif\")\n    im = cv.cvtColor(im, cv.COLOR_BGR2RGB)\n    plt.subplot(2,5,i+1)\n    plt.imshow(im)\n    plt.title(\"label \" + str(df_data.loc[df_data['id']==name, 'label'].values[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#对灰度图作均衡化\nim2 = cv.imread(\"../input/train/c8402ec2db5ab23baaaaebab8580979e40dc6586.tif\",0)\nplt.figure(figsize=(30,20))\nplt.subplot(231)\nplt.imshow(im2,cmap='gray')\nplt.title('origin image',fontsize=30)\nplt.subplot(234)\nhist = cv.calcHist([im2],[0],None,[256],[0,255])\nplt.plot(hist,'r')\n#直方图均衡化\nplt.subplot(232)\nplt.title('equalizeHist',fontsize=30)\nim2_eq = cv.equalizeHist(im2)\nplt.imshow(im2_eq, cmap='gray')\nplt.subplot(235)\nhist = cv.calcHist([im2_eq],[0],None,[256],[0,255])\nplt.plot(hist,'r')\n#自适应直方图均衡化\nclahe = cv.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\nim2_ad = clahe.apply(im2)\nplt.subplot(233)\nplt.title('CLAHE',fontsize=30)\nplt.imshow(im2_ad, cmap='gray')\nplt.subplot(236)\nhist = cv.calcHist([im2_ad],[0],None,[256],[0,255])\nplt.plot(hist,'r')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def change_img(im):\n    clahe = cv.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n    img_lab = cv.cvtColor(im, cv.COLOR_BGR2Lab)\n    l, a, b = cv.split(img_lab)\n    img_l = clahe.apply(l)\n    img_clahe = cv.merge([img_l, a, b])\n    img_clahe = cv.cvtColor(img_clahe, cv.COLOR_Lab2BGR)\n    return img_clahe","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = df_data['label']\n\ndf_train, df_val = train_test_split(df_data, test_size=0.10, random_state=101, stratify=y)\n\nprint(df_train.shape)\nprint(df_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_dir = 'base_dir'\nos.mkdir(base_dir)\ntrain_dir = os.path.join(base_dir, 'train_dir')\nos.mkdir(train_dir)\n\n\nval_dir = os.path.join(base_dir, 'val_dir')\nos.mkdir(val_dir)\n\nno_tumor_tissue = os.path.join(train_dir, 'a_no_tumor_tissue')\nos.mkdir(no_tumor_tissue)\nhas_tumor_tissue = os.path.join(train_dir, 'b_has_tumor_tissue')\nos.mkdir(has_tumor_tissue)\n\nno_tumor_tissue = os.path.join(val_dir, 'a_no_tumor_tissue')\nos.mkdir(no_tumor_tissue)\nhas_tumor_tissue = os.path.join(val_dir, 'b_has_tumor_tissue')\nos.mkdir(has_tumor_tissue)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('base_dir/train_dir')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data.set_index('id', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_list = list(df_train['id'])\nval_list = list(df_val['id'])\n\n\n\nfor image in train_list:\n    fname = image + '.tif'\n    target = df_data.loc[image,'label']\n    if target == 0:\n        label = 'a_no_tumor_tissue'\n    if target == 1:\n        label = 'b_has_tumor_tissue'\n    src = os.path.join('../input/train', fname)\n    dst = os.path.join(train_dir, label, fname)\n    shutil.copyfile(src, dst)\n    im = cv.imread(dst)\n    cv.imwrite(dst, change_img(im))\n\n\nfor image in val_list:\n    fname = image + '.tif'\n    target = df_data.loc[image,'label']\n    \n    if target == 0:\n        label = 'a_no_tumor_tissue'\n    if target == 1:\n        label = 'b_has_tumor_tissue'\n    \n    src = os.path.join('../input/train', fname)\n    dst = os.path.join(val_dir, label, fname)\n    shutil.copyfile(src, dst)\n    im = cv.imread(dst)\n    cv.imwrite(dst, change_img(im))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(os.listdir('base_dir/train_dir/a_no_tumor_tissue')))\nprint(len(os.listdir('base_dir/train_dir/b_has_tumor_tissue')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = 'base_dir/train_dir'\nvalid_path = 'base_dir/val_dir'\ntest_path = '../input/test'\n\nnum_train_samples = len(df_train)\nnum_val_samples = len(df_val)\ntrain_batch_size = 10\nval_batch_size = 10\n\n\ntrain_steps = np.ceil(num_train_samples / train_batch_size)\nval_steps = np.ceil(num_val_samples / val_batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(rescale=1.0/255)\n\ntrain_gen = datagen.flow_from_directory(train_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=train_batch_size,\n                                        class_mode='categorical')\n\nval_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=val_batch_size,\n                                        class_mode='categorical')\n\ntest_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=1,\n                                        class_mode='categorical',\n                                        shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kernel_size = (3,3)\npool_size= (2,2)\nfirst_filters = 32\nsecond_filters = 64\nthird_filters = 128\n\ndropout_conv = 0.3\ndropout_dense = 0.3\n\n\nmodel = Sequential()\nmodel.add(Conv2D(first_filters, kernel_size, activation = 'relu', input_shape = (96, 96, 3)))\nmodel.add(Conv2D(first_filters, kernel_size, activation = 'relu'))\nmodel.add(Conv2D(first_filters, kernel_size, activation = 'relu'))\nmodel.add(MaxPooling2D(pool_size = pool_size)) \nmodel.add(Dropout(dropout_conv))\n\nmodel.add(Conv2D(second_filters, kernel_size, activation ='relu'))\nmodel.add(Conv2D(second_filters, kernel_size, activation ='relu'))\nmodel.add(Conv2D(second_filters, kernel_size, activation ='relu'))\nmodel.add(MaxPooling2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\nmodel.add(Conv2D(third_filters, kernel_size, activation ='relu'))\nmodel.add(Conv2D(third_filters, kernel_size, activation ='relu'))\nmodel.add(Conv2D(third_filters, kernel_size, activation ='relu'))\nmodel.add(MaxPooling2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(dropout_dense))\nmodel.add(Dense(2, activation = \"softmax\"))\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(Adam(lr=0.0001), loss='binary_crossentropy', \n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filepath = \"model.h5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, \n                             save_best_only=True, mode='max')\n\nreduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=2, \n                                   verbose=1, mode='max', min_lr=0.00001)\n                              \n                              \ncallbacks_list = [checkpoint, reduce_lr]\n\nhistory = model.fit_generator(train_gen, steps_per_epoch=train_steps, \n                    validation_data=val_gen,\n                    validation_steps=val_steps,\n                    epochs=8, verbose=1,\n                   callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('model.h5')\n\nval_loss, val_acc = \\\nmodel.evaluate_generator(test_gen, \n                        steps=len(df_val))\n\nprint('val_loss:', val_loss)\nprint('val_acc:', val_acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nacc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(1, len(acc) + 1)\n\nplt.plot(epochs, loss, 'bo', label='Training loss')\nplt.plot(epochs, val_loss, 'b', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\nplt.figure()\n\nplt.plot(epochs, acc, 'bo', label='Training acc')\nplt.plot(epochs, val_acc, 'b', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.figure()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict_generator(test_gen, steps=len(df_val), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_preds = pd.DataFrame(predictions, columns=['no_tumor_tissue', 'has_tumor_tissue'])\n\ndf_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_true = test_gen.classes\ny_pred = df_preds['has_tumor_tissue']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\nroc_auc_score(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shutil.rmtree('base_dir')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dir = 'test_dir'\nos.mkdir(test_dir)\n    \ntest_images = os.path.join(test_dir, 'test_images')\nos.mkdir(test_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('test_dir')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_list = os.listdir('../input/test')\n\nfor image in test_list:\n    fname = image\n    src = os.path.join('../input/test', fname)\n    dst = os.path.join(test_images, fname)\n    shutil.copyfile(src, dst)\n    im = cv.imread(dst)\n    cv.imwrite(dst, change_img(im))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(os.listdir('test_dir/test_images'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_path ='test_dir'\ntest_gen = datagen.flow_from_directory(test_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=1,\n                                        class_mode='categorical',\n                                        shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_test_images = 57458\nmodel.load_weights('model.h5')\n\npredictions = model.predict_generator(test_gen, steps=num_test_images, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_preds = pd.DataFrame(predictions, columns=['no_tumor_tissue', 'has_tumor_tissue'])\n\ndf_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_filenames = test_gen.filenames\ndf_preds['file_names'] = test_filenames\ndf_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_id(x):\n    \n    # split into a list\n    a = x.split('/')\n    # split into a list\n    b = a[1].split('.')\n    extracted_id = b[0]\n    \n    return extracted_id\n\ndf_preds['id'] = df_preds['file_names'].apply(extract_id)\n\ndf_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = df_preds['has_tumor_tissue']\nimage_id = df_preds['id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'id':image_id, \n                           'label':y_pred, \n                          }).set_index('id')\n\nsubmission.to_csv('patch_preds.csv', columns=['label']) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shutil.rmtree('test_dir')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}