{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nimport cv2\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport time\nimport sklearn\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load in the training dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = pd.read_csv('../input/train_labels.csv')\nprint(train_labels.iloc[:3,:])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explore the data, visualize the cancer and benign cell images"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Find the first 6 images without cancer cells\nbenigns = []\ni = 0\nwhile len(benigns) < 6:    \n    if train_labels.iloc[i,1] == 0:\n        benigns.append(train_labels.iloc[i,0])\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find the first 6 images with cancer cells\ncancers = []\ni = 0\nwhile len(cancers) < 6:    \n    if train_labels.iloc[i,1] == 1:\n        cancers.append(train_labels.iloc[i,0])\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize the images without cancer cells in RGB format\n# read in the first image\nfig, ax = plt.subplots(nrows=2, ncols=3)\nfig.suptitle('Benign cells images')\ni = 0\nfor row in ax:\n    for col in row:        \n        sample1 = cv2.imread('../input/train/'+ '%s' % benigns[i] +'.tif')\n        sample1 = cv2.cvtColor(sample1, cv2.COLOR_BGR2RGB)\n        img1 = Image.fromarray(sample1,'RGB')\n        col.imshow(img1)\n        i += 1\n        #col.plot(x, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize the images with cancer cells in RGB format\nfig, ax = plt.subplots(nrows=2, ncols=3)\nfig.suptitle('Cancers images')\ni = 0\nfor row in ax:\n    for col in row:        \n        sample2 = cv2.imread('../input/train/'+ '%s' % cancers[i] +'.tif')\n        sample2 = cv2.cvtColor(sample2, cv2.COLOR_BGR2RGB)\n        img2 = Image.fromarray(sample2,'RGB')\n        col.imshow(img2)\n        i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find the shape of each image\nsample2.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Explore the classes\nprint('There are ', len(train_labels[train_labels['label']==0]), ' training samples in the 0 class.')\nprint('There are ', len(train_labels[train_labels['label']==1]), ' training samples in the 1 class.')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### The training dataset is not strictly balanced. The ratio for benign samples to cancer samples is about 1.5 : 1."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Stack the RGB image together into a numpy array\n# Read in images batch by batch\ndef preprocessing(start, end):\n    file_name = train_labels.iloc[start,0]\n    sample1 = cv2.imread('../input/train/'+ '%s' % file_name +'.tif')\n    sample1 = cv2.cvtColor(sample1, cv2.COLOR_BGR2RGB)\n    X_train = sample1\n    Y_train = [[train_labels.iloc[0,1]]]\n    idx = start+1\n    for file in train_labels.iloc[start+1:end,0]:\n        img = cv2.imread('../input/train/'+ '%s' % file +'.tif')\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        X_train = np.concatenate([X_train,img])\n        Y_train.extend([[train_labels.iloc[idx,1]]])\n        idx += 1\n    X_train = np.reshape(X_train,(-1,96,96,3))\n    X_train = 2.0*(X_train / 255) - 1.0\n    return X_train, Y_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test the preprocessing function with first 10 training data\ntrain_filenames = train_labels.iloc[:,0]\nX_train,Y_train = preprocessing(0, 10)\nX_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert the binary labels into one-hot format\nfrom sklearn.preprocessing import OneHotEncoder\nenc = OneHotEncoder()\nenc.fit(np.array(Y_train))\nout = enc.transform(Y_train).toarray()\nprint(out)\nprint(train_labels.iloc[:10,1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build the simple CNN model"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Resetting graph\ntf.reset_default_graph()\n\n#Defining Placeholders\nx = tf.placeholder(tf.float32,shape=[None,96,96,3])\ny_true = tf.placeholder(tf.float32,shape=[None,2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##CONVOLUTION LAYER 1\n#Weights for layer 1\nw_1 = tf.Variable(tf.truncated_normal([11,11,3,96], stddev=0.01))\n#Bias for layer 1\nb_1 = tf.Variable(tf.constant(0.0, shape=[[11,11,3,96][3]]))\n#Applying convolution\nc_1 = tf.nn.conv2d(x, w_1,strides=[1, 2, 2, 1], padding='VALID')\n#Adding bias\nc_1 = c_1 + b_1\n#Applying RELU\nc_1 = tf.nn.relu(c_1)\nprint(c_1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##POOLING LAYER1\np_1 = tf.nn.max_pool(c_1, ksize=[1, 3, 3, 1],strides=[1, 2, 2, 1], padding='VALID')\nprint(p_1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##CONVOLUTION LAYER 2\n#Weights for layer 2\nw_2 = tf.Variable(tf.truncated_normal([5,5,96,256], stddev=0.01))\n#Bias for layer 2\nb_2 = tf.Variable(tf.constant(1.0, shape=[[5,5,96,256][3]]))\n#Applying convolution\nc_2 = tf.nn.conv2d(p_1, w_2,strides=[1, 1, 1, 1], padding='SAME')\n#Adding bias\nc_2 = c_2 + b_2\n#Applying RELU\nc_2 = tf.nn.relu(c_2)\n\nprint(c_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##POOLING LAYER2\np_2 = tf.nn.max_pool(c_2, ksize=[1, 3, 3, 1],strides=[1, 2, 2, 1], padding='VALID')\nprint(p_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##CONVOLUTION LAYER 3\n#Weights for layer 3\nw_3 = tf.Variable(tf.truncated_normal([3, 3, 256, 384], stddev=0.01))\n#Bias for layer 3\nb_3 = tf.Variable(tf.constant(0.0, shape=[[3, 3, 256, 384][3]]))\n#Applying convolution\nc_3 = tf.nn.conv2d(p_2, w_3,strides=[1, 1, 1, 1], padding='SAME')\n#Adding bias\nc_3 = c_3 + b_3\n#Applying RELU\nc_3 = tf.nn.relu(c_3)\n\nprint(c_3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##CONVOLUTION LAYER 4\n#Weights for layer 4\nw_4 = tf.Variable(tf.truncated_normal([3, 3, 384, 384], stddev=0.01))\n#Bias for layer 4\nb_4 = tf.Variable(tf.constant(0.0, shape=[[3, 3, 384, 384][3]]))\n#Applying convolution\nc_4 = tf.nn.conv2d(c_3, w_4,strides=[1, 1, 1, 1], padding='SAME')\n#Adding bias\nc_4 = c_4 + b_4\n#Applying RELU\nc_4 = tf.nn.relu(c_4)\n\nprint(c_4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##CONVOLUTION LAYER 5\n#Weights for layer 5\nw_5 = tf.Variable(tf.truncated_normal([3, 3, 384, 256], stddev=0.01))\n#Bias for layer 5\nb_5 = tf.Variable(tf.constant(0.0, shape=[[3, 3, 384, 256][3]]))\n#Applying convolution\nc_5 = tf.nn.conv2d(c_4, w_5,strides=[1, 1, 1, 1], padding='SAME')\n#Adding bias\nc_5 = c_5 + b_5\n#Applying RELU\nc_5 = tf.nn.relu(c_5)\n\nprint(c_5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##POOLING LAYER3\np_3 = tf.nn.max_pool(c_5, ksize=[1, 3, 3, 1],strides=[1, 2, 2, 1], padding='VALID')\nprint(p_3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Flattening\nflattened = tf.reshape(p_3,[-1,4*4*256])\nprint(flattened)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Fully Connected Layer 1\n#Getting input nodes in FC layer 1\ninput_size = int( flattened.get_shape()[1] )\n#Weights for FC Layer 1\nw1_fc = tf.Variable(tf.truncated_normal([input_size, 2048], stddev=0.01))\n#Bias for FC Layer 1\nb1_fc = tf.Variable( tf.constant(1.0, shape=[2048] ) )\n#Summing Matrix calculations and bias\ns_fc1 = tf.matmul(flattened, w1_fc) + b1_fc\n#Applying RELU\ns_fc1 = tf.nn.relu(s_fc1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Dropout Layer 1\nhold_prob1 = tf.placeholder(tf.float32)\ns_fc1 = tf.nn.dropout(s_fc1,keep_prob=hold_prob1)\nprint(s_fc1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Fully Connected Layer 2\n#Weights for FC Layer 2\nw2_fc = tf.Variable(tf.truncated_normal([2048, 1024], stddev=0.01))\n#Bias for FC Layer 2\nb2_fc = tf.Variable( tf.constant(1.0, shape=[1024] ) )\n#Summing Matrix calculations and bias\ns_fc2 = tf.matmul(s_fc1, w2_fc) + b2_fc\n#Applying RELU\ns_fc2 = tf.nn.relu(s_fc2)\nprint(s_fc2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Dropout Layer 2\nhold_prob2 = tf.placeholder(tf.float32)\ns_fc2 = tf.nn.dropout(s_fc2,keep_prob=hold_prob1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Fully Connected Layer 3\n#Weights for FC Layer 3\nw3_fc = tf.Variable(tf.truncated_normal([1024,2], stddev=0.01))\n#Bias for FC Layer 3b3_fc = tf.Variable( tf.constant(1.0, shape=[output_classes] ) )\nb3_fc = tf.Variable( tf.constant(1.0, shape=[2] ) )\n#Summing Matrix calculations and bias\ny_pred = tf.matmul(s_fc2, w3_fc) + b3_fc\n#Applying RELU\nprint(y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Defining loss function\ncross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_true,logits=y_pred))\n\n#Defining objective\ntrain = tf.train.AdamOptimizer(learning_rate=0.00005).minimize(cross_entropy)\n\n#Defining Accuracy\nmatches = tf.equal(tf.argmax(y_pred,1),tf.argmax(y_true,1))\nacc = tf.reduce_mean(tf.cast(matches,tf.float32))\n\n#Initializing weights\ninit = tf.global_variables_initializer()\nsaver = tf.train.Saver()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use the last 5025 training data as the validation data\nvalidate_x,validate_y = preprocessing(215000,220025)\nvalidate_y = enc.transform(validate_y).toarray()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train the model with 215000 training data"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Starting Empty lists to keep results\nacc_list = []\nauc_list = []\nloss_list = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score  \n#GPU settings\nmax_accuracy = 0.0\nepochs = 30\nBATCH_SIZE = 128\nconfig = tf.ConfigProto(allow_soft_placement=True)\nconfig.gpu_options.allow_growth = True\nconfig.gpu_options.allocator_type = 'BFC'\nwith tf.Session(config=config) as sess:\n    sess.run(init)\n    num_examples = 215000\n    for i in range(epochs):\n        for offset in range(0, num_examples, BATCH_SIZE):\n            end = offset + BATCH_SIZE            \n            batch_x, batch_y = preprocessing(offset,end)\n            batch_y = enc.transform(batch_y).toarray()\n            batch_x, batch_y = sklearn.utils.shuffle(batch_x, batch_y)\n            #Feeding step_size-amount data with 0.5 keeping probabilities on DROPOUT LAYERS\n            _,c = sess.run([train,cross_entropy],\n                           feed_dict={x:batch_x , y_true:batch_y,hold_prob1:0.5,hold_prob2:0.5})\n        #Writing for loop to calculate test statistics. GTX 1050 isn't able to calculate all test data.\n        cv_auc_list = []\n        cv_acc_list = []\n        cv_loss_list = []\n        for v in range(0,len(validate_y)-int(len(validate_y) % 128),128):\n            acc_on_cv,loss_on_cv,preds = sess.run([acc,cross_entropy,tf.nn.softmax(y_pred)],\n                                                  feed_dict={x:validate_x[v:v+128] ,y_true:validate_y[v:v+128] ,hold_prob1:1.0,hold_prob2:1.0})\n            auc_on_cv = roc_auc_score(validate_y[v:v+128],preds)\n            cv_acc_list.append(acc_on_cv)\n            cv_auc_list.append(auc_on_cv)\n            cv_loss_list.append(loss_on_cv)\n        acc_cv_ = round(np.mean(cv_acc_list),5)\n        auc_cv_ = round(np.mean(cv_auc_list),5)\n        loss_cv_ = round(np.mean(cv_loss_list),5)\n        acc_list.append(acc_cv_)\n        auc_list.append(auc_cv_)\n        loss_list.append(loss_cv_)\n        print(\"Epoch:\",i+1,\"Accuracy:\",acc_cv_,\"Loss:\",loss_cv_ ,\"AUC:\",auc_cv_)\n        if acc_cv_ > max_accuracy:  # save only highest accuracy we've achieved so far\n            max_accuracy = acc_cv_\n            saver.save(sess, './best_model_save_file')            \n            print(\"Highest accuracy seen so far. Model saved.\")            \n        else:\n            print(\"Not highest accuracy seen so far. Model not saved.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize the training process through accuracy, loss and AUC on the validation data\nf,ax=plt.subplots(3,1,figsize=(15,15))\npd.Series(acc_list).plot(kind='line',title='Accuracy on validation data',ax=ax[0])\npd.Series(auc_list).plot(kind='line',title='Loss on validation data',ax=ax[1])\npd.Series(loss_list).plot(kind='line',title='AUC on validation data',ax=ax[2])\nplt.subplots_adjust(wspace=1)\nax[0].set_title('Accuracy on validation data')\nax[1].set_title('AUC on validation data')\nax[2].set_title('Loss on validation data')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load test data and use the model to predic"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the test data\ntest_labels = pd.read_csv('../input/sample_submission.csv')\ntest_filenames = test_labels.iloc[:,0]\n# Explore the classes\nprint('There are ', len(test_labels[test_labels['label']==0]), ' test samples in the 0 class.')\nprint('There are ', len(test_labels[test_labels['label']==1]), ' test samples in the 1 class.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predict the test data with probabilities\nsoftmax_logits = tf.nn.softmax(y_pred)\n\npredictions = []\nnum_examples = test_labels.shape[0]\nwith tf.Session() as sess:\n    saver = tf.train.import_meta_graph('best_model_save_file.meta')\n    saver.restore(sess,tf.train.latest_checkpoint('./'))\n    for offset in range(0, num_examples):\n            end = offset + 1\n            file_name = test_labels.iloc[offset,0]\n            img = cv2.imread('../input/test/'+ '%s' % file_name +'.tif')\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            batch_x = np.reshape(img,(-1,96,96,3))\n            batch_x = 2.0*(batch_x / 255) - 1.0\n            pred = sess.run(softmax_logits, feed_dict={x: batch_x, hold_prob1:1.0,hold_prob2:1.0})\n            predictions.append(pred[0,1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save the prediction to the submission file\ntest_pred = test_labels.copy(deep=True)\ntest_pred['label'] = predictions\ntest_pred.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}