{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this dataset,we have a large number of small pathology images to classify. Files are named with an image id. The train_labels.csv file provides the labels for the images in the train folder.We are predicting the labels for the images in the test folder. A positive label indicates that the center 32x32px region of a patch contains at least one pixel of tumor tissue. Tumor tissue in the outer region of the patch does not influence the label. This outer region is provided to enable fully-convolutional models that do not use zero-padding, to ensure consistent behavior when applied to a whole-slide image."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport random\nfrom sklearn.utils import shuffle\nfrom tqdm import tqdm_notebook\n#https://pythonhosted.org/keras-tqdm/\nimport math\nfrom keras_preprocessing.image import ImageDataGenerator\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import *\nfrom keras.optimizers import RMSprop,Adam\nimport shutil\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = \"../input/histopathologic-cancer-detection/train/\"\ntest_path = \"../input/histopathologic-cancer-detection/test/\"\n\nprint('Training Images:', len(os.listdir(train_path)))\nprint('Testing Images: ', len(os.listdir(test_path)))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/histopathologic-cancer-detection/train_labels.csv')\ntrain_data['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"O labels non - cancerous slides and 1 labels cancerous slides."},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = pd.read_csv(\"../input/histopathologic-cancer-detection/sample_submission.csv\", dtype=str)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"percentage of data labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data['label'].value_counts(normalize=True) * 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.id = train_data.id + '.tif'\ntest_data.id = test_data.id + '.tif'\nprint(train_data.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test_data.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLE_SIZE_0 = 18000\nSAMPLE_SIZE_1 = 12000\ndf_normal = train_data[train_data['label'] == 0].sample(SAMPLE_SIZE_0, random_state = 42)\ndf_cancer = train_data[train_data['label'] == 1].sample(SAMPLE_SIZE_1, random_state = 42)\n\n# Join the two data frame, so that both cancer and normal got in one data frame, \n## Remember to shuffle the data set , to avaoid biasing\ndf_subset = pd.concat([df_normal, df_cancer], axis=0).reset_index(drop=True)\n\n#shuffle the dataframe using shuffle \nfrom sklearn.utils import shuffle\ntrain_data_subset = shuffle(df_subset)\n\ntrain_data_subset.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_data_subset.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"####We can now split the dataset in trian and spllit####\n\n### Here we are split the data into TRAIN and VALIDATION ###\nfrom sklearn.model_selection import train_test_split\n\ndef split_data(df_train):\n        df_train, df_valid = train_test_split(df_train, test_size=0.2, random_state=42,\n                                     stratify=df_train['label'])\n        \n        # We have to set the iindex as 'id', otherwise was giving trouble while uploadiung\n        train_data_subset.set_index('id', inplace=True)\n        \n        train_list = list(df_train['id'])\n        valid_list = list(df_valid['id'])\n        \n        return df_train, df_valid, train_list, valid_list\n#Lets split it now###\ndf_train, df_valid, train_list, valid_list = split_data(train_data_subset)\nprint('df_train_shape', df_train.shape)\nprint('df_validation_shape', df_valid.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train=df_train.astype(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_valid=df_valid.astype(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_valid.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Neural networks process inputs using small weight values, and inputs with large integer values can disrupt or slow down the learning process. we need to scale pixel values between 0 and 1"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(\n       #horizontal_flip=True,\n       #vertical_flip=True,\n       #brightness_range=[0.5, 1.5],\n       #fill_mode='reflect',                               \n        #rotation_range=15,\n        rescale=1./255)\n        #shear_range=0.2,\n        #zoom_range=0.2)\n        #validation_split=0.15)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_datagen = ImageDataGenerator(\n    rescale=1./255)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntest_datagen = ImageDataGenerator(\n       #horizontal_flip=True,\n       #vertical_flip=True,\n       #brightness_range=[0.5, 1.5],\n       #fill_mode='reflect',                               \n        #rotation_range=15,\n        rescale=1./255)\n        #shear_range=0.2,\n        #zoom_range=0.2)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"https://medium.com/@vijayabhaskar96/tutorial-on-keras-flow-from-dataframe-1fd4493d237c"},{"metadata":{},"cell_type":"markdown","source":"function flow_from_dataframe t\nallows you to input a Pandas dataframe which contains the filenames(with or without the extensions) column and a column which has the class names and directly read the images from the directory with their respective class names mapped.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_size = 24000\nva_size = 6000\nbs = 128\n\ntr_steps = math.ceil(tr_size / bs)\nva_steps = math.ceil(va_size / bs)\n\n#math.ceil() function returns the smallest integral value greater than the number. \n#If number is already integer, same number is returned.\n\ntrain_generator = train_datagen.flow_from_dataframe(\n    dataframe = df_train,\n    directory = train_path,\n    x_col = \"id\",\n    y_col = \"label\",\n    #subset = \"training\",\n    batch_size = bs,\n    seed = 1,\n    shuffle = True,\n    class_mode = \"categorical\",\n    target_size = (96,96))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_generator = validation_datagen.flow_from_dataframe(\n    dataframe = df_valid,\n    directory = train_path,\n    x_col = \"id\",\n    y_col = \"label\",\n    #subset = \"validation\",\n    batch_size = bs,\n    seed = 1,\n    shuffle = True,\n    class_mode = \"categorical\",\n    target_size = (96,96))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_generator = test_datagen.flow_from_dataframe(\n    dataframe = test_data,\n    directory = test_path,\n    x_col = \"id\",\n    y_col = None,\n    batch_size = bs,\n    seed = 1,\n    shuffle = False,\n    class_mode = None,\n    target_size = (96,96))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def training_images(seed):\n    np.random.seed(seed)\n    train_generator.reset()\n    imgs, labels = next(train_generator)\n    tr_labels = np.argmax(labels, axis=1)\n    \n    plt.figure(figsize=(12,12))\n    for i in range(16):\n        text_class = labels[i]\n        plt.subplot(4,4,i+1)\n        plt.imshow(imgs[i,:,:,:])\n        if(text_class[0] == 0):\n            plt.text(0, -5, 'Positive', color='r')\n        else:\n            plt.text(0, -5, 'Negative', color='b')\n        plt.axis('off')\n    plt.show()\n\ntraining_images(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(filters = 16, kernel_size = 3, padding = 'same', activation = 'relu', input_shape = (96, 96, 3)))\nmodel.add(Conv2D(filters = 16, kernel_size = 3, padding = 'same', activation = 'relu'))\nmodel.add(Conv2D(filters = 16, kernel_size = 3, padding = 'same', activation = 'relu'))\nmodel.add(Dropout(0.2))\nmodel.add(MaxPooling2D(pool_size = 3))\nmodel.add(BatchNormalization())\n\nmodel.add(Conv2D(filters = 32, kernel_size = 3, padding = 'same', activation = 'relu'))\nmodel.add(Conv2D(filters = 32, kernel_size = 3, padding = 'same', activation = 'relu'))\nmodel.add(Conv2D(filters = 32, kernel_size = 3, padding = 'same', activation = 'relu'))\nmodel.add(Dropout(0.2))\nmodel.add(MaxPooling2D(pool_size = 3))\nmodel.add(BatchNormalization())\n\nmodel.add(Conv2D(filters = 64, kernel_size = 3, padding = 'same', activation = 'relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = 3, padding = 'same', activation = 'relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = 3, padding = 'same', activation = 'relu'))\nmodel.add(Dropout(0.2))\nmodel.add(MaxPooling2D(pool_size = 3))\nmodel.add(BatchNormalization())\n\nmodel.add(Conv2D(filters = 128, kernel_size = 3, padding = 'same', activation = 'relu'))\nmodel.add(Conv2D(filters = 128, kernel_size = 3, padding = 'same', activation = 'relu'))\nmodel.add(Conv2D(filters = 128, kernel_size = 3, padding = 'same', activation = 'relu'))\nmodel.add(Dropout(0.3))\nmodel.add(BatchNormalization())\n\n\nmodel.add(Flatten())\nmodel.add(Dense(128, activation='relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(2, activation = 'sigmoid'))\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 30","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\noptimizer=Adam(learning_rate=0.001,beta_1=0.9,beta_2=0.999,epsilon=1e-08)\n\nmodel.compile(optimizer=optimizer,loss=['binary_crossentropy'],metrics=['accuracy'])\n\nh1 = model.fit_generator(train_generator, steps_per_epoch=tr_steps, epochs=30, validation_data=valid_generator, validation_steps=va_steps, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"markdown","source":"steps_per_epoch: Integer or None. Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. "},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=[12,6])\nplt.subplot(1,2,1)\nplt.plot(range(1,epochs+1), h1.history['accuracy'], label='Training Accuracy')\nplt.plot(range(1,epochs+1), h1.history['val_accuracy'], label='Validation Accuracy')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.subplot(1,2,2)\nplt.plot(range(1,epochs+1), h1.history['loss'], label='Training Loss')\nplt.plot(range(1,epochs+1), h1.history['val_loss'], label='Validation Loss')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\noptimizer=Adam(learning_rate=0.0001,beta_1=0.9,beta_2=0.999,epsilon=1e-08)\n\nmodel.compile(optimizer=optimizer,loss=['binary_crossentropy'],metrics=['accuracy'])\n\nh2 = model.fit_generator(train_generator, steps_per_epoch=tr_steps, epochs=15, validation_data=valid_generator, validation_steps=va_steps, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 15","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=[12,6])\nplt.subplot(1,2,1)\nplt.plot(range(1,epochs+1), h2.history['accuracy'], label='Training Accuracy')\nplt.plot(range(1,epochs+1), h2.history['val_accuracy'], label='Validation Accuracy')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.subplot(1,2,2)\nplt.plot(range(1,epochs+1), h2.history['loss'], label='Training Loss')\nplt.plot(range(1,epochs+1), h2.history['val_loss'], label='Validation Loss')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs=6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\noptimizer=Adam(learning_rate=0.00001,beta_1=0.9,beta_2=0.999,epsilon=1e-08)\n\nmodel.compile(optimizer=optimizer,loss=['binary_crossentropy'],metrics=['accuracy'])\n\nh3 = model.fit_generator(train_generator, steps_per_epoch=tr_steps, epochs=6, validation_data=valid_generator, validation_steps=va_steps, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"epochs = 6"},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 6\nplt.figure(figsize=[12,6])\nplt.subplot(1,2,1)\nplt.plot(range(1,epochs+1), h3.history['accuracy'], label='Training Accuracy')\nplt.plot(range(1,epochs+1), h3.history['val_accuracy'], label='Validation Accuracy')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.subplot(1,2,2)\nplt.plot(range(1,epochs+1), h3.history['loss'], label='Training Loss')\nplt.plot(range(1,epochs+1), h3.history['val_loss'], label='Validation Loss')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\noptimizer=Adam(learning_rate=0.000001,beta_1=0.9,beta_2=0.999,epsilon=1e-08)\n\nmodel.compile(optimizer=optimizer,loss=['binary_crossentropy'],metrics=['accuracy'])\n\nh4 = model.fit_generator(train_generator, steps_per_epoch=tr_steps, epochs=5, validation_data=valid_generator, validation_steps=va_steps, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=[12,6]) \nplt.subplot(1,2,1) plt.plot(range(1,epochs+1), h4.history['accuracy'], label='Training Accuracy') plt.plot(range(1,epochs+1), h4.history['val_accuracy'], label='Validation Accuracy') \nplt.xlabel('Epoch') \nplt.legend()\n\nplt.subplot(1,2,2) plt.plot(range(1,epochs+1), h4.history['loss'], label='Training Loss') plt.plot(range(1,epochs+1), h4.history['val_loss'], label='Validation Loss') \nplt.xlabel(' Epoch') \nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 5\nplt.figure(figsize=[12,6])\nplt.subplot(1,2,1)\nplt.plot(range(1,epochs+1), h4.history['accuracy'], label='Training Accuracy')\nplt.plot(range(1,epochs+1), h4.history['val_accuracy'], label='Validation Accuracy')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.subplot(1,2,2)\nplt.plot(range(1,epochs+1), h4.history['loss'], label='Training Loss')\nplt.plot(range(1,epochs+1), h4.history['val_loss'], label='Validation Loss')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('cnn_v01.h4')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pred = model.predict_generator(test_generator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test_pred[:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_filenames = test_generator.filenames\ntest_filenames[ :5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_filenames = [x.split(\".\")[0] for x in test_filenames]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_filenames[ :5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(test_filenames)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = list(np.argmax(test_pred, axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'id':test_filenames,\n     'label':classes\n    })\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}