{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Histopathic Cancer Detection (HCD)\n### Taylor Kern","metadata":{}},{"cell_type":"markdown","source":"# Import Statements","metadata":{}},{"cell_type":"markdown","source":"The following cell contains import statements ","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport pickle\nimport os\n\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import * \n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import backend as k\n\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' ","metadata":{"execution":{"iopub.status.busy":"2022-05-03T18:48:28.578961Z","iopub.execute_input":"2022-05-03T18:48:28.579639Z","iopub.status.idle":"2022-05-03T18:48:34.901835Z","shell.execute_reply.started":"2022-05-03T18:48:28.579537Z","shell.execute_reply":"2022-05-03T18:48:34.901091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def merge_history(hlist):\n    history = {}\n    for k in hlist[0].history.keys():\n        history[k] = sum([h.history[k] for h in hlist], [])\n    return history\n\ndef vis_training(h, start=1):\n    epoch_range = range(start, len(h['loss'])+1)\n    s = slice(start-1, None)\n\n    plt.figure(figsize=[14,4])\n\n    n = int(len(h.keys()) / 2)\n\n    for i in range(n):\n        k = list(h.keys())[i]\n        plt.subplot(1,n,i+1)\n        plt.plot(epoch_range, h[k][s], label='Training')\n        plt.plot(epoch_range, h['val_' + k][s], label='Validation')\n        plt.xlabel('Epoch'); plt.ylabel(k); plt.title(k)\n        plt.grid()\n        plt.legend()\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-03T18:48:34.903687Z","iopub.execute_input":"2022-05-03T18:48:34.903943Z","iopub.status.idle":"2022-05-03T18:48:34.914554Z","shell.execute_reply.started":"2022-05-03T18:48:34.903908Z","shell.execute_reply":"2022-05-03T18:48:34.913815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Training DataFrame","metadata":{}},{"cell_type":"markdown","source":"When we are loading the training dataframe, we are displaying the images. We are also removing the file extension .tif","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/histopathologic-cancer-detection/train_labels.csv', dtype=str)\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-03T18:48:34.91562Z","iopub.execute_input":"2022-05-03T18:48:34.91586Z","iopub.status.idle":"2022-05-03T18:48:35.301445Z","shell.execute_reply.started":"2022-05-03T18:48:34.915826Z","shell.execute_reply":"2022-05-03T18:48:35.300663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-03T18:48:45.860076Z","iopub.execute_input":"2022-05-03T18:48:45.860403Z","iopub.status.idle":"2022-05-03T18:48:45.880568Z","shell.execute_reply.started":"2022-05-03T18:48:45.860374Z","shell.execute_reply":"2022-05-03T18:48:45.879847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.id = train.id + '.tif'","metadata":{"execution":{"iopub.status.busy":"2022-05-03T18:48:46.101586Z","iopub.execute_input":"2022-05-03T18:48:46.102189Z","iopub.status.idle":"2022-05-03T18:48:46.145708Z","shell.execute_reply.started":"2022-05-03T18:48:46.102156Z","shell.execute_reply":"2022-05-03T18:48:46.145037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-03T18:48:46.426406Z","iopub.execute_input":"2022-05-03T18:48:46.426653Z","iopub.status.idle":"2022-05-03T18:48:46.434559Z","shell.execute_reply.started":"2022-05-03T18:48:46.426625Z","shell.execute_reply":"2022-05-03T18:48:46.433915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label Distribution","metadata":{}},{"cell_type":"markdown","source":"We are now finding the series containing the counts of unique values. We then sort them and format them properly","metadata":{}},{"cell_type":"code","source":"(train.label.value_counts() / len(train)).to_frame().sort_index().T","metadata":{"execution":{"iopub.status.busy":"2022-04-05T19:43:20.308603Z","iopub.execute_input":"2022-04-05T19:43:20.308896Z","iopub.status.idle":"2022-04-05T19:43:20.347458Z","shell.execute_reply.started":"2022-04-05T19:43:20.30886Z","shell.execute_reply":"2022-04-05T19:43:20.346148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract Images","metadata":{}},{"cell_type":"markdown","source":"Here we create the filepath on the training data. We use mpimg and read in the trainingpath, and loop over each row of the id column. From here we display the images","metadata":{}},{"cell_type":"code","source":"train_path = \"../input/histopathologic-cancer-detection/train\"\n\nsample = train.sample(n=16).reset_index()\n\nplt.figure(figsize=(6,6))\n\nfor i, row in sample.iterrows():\n\n    img = mpimg.imread(f'../input/histopathologic-cancer-detection/train/{row.id}')    \n    label = row.label\n\n    plt.subplot(4,4,i+1)\n    plt.imshow(img)\n    plt.text(0, -5, f'Class {label}', color='k')\n        \n    plt.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-03T18:49:26.705499Z","iopub.execute_input":"2022-05-03T18:49:26.705875Z","iopub.status.idle":"2022-05-03T18:49:27.563411Z","shell.execute_reply.started":"2022-05-03T18:49:26.705844Z","shell.execute_reply":"2022-05-03T18:49:27.561007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training and Validation Sets","metadata":{}},{"cell_type":"markdown","source":"Now we are splitting the dataset into train_df, and valid_df","metadata":{}},{"cell_type":"code","source":"train_df, valid_df = train_test_split(train, test_size=0.2, random_state=1, stratify=train.label)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T19:43:21.167382Z","iopub.execute_input":"2022-04-05T19:43:21.167596Z","iopub.status.idle":"2022-04-05T19:43:21.465865Z","shell.execute_reply.started":"2022-04-05T19:43:21.167566Z","shell.execute_reply":"2022-04-05T19:43:21.465135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Generators","metadata":{}},{"cell_type":"markdown","source":"Now we're create two datagenerators, train_datagen and validation_datagen. We then create the train_loader and valid_loader from the train_datagen and validation_datagen. \n\nwe set each datafram to valid_df, with the directory of train_path. The x column and y column of the dataset we can see when we originally showed the files in the dataset. These were id, and label. Furthermore, we are setting the x_col to 'id' and the y_col to 'label'. This is a categorical project, therefore we set class_mode to categorical. Since the size of the images are 96 x 96. Therefore we set the target_size to 96, 96.\n\nFinally, we find the length of each loader and print it.","metadata":{}},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1/255)\nvalidation_datagen = ImageDataGenerator(rescale=1/255)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T19:43:21.468398Z","iopub.execute_input":"2022-04-05T19:43:21.468658Z","iopub.status.idle":"2022-04-05T19:43:21.473378Z","shell.execute_reply.started":"2022-04-05T19:43:21.468623Z","shell.execute_reply":"2022-04-05T19:43:21.472362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 64\n\ntrain_loader = train_datagen.flow_from_dataframe(\n    dataframe = valid_df,\n    directory = train_path,\n    x_col = 'id',\n    y_col = 'label',\n    batch_size = BATCH_SIZE,\n    seed = 1,\n    shuffle = True,\n    class_mode = 'categorical',\n    target_size = (96,96)\n)\n\nvalid_loader = train_datagen.flow_from_dataframe(\n    dataframe = valid_df,\n    directory = train_path,\n    x_col = 'id',\n    y_col = 'label',\n    batch_size = BATCH_SIZE,\n    seed = 1,\n    shuffle = True,\n    class_mode = 'categorical',\n    target_size = (96,96)\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T19:43:21.474872Z","iopub.execute_input":"2022-04-05T19:43:21.475194Z","iopub.status.idle":"2022-04-05T19:45:09.768035Z","shell.execute_reply.started":"2022-04-05T19:43:21.475159Z","shell.execute_reply":"2022-04-05T19:45:09.767267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TR_STEPS = len(train_loader)\nVA_STEPS = len(valid_loader)\n\nprint(TR_STEPS)\nprint(VA_STEPS)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T19:45:09.769366Z","iopub.execute_input":"2022-04-05T19:45:09.769806Z","iopub.status.idle":"2022-04-05T19:45:09.775214Z","shell.execute_reply.started":"2022-04-05T19:45:09.769758Z","shell.execute_reply":"2022-04-05T19:45:09.774491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Base Model","metadata":{}},{"cell_type":"markdown","source":"Here, we are creating our base model. From tensorflow, and keras, we use the function ResNet50. We are able to input the size of the images, and include imagenet as the weight. \n\nWe must set base_model.trainable to False for the best results. \n\nFinally, we print out the summary of the results","metadata":{}},{"cell_type":"code","source":"base_model = tf.keras.applications.ResNet50(\n    input_shape=(96,96,3), \n    include_top=False, \n    weights='imagenet'\n)\n\nbase_model.trainable = False\n\nbase_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T19:45:09.776583Z","iopub.execute_input":"2022-04-05T19:45:09.777049Z","iopub.status.idle":"2022-04-05T19:45:14.212338Z","shell.execute_reply.started":"2022-04-05T19:45:09.777008Z","shell.execute_reply":"2022-04-05T19:45:14.211629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build and Train","metadata":{}},{"cell_type":"markdown","source":"Now we create the cn, which contains all the content from the base model, and then we apply flatte, dense, batchnormalization and dropout","metadata":{}},{"cell_type":"code","source":"np.random.seed(1)\ntf.random.set_seed(1)\n\ncnn = Sequential([\n    base_model,\n    BatchNormalization(),\n\n    Flatten(),\n    \n    Dense(16, activation='relu'),\n    Dropout(0.5),\n    Dense(8, activation='relu'),\n    Dropout(0.5),\n    BatchNormalization(),\n    Dense(2, activation='softmax')\n])\n\ncnn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T19:45:14.213343Z","iopub.execute_input":"2022-04-05T19:45:14.213971Z","iopub.status.idle":"2022-04-05T19:45:14.647066Z","shell.execute_reply.started":"2022-04-05T19:45:14.213916Z","shell.execute_reply":"2022-04-05T19:45:14.64627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here is where we set the learning rate. At the moment, the learning rate is quite high which means the product of this training set will not be very accurate. \n\nSince we are messuring the loss, accuracy and AUC, we include those as well while compiling the model","metadata":{}},{"cell_type":"code","source":"opt = tf.keras.optimizers.Adam(0.001)\ncnn.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', tf.keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2022-04-05T19:45:14.648237Z","iopub.execute_input":"2022-04-05T19:45:14.648491Z","iopub.status.idle":"2022-04-05T19:45:14.67108Z","shell.execute_reply.started":"2022-04-05T19:45:14.648456Z","shell.execute_reply":"2022-04-05T19:45:14.670389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we train the train_loader for 40 epochs with the learning rate of .001","metadata":{}},{"cell_type":"code","source":"%%time \n\nh1 = cnn.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 40,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T19:45:14.672327Z","iopub.execute_input":"2022-04-05T19:45:14.672555Z","iopub.status.idle":"2022-04-05T19:47:32.981918Z","shell.execute_reply.started":"2022-04-05T19:45:14.67252Z","shell.execute_reply":"2022-04-05T19:47:32.98122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = merge_history([h1])\nvis_training(history)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine Tuning","metadata":{}},{"cell_type":"markdown","source":"Here is where we set the learning rate or the second training set. Now, the learning rate is lower which means the product of this training set will substantially more accurate. \n\nAgain, we include the loss, accuracy and AUC ","metadata":{}},{"cell_type":"code","source":"base_model.trainable = True\nk.set_value(cnn.optimizer.learning_rate, 0.00001)\ncnn.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', tf.keras.metrics.AUC()])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We include a summary of the output","metadata":{}},{"cell_type":"code","source":"cnn.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train 2","metadata":{}},{"cell_type":"markdown","source":"Now we train the train_loader for another 30 epochs with the learning rate of .00001","metadata":{}},{"cell_type":"code","source":"%%time \n\nh2 = cnn.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 30,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h2.history['auc'] = h2.history['auc_1']\nh2.history['val_auc'] = h2.history['val_auc_1']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = merge_history([h1, h2])\nvis_training(history, start=10)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training 3","metadata":{}},{"cell_type":"markdown","source":"We now train the train_loader for another 20 epochs with the learning rate of .00001. This is the last time we train.","metadata":{}},{"cell_type":"code","source":"%%time \n\nh3 = cnn.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 20,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h3.history['auc'] = h3.history['auc_1'] \nh3.history['val_auc'] = h3.history['val_auc_1'] ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = merge_history([h1, h2, h3])\nvis_training(history, start=10)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn.save('HCDv01.h5')\npickle.dump(history, open(f'HCDv01.pkl', 'wb'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\n\nprint('Test Set Size:', test.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['filename'] = test.id + '.tif'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = \"../input/histopathologic-cancer-detection/test\"\nprint('Test Images:', len(os.listdir(test_path)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 64\n\ntest_datagen = ImageDataGenerator(rescale=1/255)\n\ntest_loader = test_datagen.flow_from_dataframe(\n    dataframe = test,\n    directory = test_path,\n    x_col = 'filename',\n    batch_size = BATCH_SIZE,\n    shuffle = False,\n    class_mode = None,\n    target_size = (96,96)\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_probs = cnn.predict(test_loader)\nprint(test_probs.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(test_loader))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_probs[:10,].round(2))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = np.argmax(test_probs, axis=1)\nprint(test_pred[:10])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\nsubmission.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.label = test_probs[:,1]\nsubmission.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', header=True, index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}