{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":11848,"databundleVersionId":862157,"sourceType":"competition"}],"dockerImageVersionId":30527,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1. Load Library","metadata":{}},{"cell_type":"code","source":"!pip install -U efficientnet -qq","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport tensorflow.keras.layers as tfl\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.initializers import random_uniform, glorot_uniform\nimport efficientnet.tfkeras as efn\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import * \nimport os\nimport shutil\nimport json\n\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:08:21.261190Z","iopub.execute_input":"2024-07-23T00:08:21.261540Z","iopub.status.idle":"2024-07-23T00:08:21.267546Z","shell.execute_reply.started":"2024-07-23T00:08:21.261511Z","shell.execute_reply":"2024-07-23T00:08:21.266578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a function to streamline the Train data set   \ndef train_img_path(id_str):\n    return os.path.join(r\"/kaggle/input/histopathologic-cancer-detection/train\", f\"{id_str}.tif\")","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:08:21.268768Z","iopub.execute_input":"2024-07-23T00:08:21.269131Z","iopub.status.idle":"2024-07-23T00:08:21.279597Z","shell.execute_reply.started":"2024-07-23T00:08:21.269101Z","shell.execute_reply":"2024-07-23T00:08:21.278791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"example_path = \"/kaggle/input/histopathologic-cancer-detection/train/f38a6374c348f90b587e046aac6079959adf3835.tif\"\nexample_img = Image.open(example_path)\nexample_array = np.array(example_img)\nprint(f\"Image Shape = {example_array.shape}\")\nplt.imshow(example_img)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:08:21.280987Z","iopub.execute_input":"2024-07-23T00:08:21.281281Z","iopub.status.idle":"2024-07-23T00:08:21.537794Z","shell.execute_reply.started":"2024-07-23T00:08:21.281257Z","shell.execute_reply":"2024-07-23T00:08:21.536685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels_df = pd.read_csv('/kaggle/input/histopathologic-cancer-detection/train_labels.csv')\ntrain_labels_df[\"filename\"] = train_labels_df[\"id\"].apply(train_img_path)\ntrain_labels_df[\"label\"] = train_labels_df[\"label\"].astype(str)\ntrain_labels_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:09:54.989724Z","iopub.execute_input":"2024-07-23T00:09:54.990716Z","iopub.status.idle":"2024-07-23T00:09:56.041104Z","shell.execute_reply.started":"2024-07-23T00:09:54.990678Z","shell.execute_reply":"2024-07-23T00:09:56.040019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:09:56.043396Z","iopub.execute_input":"2024-07-23T00:09:56.044293Z","iopub.status.idle":"2024-07-23T00:09:56.050251Z","shell.execute_reply.started":"2024-07-23T00:09:56.044253Z","shell.execute_reply":"2024-07-23T00:09:56.049272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set(train_labels_df['label'])","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:09:56.051528Z","iopub.execute_input":"2024-07-23T00:09:56.051842Z","iopub.status.idle":"2024-07-23T00:09:56.102091Z","shell.execute_reply.started":"2024-07-23T00:09:56.051794Z","shell.execute_reply":"2024-07-23T00:09:56.101142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have 220,025 images in the train data set with 2 unique labels. 0 for not cancerous and 1 for cancerous tissues.","metadata":{}},{"cell_type":"code","source":"train_labels_df['label'].value_counts(normalize = True)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:09:57.736455Z","iopub.execute_input":"2024-07-23T00:09:57.737186Z","iopub.status.idle":"2024-07-23T00:09:57.786548Z","shell.execute_reply.started":"2024-07-23T00:09:57.737152Z","shell.execute_reply":"2024-07-23T00:09:57.785459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_data = np.empty((100, 96, 96, 3), dtype=np.uint8)\nsample_labels = np.empty(100, dtype=np.int8)\nfor i in range(len(train_labels_df))[:100]:\n    img_path = train_img_path(train_labels_df['id'][i])\n    img = Image.open(img_path)\n    sample_data[i] = np.array(img)\n    sample_labels[i] = train_labels_df['label'][i]","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:09:58.097334Z","iopub.execute_input":"2024-07-23T00:09:58.097723Z","iopub.status.idle":"2024-07-23T00:09:58.937653Z","shell.execute_reply.started":"2024-07-23T00:09:58.097691Z","shell.execute_reply":"2024-07-23T00:09:58.936791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Non-Cancerous Images\")\n\nselected_images = np.random.choice(sample_data[sample_labels == 0].shape[0], 12, replace=False)\ngrid_size = int(np.ceil(np.sqrt(12)))\n\nfig, axs = plt.subplots(grid_size, grid_size, figsize=(5, 5))\n\nfor i, ax in enumerate(axs.flatten()):\n    if i < 12:\n        ax.imshow(sample_data[sample_labels == 0][selected_images[i]])\n        ax.axis('off') \n    else:\n        fig.delaxes(ax) \n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:09:58.939378Z","iopub.execute_input":"2024-07-23T00:09:58.939689Z","iopub.status.idle":"2024-07-23T00:09:59.768259Z","shell.execute_reply.started":"2024-07-23T00:09:58.939660Z","shell.execute_reply":"2024-07-23T00:09:59.766670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Cancerous Images\")\n\nselected_images = np.random.choice(sample_data[sample_labels == 1].shape[0], 12, replace=False)\ngrid_size = int(np.ceil(np.sqrt(12)))\n\nfig, axs = plt.subplots(grid_size, grid_size, figsize=(5, 5))\n\nfor i, ax in enumerate(axs.flatten()):\n    if i < 12:\n        ax.imshow(sample_data[sample_labels == 1][selected_images[i]])\n        ax.axis('off') \n    else:\n        fig.delaxes(ax) \n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:09:59.769967Z","iopub.execute_input":"2024-07-23T00:09:59.770738Z","iopub.status.idle":"2024-07-23T00:10:00.475608Z","shell.execute_reply.started":"2024-07-23T00:09:59.770681Z","shell.execute_reply":"2024-07-23T00:10:00.474528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Model Designing","metadata":{}},{"cell_type":"markdown","source":"In this part we will perform various steps required to properly create the Resnet 50 model. We will load the data from the disk, specify train and validation data generators while creating test generator for the final submission.\n\nWe will then create a modified Resnet 50 suited for this task.","metadata":{}},{"cell_type":"code","source":"test_path = \"/kaggle/input/histopathologic-cancer-detection/test\"\ntest_ids = [filename[:-4] for filename in os.listdir(test_path)]\ntest_filenames = [os.path.join(test_path, filename) for filename in os.listdir(test_path)]\ntest_df = pd.DataFrame()\ntest_df[\"id\"] = test_ids\ntest_df[\"filename\"] = test_filenames","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:00.477992Z","iopub.execute_input":"2024-07-23T00:10:00.478364Z","iopub.status.idle":"2024-07-23T00:10:02.761455Z","shell.execute_reply.started":"2024-07-23T00:10:00.478333Z","shell.execute_reply":"2024-07-23T00:10:02.760575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale = 1/255, validation_split = 0.2)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:02.762892Z","iopub.execute_input":"2024-07-23T00:10:02.763203Z","iopub.status.idle":"2024-07-23T00:10:02.768082Z","shell.execute_reply.started":"2024-07-23T00:10:02.763175Z","shell.execute_reply":"2024-07-23T00:10:02.767065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = datagen.flow_from_dataframe(\n    shuffle = True,\n    dataframe = train_labels_df,\n    x_col = \"filename\",\n    y_col = \"label\",\n    target_size = (96, 96),\n    color_mode = \"rgb\",\n    batch_size = 32,\n    class_mode = \"binary\",\n    subset = \"training\",\n    validate_filenames = False,\n    seed = 10\n)\n\nvalidation_generator = datagen.flow_from_dataframe(\n    shuffle = True,\n    dataframe=train_labels_df,\n    x_col = \"filename\",\n    y_col = \"label\",\n    target_size=(96, 96),\n    color_mode = \"rgb\",\n    batch_size = 32,\n    class_mode = \"binary\",\n    subset = \"validation\",\n    validate_filenames = False,\n    seed = 10\n)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:02.769282Z","iopub.execute_input":"2024-07-23T00:10:02.769710Z","iopub.status.idle":"2024-07-23T00:10:04.324039Z","shell.execute_reply.started":"2024-07-23T00:10:02.769672Z","shell.execute_reply":"2024-07-23T00:10:04.322992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_generator = datagen.flow_from_dataframe(\n    dataframe = test_df,\n    x_col = \"filename\",\n    y_col = None,\n    target_size = (96, 96),\n    color_mode = \"rgb\",\n    batch_size = 64,\n    shuffle = False,\n    class_mode = None,\n    validate_filenames = False,\n    seed = 10\n)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:17.650978Z","iopub.execute_input":"2024-07-23T00:10:17.651362Z","iopub.status.idle":"2024-07-23T00:10:17.791521Z","shell.execute_reply.started":"2024-07-23T00:10:17.651329Z","shell.execute_reply":"2024-07-23T00:10:17.790671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_steps = 176020//32  # 8000 images for training\nval_steps = 44005//32  ","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:18.082319Z","iopub.execute_input":"2024-07-23T00:10:18.082707Z","iopub.status.idle":"2024-07-23T00:10:18.087633Z","shell.execute_reply.started":"2024-07-23T00:10:18.082676Z","shell.execute_reply":"2024-07-23T00:10:18.086467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we will create Resnet50 model.\n\nIt is to be noted that we can actually use transfer learning and import the model and its trained parameters based on imagenet from the tensorflow API. But for this project we will create a modified model.\n\nThere will be two types of blocks, identity blocks where input and output dimension remains the same, and convolutional blocks where input and output dimensions are allowed to be changed. In both blocks we will use strong skip connections.\n\nAfter convolutional we will use fully collected layers with output being a dense Sigmoid unit.\n\nIn the end we will have a Resnet model with 50 layers.","metadata":{}},{"cell_type":"code","source":"def identity_block(X, f, filters, training=True, initializer = random_uniform):\n    \"\"\"\n    \n    Arguments:\n    X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)\n    f -- integer, specifying the shape of the middle CONV's window for the main path\n    filters -- python list of integers, defining the number of filters in the CONV layers of the main path\n    training -- True: Behave in training mode\n                False: Behave in inference mode\n    initializer -- to set up the initial weights of a layer. Equals to random uniform initializer\n    \n    Returns:\n    X -- output of the identity block, tensor of shape (m, n_H, n_W, n_C)\n    \"\"\"\n    \n    # Filters\n    F1, F2, F3 = filters\n    \n    # Save the input value. You'll need this later to add back to the main path. \n    X_shortcut = X\n    \n    # First component of main path\n    X = tfl.Conv2D(filters = F1, kernel_size = 1, strides = (1,1), padding = 'valid', kernel_initializer = initializer(seed=0))(X)\n    X = tfl.BatchNormalization(axis = 3)(X, training = training) # Default axis\n    X = tfl.Activation('relu')(X)\n    \n    ## Set the padding = 'same'\n    X = tfl.Conv2D(filters = F2, kernel_size = f, strides = 1, padding = 'same', kernel_initializer = initializer(seed=0))(X)\n    X = tfl.BatchNormalization(axis = 3)(X, training = training)\n    X = tfl.Activation('relu')(X)\n\n\n    ## Set the padding = 'valid'\n    X = tfl.Conv2D(filters = F3, kernel_size = 1, strides = 1, padding = 'valid', kernel_initializer = initializer(seed = 0))(X)\n    X = tfl.BatchNormalization(axis = 3)(X, training = training) \n    \n    ## Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines)\n    X = tfl.Add()([X, X_shortcut])\n    X = tfl.Activation('relu')(X)\n\n    return X\n","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:24.440174Z","iopub.execute_input":"2024-07-23T00:10:24.440570Z","iopub.status.idle":"2024-07-23T00:10:24.452470Z","shell.execute_reply.started":"2024-07-23T00:10:24.440538Z","shell.execute_reply":"2024-07-23T00:10:24.451466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convolutional_block(X, f, filters, s = 2, training=True, initializer = glorot_uniform):\n    \"\"\"\n    Implementation of the convolutional block\n    \n    Arguments:\n    X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)\n    f -- integer, specifying the shape of the middle CONV's window for the main path\n    filters -- python list of integers, defining the number of filters in the CONV layers of the main path\n    s -- Integer, specifying the stride to be used\n    training -- True: Behave in training mode\n                False: Behave in inference mode\n    initializer -- to set up the initial weights of a layer. Equals to Glorot uniform initializer, \n                   also called Xavier uniform initializer.\n    \n    Returns:\n    X -- output of the convolutional block, tensor of shape (m, n_H, n_W, n_C)\n    \"\"\"\n    \n    # Filters\n    F1, F2, F3 = filters\n    \n    # Save the input value\n    X_shortcut = X\n\n    X = tfl.Conv2D(filters = F1, kernel_size = 1, strides = (s, s), padding='valid', kernel_initializer = initializer(seed=0))(X)\n    X = tfl.BatchNormalization(axis = 3)(X, training=training)\n    X = tfl.Activation('relu')(X)\n    \n    X = tfl.Conv2D(filters = F2, kernel_size = (f,f), strides = 1, padding='same', kernel_initializer = initializer(seed=0))(X) \n    X = tfl.BatchNormalization(axis = 3)(X, training=training)\n    X = tfl.Activation('relu')(X) \n\n    X = tfl.Conv2D(filters = F3, kernel_size = 1, strides = 1, padding='valid', kernel_initializer = initializer(seed=0))(X)\n    X = tfl.BatchNormalization(axis = 3)(X, training=training) \n\n    X_shortcut = tfl.Conv2D(filters = F3, kernel_size = (1,1), strides = (s, s), padding='valid', kernel_initializer = initializer(seed=0))(X_shortcut)\n    X_shortcut = tfl.BatchNormalization(axis = 3)(X_shortcut, training=training)\n    \n    X = tfl.Add()([X, X_shortcut])\n    X = tfl.Activation('relu')(X)\n    \n    return X","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:24.542208Z","iopub.execute_input":"2024-07-23T00:10:24.542579Z","iopub.status.idle":"2024-07-23T00:10:24.558173Z","shell.execute_reply.started":"2024-07-23T00:10:24.542546Z","shell.execute_reply":"2024-07-23T00:10:24.557109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ResNet50(input_shape = (96, 96, 3)):\n    \"\"\"\n    Stage-wise implementation of the architecture of the popular ResNet50:\n    CONV2D -> BATCHNORM -> RELU -> MAXPOOL -> CONVBLOCK -> IDBLOCK*2 -> CONVBLOCK -> IDBLOCK*3\n    -> CONVBLOCK -> IDBLOCK*5 -> CONVBLOCK -> IDBLOCK*2 -> AVGPOOL -> FLATTEN -> DENSE \n\n    Arguments:\n    input_shape -- shape of the images of the dataset\n\n    Returns:\n    model -- a Model() instance in Keras\n    \"\"\"\n    \n    # Define the input as a tensor with shape input_shape\n    X_input = tfl.Input(input_shape)\n\n    \n    # Zero-Padding\n    X = tfl.ZeroPadding2D((3, 3))(X_input)\n    \n    # Stage 1\n    X = tfl.Conv2D(96, (7, 7), strides = (2, 2), kernel_initializer = glorot_uniform(seed=0))(X)\n    X = tfl.BatchNormalization(axis = 3)(X)\n    X = tfl.Activation('relu')(X)\n    X = tfl.MaxPooling2D((3, 3), strides=(2, 2))(X)\n\n    # Stage 2\n    X = convolutional_block(X, f = 3, filters = [64, 64, 256], s = 1)\n    X = identity_block(X, 3, [64, 64, 256])\n    X = identity_block(X, 3, [64, 64, 256])\n\n    ### START CODE HERE\n    \n    # Use the instructions above in order to implement all of the Stages below\n    # Make sure you don't miss adding any required parameter\n    \n    ## Stage 3 (≈4 lines)\n    # `convolutional_block` with correct values of `f`, `filters` and `s` for this stage\n    X = convolutional_block(X, f = 3, filters = [128, 128, 512], s = 2)\n    \n    # the 3 `identity_block` with correct values of `f` and `filters` for this stage\n    X = identity_block(X, 3, [128, 128, 512])\n    X = identity_block(X, 3, [128, 128, 512])\n    X = identity_block(X, 3, [128, 128, 512])\n\n    # Stage 4 (≈6 lines)\n    # add `convolutional_block` with correct values of `f`, `filters` and `s` for this stage\n    X = convolutional_block(X, f = 3, filters = [256, 256, 1024], s = 2)\n    \n    # the 5 `identity_block` with correct values of `f` and `filters` for this stage\n    X = identity_block(X, 3, [256, 256, 1024])\n    X = identity_block(X, 3, [256, 256, 1024])\n    X = identity_block(X, 3, [256, 256, 1024])\n    X = identity_block(X, 3,[256, 256, 1024])\n    X = identity_block(X, 3, [256, 256, 1024])\n\n    # Stage 5 (≈3 lines)\n    # add `convolutional_block` with correct values of `f`, `filters` and `s` for this stage\n    X = convolutional_block(X, f = 3, filters = [512, 512, 2048], s = 2)\n    \n    # the 2 `identity_block` with correct values of `f` and `filters` for this stage\n    X = identity_block(X, 3, [512, 512, 2048])\n    X = identity_block(X, 3, [512, 512, 2048])\n\n    # AVGPOOL (≈1 line). Use \"X = AveragePooling2D()(X)\"\n    X = tfl.AveragePooling2D(pool_size = (2,2))(X)\n    \n\n    # output layer\n    X = tfl.Flatten()(X)\n    X = tfl.Dense(1, kernel_initializer = glorot_uniform(seed=0))(X)\n    \n    \n    # Create model\n    model = Model(inputs = X_input, outputs = X)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:25.026107Z","iopub.execute_input":"2024-07-23T00:10:25.027085Z","iopub.status.idle":"2024-07-23T00:10:25.042943Z","shell.execute_reply.started":"2024-07-23T00:10:25.027048Z","shell.execute_reply":"2024-07-23T00:10:25.041874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ResNet50(input_shape = (96, 96, 3))\nprint(model.summary())","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:26.669948Z","iopub.execute_input":"2024-07-23T00:10:26.670960Z","iopub.status.idle":"2024-07-23T00:10:31.041361Z","shell.execute_reply.started":"2024-07-23T00:10:26.670924Z","shell.execute_reply":"2024-07-23T00:10:31.040332Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Above is the model summary. We have 23,551,681 trainable parameters. If we use more deeper network number of parameters will increase with possible increase in Accuracy or AUC. But for this model we will use resnet 50 only.","metadata":{}},{"cell_type":"markdown","source":"## 4. Model Deployment","metadata":{}},{"cell_type":"code","source":"from IPython.display import Image\nImage(url=\"https://wisdomml.in/wp-content/uploads/2023/03/resnet_bannner.png\")","metadata":{"execution":{"iopub.status.busy":"2024-07-23T03:56:52.805535Z","iopub.execute_input":"2024-07-23T03:56:52.805883Z","iopub.status.idle":"2024-07-23T03:56:52.812832Z","shell.execute_reply.started":"2024-07-23T03:56:52.805848Z","shell.execute_reply":"2024-07-23T03:56:52.811859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this part we will use the model created and fit the model with Adam optimizer and loss as binary cross entropy. We have used from logits = True for better accuracy. If reader wants to avoid it they can modify the last output layer in Resnet50 above and make the Activation function as sigmoid instead of linear.","metadata":{}},{"cell_type":"code","source":"model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = 0.001),\n              loss=tf.keras.losses.BinaryCrossentropy(from_logits = True),\n              metrics=['accuracy', tf.keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:52.154202Z","iopub.execute_input":"2024-07-23T00:10:52.154974Z","iopub.status.idle":"2024-07-23T00:10:52.191588Z","shell.execute_reply.started":"2024-07-23T00:10:52.154938Z","shell.execute_reply":"2024-07-23T00:10:52.190469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\ntf.test.gpu_device_name()","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:53.944929Z","iopub.execute_input":"2024-07-23T00:10:53.945322Z","iopub.status.idle":"2024-07-23T00:10:53.953751Z","shell.execute_reply.started":"2024-07-23T00:10:53.945292Z","shell.execute_reply":"2024-07-23T00:10:53.952650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    train_generator,\n    steps_per_epoch = train_steps,\n    validation_data = validation_generator,\n    validation_steps = val_steps,\n    epochs = 10\n)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T00:10:54.847999Z","iopub.execute_input":"2024-07-23T00:10:54.848350Z","iopub.status.idle":"2024-07-23T01:50:53.395865Z","shell.execute_reply.started":"2024-07-23T00:10:54.848323Z","shell.execute_reply":"2024-07-23T01:50:53.394901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def merge_history(hlist):\n    history = {}\n    for k in hlist[0].history.keys():\n        history[k] = sum([h.history[k] for h in hlist], [])\n    return history\n\ndef vis_training(h, start=1):\n    epoch_range = range(start, len(h['loss'])+1)\n    s = slice(start-1, None)\n\n    plt.figure(figsize=[14,4])\n\n    n = int(len(h.keys()) / 2)\n\n    for i in range(n):\n        k = list(h.keys())[i]\n        plt.subplot(1,n,i+1)\n        plt.plot(epoch_range, h[k][s], label='Training')\n        plt.plot(epoch_range, h['val_' + k][s], label='Validation')\n        plt.xlabel('Epoch'); plt.ylabel(k); plt.title(k)\n        plt.grid()\n        plt.legend()\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-23T02:05:30.432897Z","iopub.execute_input":"2024-07-23T02:05:30.433865Z","iopub.status.idle":"2024-07-23T02:05:30.444159Z","shell.execute_reply.started":"2024-07-23T02:05:30.433802Z","shell.execute_reply":"2024-07-23T02:05:30.443072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_modelrestnet = merge_history([history])\nvis_training(history_modelrestnet)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T02:05:31.659294Z","iopub.execute_input":"2024-07-23T02:05:31.659686Z","iopub.status.idle":"2024-07-23T02:05:32.659883Z","shell.execute_reply.started":"2024-07-23T02:05:31.659657Z","shell.execute_reply":"2024-07-23T02:05:32.658789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It takes around 2 hours to train the model with 10 epochs and Kaggle P100 GPU. We can increase the number of epochs but only Training loss decrease for some time after 10 epochs with little change in validation loss and accuracy. In fact we can see from above that after 6th Epoch Validation Accruacy has not changed much.\n\nSo, it will not be efficient to train with more epochs but readers can do it if they have resources and time for this.","metadata":{}},{"cell_type":"markdown","source":"## 5. Model Evaluation","metadata":{}},{"cell_type":"markdown","source":"Model evaluation is limited here as we don't have test dataset labels. We will still perform basic evaluation and see why we cant do indepth model evaluation here.","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, roc_curve\n\n\nval_predictions = tf.nn.sigmoid(model.predict(validation_generator)).numpy()\nval_pred_classes = (val_predictions > 0.5).astype(int).flatten() #Threshold is assumed to be 0.5 for this cell\n\n# True labels\ntrue_labels = validation_generator.classes\n\n# Ensure the lengths match\nval_pred_classes = val_pred_classes[:len(true_labels)]\n\n# Calculate metrics\naccuracy = accuracy_score(true_labels, val_pred_classes)\nprecision = precision_score(true_labels, val_pred_classes)\nrecall = recall_score(true_labels, val_pred_classes)\nf1 = f1_score(true_labels, val_pred_classes)\nroc_auc = roc_auc_score(true_labels, val_predictions)\n\nprint(f\"Accuracy: {accuracy:.4f}\")\nprint(f\"Precision: {precision:.4f}\")\nprint(f\"Recall: {recall:.4f}\")\nprint(f\"F1 Score: {f1:.4f}\")\nprint(f\"ROC-AUC: {roc_auc:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2024-07-23T02:05:40.034958Z","iopub.execute_input":"2024-07-23T02:05:40.035886Z","iopub.status.idle":"2024-07-23T02:06:56.997253Z","shell.execute_reply.started":"2024-07-23T02:05:40.035844Z","shell.execute_reply":"2024-07-23T02:06:56.996212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Above we have used a threshold of 0.5 which is just assumed and is totally not correct. There was no decision boundary given with the dataset so we cant use above metrics to test the performance of the model.\n\nIn order to test the model performance we need to create a submission and get the AUC we will obtain from such submission.","metadata":{}},{"cell_type":"code","source":"best_loss = history['val_loss'][i_min]\nbest_accuracy = history['val_accuracy'][i_min]\nbest_auc = history['val_auc'][i_min]\n\n\n# best_loss = min(history['val_loss'])\n# best_accuracy = max(history['val_accuracy'])\n# best_auc = max(history['val_auc'])\n\n# best_accuracy, best_auc, best_loss","metadata":{"execution":{"iopub.status.busy":"2024-07-23T02:09:20.073904Z","iopub.execute_input":"2024-07-23T02:09:20.074843Z","iopub.status.idle":"2024-07-23T02:09:20.082924Z","shell.execute_reply.started":"2024-07-23T02:09:20.074783Z","shell.execute_reply":"2024-07-23T02:09:20.081756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = [0.] + history.history['accuracy']\nval_acc = [0.] + history.history['val_accuracy']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nplt.figure(figsize=(8, 8))\nplt.subplot(2, 1, 1)\nplt.plot(acc, label='Training Accuracy')\nplt.plot(val_acc, label='Validation Accuracy')\nplt.legend(loc='lower right')\nplt.ylabel('Accuracy')\nplt.ylim([min(plt.ylim()),1])\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(2, 1, 2)\nplt.plot(loss, label='Training Loss')\nplt.plot(val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.ylabel('Cross Entropy')\nplt.ylim([0,1.0])\nplt.title('Training and Validation Loss')\nplt.xlabel('epoch')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-23T02:09:37.203981Z","iopub.execute_input":"2024-07-23T02:09:37.204670Z","iopub.status.idle":"2024-07-23T02:09:37.209712Z","shell.execute_reply.started":"2024-07-23T02:09:37.204633Z","shell.execute_reply":"2024-07-23T02:09:37.208689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 2 --> EfficientNetB0","metadata":{}},{"cell_type":"code","source":"from IPython.display import Image\nImage(url=\"https://media.licdn.com/dms/image/D4E12AQHL32j_wE3JTw/article-cover_image-shrink_720_1280/0/1669236519664?e=1727308800&v=beta&t=BM4KMl52c-TXWz0C-SCD_MbHo1_rdXaC6UWRhTNqZqo\")","metadata":{"execution":{"iopub.status.busy":"2024-07-23T03:58:10.103197Z","iopub.execute_input":"2024-07-23T03:58:10.104127Z","iopub.status.idle":"2024-07-23T03:58:10.111498Z","shell.execute_reply.started":"2024-07-23T03:58:10.104086Z","shell.execute_reply":"2024-07-23T03:58:10.110324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model = efn.EfficientNetB0(input_shape=(96,96,3), include_top=False, weights='imagenet')\n\ncnn = Sequential([\n    base_model,\n    \n    Flatten(),\n    \n    Dense(64, activation='relu'),\n    Dropout(0.5),\n    Dense(32, activation='relu'),\n    Dropout(0.5),\n    BatchNormalization(),\n    Dense(1, activation='sigmoid')\n])\n\ncnn.summary()","metadata":{"execution":{"iopub.status.busy":"2024-07-23T02:37:49.098575Z","iopub.execute_input":"2024-07-23T02:37:49.099535Z","iopub.status.idle":"2024-07-23T02:37:51.726731Z","shell.execute_reply.started":"2024-07-23T02:37:49.099499Z","shell.execute_reply":"2024-07-23T02:37:51.725748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn.compile(optimizer = tf.keras.optimizers.Adam(0.001),\n              loss=tf.keras.losses.BinaryCrossentropy(from_logits = True),\n              metrics=['accuracy', tf.keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2024-07-23T02:37:51.728379Z","iopub.execute_input":"2024-07-23T02:37:51.728690Z","iopub.status.idle":"2024-07-23T02:37:51.751962Z","shell.execute_reply.started":"2024-07-23T02:37:51.728663Z","shell.execute_reply":"2024-07-23T02:37:51.750858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h1 = cnn.fit(\n    train_generator,\n    steps_per_epoch = train_steps,\n    validation_data = validation_generator,\n    validation_steps = val_steps,\n    epochs = 10\n)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T02:37:59.437799Z","iopub.execute_input":"2024-07-23T02:37:59.438209Z","iopub.status.idle":"2024-07-23T03:56:52.803596Z","shell.execute_reply.started":"2024-07-23T02:37:59.438174Z","shell.execute_reply":"2024-07-23T03:56:52.802625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_model_B0 = merge_history([h1])\nvis_training(history_model_B0)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T03:58:20.572667Z","iopub.execute_input":"2024-07-23T03:58:20.573068Z","iopub.status.idle":"2024-07-23T03:58:21.575256Z","shell.execute_reply.started":"2024-07-23T03:58:20.573033Z","shell.execute_reply":"2024-07-23T03:58:21.574160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 3 --> Vision Transformer (ViT)","metadata":{}},{"cell_type":"code","source":"from IPython.display import Image\nImage(url=\"https://cdn-lfs.huggingface.co/datasets/huggingface/documentation-images/142f1b9c07445fbc30f72e4e8e629a9cf11488a78ca6babc8bdb49c11e42b672?response-content-disposition=inline%3B+filename*%3DUTF-8%27%27vit_architecture.jpg%3B+filename%3D%22vit_architecture.jpg%22%3B&response-content-type=image%2Fjpeg&Expires=1721966721&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMTk2NjcyMX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9kYXRhc2V0cy9odWdnaW5nZmFjZS9kb2N1bWVudGF0aW9uLWltYWdlcy8xNDJmMWI5YzA3NDQ1ZmJjMzBmNzJlNGU4ZTYyOWE5Y2YxMTQ4OGE3OGNhNmJhYmM4YmRiNDljMTFlNDJiNjcyP3Jlc3BvbnNlLWNvbnRlbnQtZGlzcG9zaXRpb249KiZyZXNwb25zZS1jb250ZW50LXR5cGU9KiJ9XX0_&Signature=m9lRPCippy3Q01FqLAaM-6Y0BHJAQtDdNCNBDWQ3iX78TSKQC-5dxBaFaWBZ9uAmHjNQBpgzJeW9rlKrB9lQDMRPo8-%7Efl9g7djOVm%7EkE85ZxX3N8s8DZ664DqQ6pxt2l5mPu8AYqEU5ncNi2GotdmMH3zHxenmSM2gr-rI9xges0maUrhvu0WDC44MA0W2pH7Mrsg3GicEqM9gL0Qmr8rs5f%7EVFhVZKz8EtVocO32lsp0L%7Emia2JnGXiBQVwFcWdT%7EUBMpss1nzJSAwsrx8wAuQ%7E9g82ASAiEq6Tp1sy-onzbolXqh41pskL1YXZw8A95Yiyypv2mgzQ661MxTeeg__&Key-Pair-Id=K3ESJI6DHPFC7\")","metadata":{"execution":{"iopub.status.busy":"2024-07-23T05:08:46.966765Z","iopub.execute_input":"2024-07-23T05:08:46.967084Z","iopub.status.idle":"2024-07-23T05:08:46.974450Z","shell.execute_reply.started":"2024-07-23T05:08:46.967056Z","shell.execute_reply":"2024-07-23T05:08:46.973482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import Input, Dense, Flatten, Embedding, LayerNormalization, MultiHeadAttention, Dropout\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2024-07-23T04:01:56.894378Z","iopub.execute_input":"2024-07-23T04:01:56.895357Z","iopub.status.idle":"2024-07-23T04:01:56.902308Z","shell.execute_reply.started":"2024-07-23T04:01:56.895317Z","shell.execute_reply":"2024-07-23T04:01:56.901206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the Vision Transformer model\nclass VisionTransformer(Model):\n    def __init__(self, img_size, patch_size, num_classes, embed_dim=256, num_heads=4, num_layers=4, mlp_dim=512):\n        super(VisionTransformer, self).__init__()\n        \n        self.patch_size = patch_size\n        self.img_size = img_size\n        self.num_patches = (img_size // patch_size) ** 2\n\n        # Define the input layer\n        self.input_layer = Input(shape=(img_size, img_size, 3))\n        \n        # Patch Embedding\n        self.patch_embedding = tf.keras.layers.Conv2D(filters=embed_dim, kernel_size=patch_size, strides=patch_size, padding='valid')\n        \n        # Position Embedding\n        self.position_embedding = Embedding(input_dim=self.num_patches, output_dim=embed_dim)\n        \n        # Transformer Encoder Layers\n        self.transformer_layers = [tf.keras.layers.LayerNormalization(epsilon=1e-6) for _ in range(num_layers)]\n        self.multi_head_attention = MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.mlp = tf.keras.Sequential([\n            Dense(mlp_dim, activation='relu'),\n            Dense(embed_dim)\n        ])\n        \n        # Output layer\n        self.flatten = Flatten()\n        self.output_layer = Dense(num_classes, activation='sigmoid')\n\n    def call(self, inputs):\n        # Patch embeddings\n        x = self.patch_embedding(inputs)\n        x = tf.reshape(x, (-1, self.num_patches, x.shape[-1]))\n\n        # Add position embeddings\n        x += self.position_embedding(tf.range(start=0, limit=self.num_patches))\n        \n        for layer in self.transformer_layers:\n            # Multi-head attention\n            x = layer(x)\n            x = self.multi_head_attention(x, x)\n            x = Dropout(0.1)(x)\n            \n            # MLP\n            x = self.mlp(x)\n        \n        x = self.flatten(x)\n        x = self.output_layer(x)\n        return x\n\n# Parameters\nimg_size = 96  # Size of input images\npatch_size = 16  # Size of patches\nnum_classes = 1  # Number of output classes\n\n# Instantiate and compile the model\nvit_model = VisionTransformer(img_size=img_size, patch_size=patch_size, num_classes=num_classes)\nvit_model.compile(optimizer=Adam(learning_rate=1e-4), loss='binary_crossentropy',\n                  metrics=['accuracy', tf.keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2024-07-23T04:04:17.656967Z","iopub.execute_input":"2024-07-23T04:04:17.657359Z","iopub.status.idle":"2024-07-23T04:04:17.705768Z","shell.execute_reply.started":"2024-07-23T04:04:17.657328Z","shell.execute_reply":"2024-07-23T04:04:17.704875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vit_h = vit_model.fit(\n    train_generator,\n    steps_per_epoch = train_steps,\n    validation_data = validation_generator,\n    validation_steps = val_steps,\n    epochs = 10\n)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T04:04:19.459139Z","iopub.execute_input":"2024-07-23T04:04:19.459957Z","iopub.status.idle":"2024-07-23T05:08:46.964981Z","shell.execute_reply.started":"2024-07-23T04:04:19.459918Z","shell.execute_reply":"2024-07-23T05:08:46.964093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}