{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Libraries","metadata":{"id":"w_UApzyTr2hK"}},{"cell_type":"code","source":"pip install tf-explain","metadata":{"id":"cdTPjijfYngV","outputId":"6fb84dbd-a7bf-41d6-ca61-ca44720c4f2f","execution":{"iopub.status.busy":"2023-04-15T17:36:14.568403Z","iopub.execute_input":"2023-04-15T17:36:14.568695Z","iopub.status.idle":"2023-04-15T17:36:26.964065Z","shell.execute_reply.started":"2023-04-15T17:36:14.568666Z","shell.execute_reply":"2023-04-15T17:36:26.962766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom keras.utils import to_categorical\nfrom keras import optimizers\nfrom keras import layers\nfrom keras import models\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications.vgg19 import preprocess_input\nfrom keras.models import load_model\nimport keras.utils as image\nfrom keras.layers import Input, Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.applications.vgg19 import VGG19\nfrom tensorflow.keras.applications import ResNet50\nfrom sklearn.model_selection import train_test_split\nfrom PIL import Image\nimport cv2\nimport random\nfrom tf_explain.core.grad_cam import GradCAM\nfrom tqdm import tqdm\nfrom keras.callbacks import LearningRateScheduler","metadata":{"id":"poD6npdBrv-L","execution":{"iopub.status.busy":"2023-04-15T22:44:43.830616Z","iopub.execute_input":"2023-04-15T22:44:43.831220Z","iopub.status.idle":"2023-04-15T22:44:51.690960Z","shell.execute_reply.started":"2023-04-15T22:44:43.831162Z","shell.execute_reply":"2023-04-15T22:44:51.689864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Visualization and preparation","metadata":{"id":"VhMei4Cgb7Nl"}},{"cell_type":"code","source":"train_dir = '/kaggle/input/state-farm-distracted-driver-detection/imgs/train'\ndf_train = pd.read_csv('/kaggle/input/state-farm-distracted-driver-detection/driver_imgs_list.csv')\ndf_test = pd.read_csv('/kaggle/input/state-farm-distracted-driver-detection/sample_submission.csv')\n\ndf_train['path'] = train_dir + '/' + df_train['classname']+ '/' + df_train['img']","metadata":{"id":"_AE-rEnIb02W","execution":{"iopub.status.busy":"2023-04-15T17:36:36.604134Z","iopub.execute_input":"2023-04-15T17:36:36.604942Z","iopub.status.idle":"2023-04-15T17:36:36.821959Z","shell.execute_reply.started":"2023-04-15T17:36:36.604900Z","shell.execute_reply":"2023-04-15T17:36:36.820903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"id":"L6R1MBh-cqoc","outputId":"f6a8a12b-575c-4919-d405-96ac04700450","execution":{"iopub.status.busy":"2023-04-15T17:36:39.213515Z","iopub.execute_input":"2023-04-15T17:36:39.214535Z","iopub.status.idle":"2023-04-15T17:36:39.230665Z","shell.execute_reply.started":"2023-04-15T17:36:39.214494Z","shell.execute_reply":"2023-04-15T17:36:39.229582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(df_train, x = 'classname')","metadata":{"id":"7_qpXJyVcr4H","outputId":"db33486b-c88c-43bd-d7b9-5eccecfd4115","execution":{"iopub.status.busy":"2023-04-15T17:36:39.648451Z","iopub.execute_input":"2023-04-15T17:36:39.649331Z","iopub.status.idle":"2023-04-15T17:36:39.908702Z","shell.execute_reply.started":"2023-04-15T17:36:39.649295Z","shell.execute_reply":"2023-04-15T17:36:39.907672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = {\n          'c0': 'normal driving',\n          'c1': 'texting - right',\n          'c2': 'talking on the phone - right',\n          'c3': 'texting - left',\n          'c4': 'talking on the phone - left',\n          'c5': 'operating the radio',\n          'c6': 'drinking',\n          'c7': 'reaching behind',\n          'c8': 'hair and makeup',\n          'c9': 'talking to passenger'}","metadata":{"id":"EoPfAeGKc23S","execution":{"iopub.status.busy":"2023-04-15T23:09:38.980625Z","iopub.execute_input":"2023-04-15T23:09:38.980991Z","iopub.status.idle":"2023-04-15T23:09:38.986343Z","shell.execute_reply.started":"2023-04-15T23:09:38.980960Z","shell.execute_reply":"2023-04-15T23:09:38.985239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 30))\nfor ind, i in enumerate(classes.keys()):\n  for n in range(5):\n    plt.subplot(10, 5, (5 * ind) + n + 1)\n    img_path = f\"{train_dir}/{i}/{random.sample(os.listdir(f'{train_dir}/{i}'),1)[0]}\"\n    img = Image.open(img_path)\n    img = img.resize((224,224))\n    plt.grid(False)\n    plt.xticks([])\n    plt.yticks([])\n    plt.imshow(img)\n    plt.xlabel(classes[i])","metadata":{"id":"bAoVYxqIdKL5","outputId":"6ea1c522-1e7d-4cae-ac24-1d986af6dea2","execution":{"iopub.status.busy":"2023-04-15T17:36:42.946231Z","iopub.execute_input":"2023-04-15T17:36:42.947170Z","iopub.status.idle":"2023-04-15T17:36:51.831404Z","shell.execute_reply.started":"2023-04-15T17:36:42.947106Z","shell.execute_reply":"2023-04-15T17:36:51.830084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split the data","metadata":{"id":"o8QR3cZcorpn"}},{"cell_type":"code","source":"X, y = df_train[['path', 'classname']], df_train['classname']","metadata":{"id":"A5MAvu-VosIT","execution":{"iopub.status.busy":"2023-04-15T17:36:51.834219Z","iopub.execute_input":"2023-04-15T17:36:51.834702Z","iopub.status.idle":"2023-04-15T17:36:51.845446Z","shell.execute_reply.started":"2023-04-15T17:36:51.834651Z","shell.execute_reply":"2023-04-15T17:36:51.844620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, _, _ = train_test_split(X, y, stratify= y ,test_size=0.2, random_state=42)","metadata":{"id":"ocxf6rzTowwQ","execution":{"iopub.status.busy":"2023-04-15T17:36:51.847152Z","iopub.execute_input":"2023-04-15T17:36:51.847802Z","iopub.status.idle":"2023-04-15T17:36:51.886595Z","shell.execute_reply.started":"2023-04-15T17:36:51.847766Z","shell.execute_reply":"2023-04-15T17:36:51.885457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.head()","metadata":{"id":"-3qYwKW6o20P","outputId":"2c06c446-3810-440f-b3db-19074a080c73","execution":{"iopub.status.busy":"2023-04-15T17:36:51.890215Z","iopub.execute_input":"2023-04-15T17:36:51.890921Z","iopub.status.idle":"2023-04-15T17:36:51.901726Z","shell.execute_reply.started":"2023-04-15T17:36:51.890882Z","shell.execute_reply":"2023-04-15T17:36:51.900713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building custom network","metadata":{"id":"rwmqWji9kcqy"}},{"cell_type":"code","source":"model = models.Sequential()\nmodel.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(224, 224, 3)))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Conv2D(64, (3, 3), activation='relu'))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Conv2D(128, (3, 3), activation='relu'))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Conv2D(128, (3, 3), activation='relu'))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(512, activation='relu'))\nmodel.add(layers.Dense(10, activation='softmax'))","metadata":{"id":"8Ef1iVkzkqw1","execution":{"iopub.status.busy":"2023-04-15T17:36:51.903074Z","iopub.execute_input":"2023-04-15T17:36:51.903626Z","iopub.status.idle":"2023-04-15T17:36:54.357182Z","shell.execute_reply.started":"2023-04-15T17:36:51.903583Z","shell.execute_reply":"2023-04-15T17:36:54.356141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"id":"JmgZHvsnk11i","outputId":"07309dd9-d248-425e-9ee8-53f94b65e2fd","execution":{"iopub.status.busy":"2023-04-15T17:36:54.358682Z","iopub.execute_input":"2023-04-15T17:36:54.359069Z","iopub.status.idle":"2023-04-15T17:36:54.398719Z","shell.execute_reply.started":"2023-04-15T17:36:54.359032Z","shell.execute_reply":"2023-04-15T17:36:54.397769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(learning_rate = 1e-4),\n              metrics=['accuracy'])","metadata":{"id":"_0vu-R2HmNuh","execution":{"iopub.status.busy":"2023-04-15T17:36:54.399717Z","iopub.execute_input":"2023-04-15T17:36:54.400074Z","iopub.status.idle":"2023-04-15T17:36:54.428295Z","shell.execute_reply.started":"2023-04-15T17:36:54.400037Z","shell.execute_reply":"2023-04-15T17:36:54.427219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen  = ImageDataGenerator(\n    rescale=1./255,\n    rotation_range=20,\n    zoom_range=0.10,\n    channel_shift_range=0.7,\n    width_shift_range=0.15,\n    height_shift_range=0.15,\n    shear_range=0.15,\n    horizontal_flip=True,\n    fill_mode='nearest'\n) \n\n# Note that the validation data should not be augmented!\ntest_datagen = ImageDataGenerator(rescale=1./255)","metadata":{"id":"fnaEq7KQi9Xy","execution":{"iopub.status.busy":"2023-04-15T17:36:54.429988Z","iopub.execute_input":"2023-04-15T17:36:54.430676Z","iopub.status.idle":"2023-04-15T17:36:54.437121Z","shell.execute_reply.started":"2023-04-15T17:36:54.430636Z","shell.execute_reply":"2023-04-15T17:36:54.436009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = train_datagen .flow_from_dataframe(\n        dataframe=X_train, \n        x_col='path',\n        y_col = 'classname', \n        class_mode='categorical',\n        # All images will be resized to 224*224\n        target_size=(224, 224),\n        batch_size=64,\n)\n\nvalidation_generator = test_datagen .flow_from_dataframe(\n        dataframe=X_val,\n        x_col = 'path', \n        y_col = 'classname', \n        class_mode='categorical',\n        # All images will be resized to 224*224\n        target_size=(224, 224),\n        batch_size=64,\n)","metadata":{"id":"Yh4l7vYRi_Ed","outputId":"429fc25d-2f82-49cd-b9df-acc1f1ff852b","execution":{"iopub.status.busy":"2023-04-15T17:36:54.438819Z","iopub.execute_input":"2023-04-15T17:36:54.439624Z","iopub.status.idle":"2023-04-15T17:37:42.470707Z","shell.execute_reply.started":"2023-04-15T17:36:54.439577Z","shell.execute_reply":"2023-04-15T17:37:42.469755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for data_batch, labels_batch in train_generator:\n    print('data batch shape:', data_batch.shape)\n    print('labels batch shape:', labels_batch.shape)\n    break","metadata":{"id":"E8_TkC8Vi-4u","outputId":"f3e7732d-1078-4c9c-8c6c-e53f99276036","execution":{"iopub.status.busy":"2023-04-15T17:37:42.473608Z","iopub.execute_input":"2023-04-15T17:37:42.474101Z","iopub.status.idle":"2023-04-15T17:37:43.826397Z","shell.execute_reply.started":"2023-04-15T17:37:42.474059Z","shell.execute_reply":"2023-04-15T17:37:43.824458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n          train_generator,\n          steps_per_epoch = len(X_train) // 64,\n          epochs=5,\n          validation_data=validation_generator,\n          validation_steps=len(X_val) // 64)","metadata":{"id":"MoUoua0-i-x6","outputId":"8f720d6e-741d-444e-c76e-8dad1880d0e7","execution":{"iopub.status.busy":"2023-04-15T17:37:51.258956Z","iopub.execute_input":"2023-04-15T17:37:51.259907Z","iopub.status.idle":"2023-04-15T18:08:45.371852Z","shell.execute_reply.started":"2023-04-15T17:37:51.259853Z","shell.execute_reply":"2023-04-15T18:08:45.370742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'bo', label='Training accuracy')\nplt.plot(epochs, val_acc, 'b', label='Validation accuracy')\nplt.title('Training and validation accuracy')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, 'bo', label='Training loss')\nplt.plot(epochs, val_loss, 'b', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"id":"j3xX_ofSnK8_","outputId":"2b5567d4-2e2d-45ec-c861-d124f1823674","execution":{"iopub.status.busy":"2023-04-15T18:08:52.102501Z","iopub.execute_input":"2023-04-15T18:08:52.103501Z","iopub.status.idle":"2023-04-15T18:08:52.565693Z","shell.execute_reply.started":"2023-04-15T18:08:52.103444Z","shell.execute_reply":"2023-04-15T18:08:52.564748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using VGG19 Pretrained Model","metadata":{"id":"tB6-yTGOMsQm"}},{"cell_type":"code","source":"vgg19 = VGG19(weights='imagenet',\n                  include_top=False,\n                  input_shape=(224,224,3))","metadata":{"id":"SKsfT7Z8nNuZ","outputId":"70e51e60-4a63-4f94-8230-a513fd2d9915","execution":{"iopub.status.busy":"2023-04-15T22:59:45.013308Z","iopub.execute_input":"2023-04-15T22:59:45.014239Z","iopub.status.idle":"2023-04-15T22:59:50.738139Z","shell.execute_reply.started":"2023-04-15T22:59:45.014188Z","shell.execute_reply":"2023-04-15T22:59:50.737099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg19.trainable = False\nvgg19.summary()","metadata":{"id":"0p0Z8FEMMzLp","outputId":"de4b1215-8c66-4a75-c16e-1a46c6c4ed3a","execution":{"iopub.status.busy":"2023-04-15T18:09:09.650697Z","iopub.execute_input":"2023-04-15T18:09:09.651437Z","iopub.status.idle":"2023-04-15T18:09:09.724146Z","shell.execute_reply.started":"2023-04-15T18:09:09.651402Z","shell.execute_reply":"2023-04-15T18:09:09.723228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last = vgg19.output\nx = layers.Flatten()(last)\nx = layers.Dropout(0.3)(x)\noutput = layers.Dense(10, activation='softmax')(x)\nmodel = models.Model(vgg19.input, output)\n\nmodel.summary()","metadata":{"id":"UCKTV-xbM4Wc","outputId":"efdc59e9-87ef-4d85-f1b4-9442c8b22f3a","execution":{"iopub.status.busy":"2023-04-15T23:00:04.806453Z","iopub.execute_input":"2023-04-15T23:00:04.807607Z","iopub.status.idle":"2023-04-15T23:00:04.896676Z","shell.execute_reply.started":"2023-04-15T23:00:04.807554Z","shell.execute_reply":"2023-04-15T23:00:04.895924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen  = ImageDataGenerator(\n    preprocessing_function= preprocess_input,\n    rotation_range=20,\n    zoom_range=0.10,\n    channel_shift_range=0.7,\n    width_shift_range=0.15,\n    height_shift_range=0.15,\n    shear_range=0.15,\n    horizontal_flip=True,\n    fill_mode='nearest'\n) \n\n# Note that the validation data should not be augmented!\ntest_datagen = ImageDataGenerator(preprocessing_function= preprocess_input)","metadata":{"id":"13SOrX5mM45i","execution":{"iopub.status.busy":"2023-04-15T18:09:16.916711Z","iopub.execute_input":"2023-04-15T18:09:16.917441Z","iopub.status.idle":"2023-04-15T18:09:16.923408Z","shell.execute_reply.started":"2023-04-15T18:09:16.917403Z","shell.execute_reply":"2023-04-15T18:09:16.922258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = train_datagen.flow_from_dataframe(\n        dataframe=X_train, \n        x_col='path',\n        y_col = 'classname', \n        class_mode='categorical',\n        # All images will be resized to 224*224\n        target_size=(224, 224),\n        batch_size=64,\n)\n\nvalidation_generator = test_datagen.flow_from_dataframe(\n        dataframe=X_val,\n        x_col = 'path', \n        y_col = 'classname', \n        class_mode='categorical',\n        # All images will be resized to 224*224\n        target_size=(224, 224),\n        batch_size=64,\n)","metadata":{"id":"iGyqGbquNmlG","outputId":"2ac5f0f0-309b-4306-90ac-fd1853cb8e5f","execution":{"iopub.status.busy":"2023-04-15T18:09:19.043683Z","iopub.execute_input":"2023-04-15T18:09:19.044361Z","iopub.status.idle":"2023-04-15T18:09:25.773147Z","shell.execute_reply.started":"2023-04-15T18:09:19.044323Z","shell.execute_reply":"2023-04-15T18:09:25.772180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer= optimizers.Adam(learning_rate = 0.001),\n              loss='categorical_crossentropy',\n              metrics=['accuracy'])","metadata":{"id":"CJlDlAN6O_Uf","execution":{"iopub.status.busy":"2023-04-15T18:09:31.656295Z","iopub.execute_input":"2023-04-15T18:09:31.656654Z","iopub.status.idle":"2023-04-15T18:09:31.668811Z","shell.execute_reply.started":"2023-04-15T18:09:31.656622Z","shell.execute_reply":"2023-04-15T18:09:31.667664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = ModelCheckpoint('pre_model.h5', monitor='val_accuracy', save_best_only=True, mode='max')\nmy_callbacks = [EarlyStopping(patience = 3, restore_best_weights= True), checkpoint]","metadata":{"id":"HtgQeKVYdyvS","execution":{"iopub.status.busy":"2023-04-15T18:09:34.125389Z","iopub.execute_input":"2023-04-15T18:09:34.126104Z","iopub.status.idle":"2023-04-15T18:09:34.131613Z","shell.execute_reply.started":"2023-04-15T18:09:34.126063Z","shell.execute_reply":"2023-04-15T18:09:34.130174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n          train_generator,\n          steps_per_epoch = len(X_train) // 64,\n          epochs=7,\n          validation_data=validation_generator,\n          validation_steps=len(X_val) // 64,\n          callbacks = my_callbacks)","metadata":{"id":"4KcrkqXlUe1F","outputId":"765b4e24-7559-4ed2-c565-dcab07b1251e","execution":{"iopub.status.busy":"2023-04-15T18:09:38.101619Z","iopub.execute_input":"2023-04-15T18:09:38.102888Z","iopub.status.idle":"2023-04-15T18:51:36.604011Z","shell.execute_reply.started":"2023-04-15T18:09:38.102835Z","shell.execute_reply":"2023-04-15T18:51:36.602888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine tune VGG19","metadata":{"id":"RQq-AkvXYvkV"}},{"cell_type":"code","source":"# def lr_scheduler(epoch):\n#     initial_lr = 1e-5\n#     drop = 0.1\n#     epochs_drop = 7\n#     lr = initial_lr * drop**((epoch)//epochs_drop)\n#     return lr\n\n# reduce_lr = LearningRateScheduler(lr_scheduler)","metadata":{"execution":{"iopub.status.busy":"2023-04-14T22:18:24.029930Z","iopub.execute_input":"2023-04-14T22:18:24.030956Z","iopub.status.idle":"2023-04-14T22:18:24.039277Z","shell.execute_reply.started":"2023-04-14T22:18:24.030915Z","shell.execute_reply":"2023-04-14T22:18:24.038072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = ModelCheckpoint('best_model.h5', monitor='val_accuracy', save_best_only=True, mode='max')\nmy_callbacks = [EarlyStopping(patience = 3, restore_best_weights= True), checkpoint, ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience = 1, min_lr=1e-7)]","metadata":{"id":"zgKu95RGZ1V4","execution":{"iopub.status.busy":"2023-04-15T19:03:38.978222Z","iopub.execute_input":"2023-04-15T19:03:38.978835Z","iopub.status.idle":"2023-04-15T19:03:38.986560Z","shell.execute_reply.started":"2023-04-15T19:03:38.978770Z","shell.execute_reply":"2023-04-15T19:03:38.985554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.trainable = True\nmodel.summary()","metadata":{"id":"K9LlwmGmVgni","outputId":"535faa3f-5d20-4ba5-c709-8f8c8437b5c5","execution":{"iopub.status.busy":"2023-04-15T19:03:39.302649Z","iopub.execute_input":"2023-04-15T19:03:39.303371Z","iopub.status.idle":"2023-04-15T19:03:39.352005Z","shell.execute_reply.started":"2023-04-15T19:03:39.303330Z","shell.execute_reply":"2023-04-15T19:03:39.351222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer= optimizers.Adam(learning_rate = 1e-5),\n                    loss='categorical_crossentropy',\n                    metrics=['accuracy'])","metadata":{"id":"AZFPRc6QWOay","execution":{"iopub.status.busy":"2023-04-15T19:03:42.344952Z","iopub.execute_input":"2023-04-15T19:03:42.346059Z","iopub.status.idle":"2023-04-15T19:03:42.358977Z","shell.execute_reply.started":"2023-04-15T19:03:42.346011Z","shell.execute_reply":"2023-04-15T19:03:42.357806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n                    train_generator,\n                    steps_per_epoch = len(X_train) // 64,\n                    epochs=30,\n                    validation_data=validation_generator,\n                    validation_steps=len(X_val) // 64,\n                    callbacks = my_callbacks,\n                    )","metadata":{"id":"MX0nNSHJVn0i","outputId":"4c7f0782-8e4d-4b78-83cd-b08656b5cf0c","execution":{"iopub.status.busy":"2023-04-15T19:03:43.935680Z","iopub.execute_input":"2023-04-15T19:03:43.936393Z","iopub.status.idle":"2023-04-15T20:38:44.494007Z","shell.execute_reply.started":"2023-04-15T19:03:43.936356Z","shell.execute_reply":"2023-04-15T20:38:44.492784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'bo', label='Training accuracy')\nplt.plot(epochs, val_acc, 'r', label='Validation accuracy')\nplt.title('Training and validation accuracy for VGG19')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, 'bo', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Training and validation loss for VGG19')\nplt.legend()\n\nplt.show()","metadata":{"id":"4Icy4syDRZd4","outputId":"81ebdf07-5d57-4700-d195-9b50b90126b7","execution":{"iopub.status.busy":"2023-04-15T20:59:36.787539Z","iopub.execute_input":"2023-04-15T20:59:36.788270Z","iopub.status.idle":"2023-04-15T20:59:37.236712Z","shell.execute_reply.started":"2023-04-15T20:59:36.788233Z","shell.execute_reply":"2023-04-15T20:59:37.235712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing VGG19 intermediate activations","metadata":{"id":"FYzKqZt97Z31"}},{"cell_type":"code","source":"# Step 1: we will load the VGG19 model as follows:\n# model = load_model('/kaggle/input/state-farm-distracted-driver-detection/best_model.h5')\nmodel.summary()  # As a reminder.","metadata":{"id":"t6zkf2jMdCO5","outputId":"3f4ac8b2-f988-4357-c148-57ebc9180776","execution":{"iopub.status.busy":"2023-04-15T21:04:36.207320Z","iopub.execute_input":"2023-04-15T21:04:36.208066Z","iopub.status.idle":"2023-04-15T21:04:36.254496Z","shell.execute_reply.started":"2023-04-15T21:04:36.208024Z","shell.execute_reply":"2023-04-15T21:04:36.253718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"layer1 = models.Model(inputs=model.input, outputs=model.get_layer('block1_conv1').output)\nlayer2 = models.Model(inputs=model.input, outputs=model.get_layer('block3_conv1').output)\nlayer3 = models.Model(inputs=model.input, outputs=model.get_layer('block5_conv4').output)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T21:05:45.994027Z","iopub.execute_input":"2023-04-15T21:05:45.995030Z","iopub.status.idle":"2023-04-15T21:05:46.012605Z","shell.execute_reply.started":"2023-04-15T21:05:45.994992Z","shell.execute_reply":"2023-04-15T21:05:46.011529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for ind, i in enumerate(classes.keys()):\n  # Make sure to resize the image to the appropriate size expected by the VGG19 model.\n\n  img_path = f\"{train_dir}/{i}/{random.sample(os.listdir(f'{train_dir}/{i}'),1)[0]}\"\n  img = image.load_img(img_path, target_size=(224, 224))\n  x = image.img_to_array(img)\n  x = np.expand_dims(x, axis=0)\n  x = preprocess_input(x)\n\n  # Get the intermediate layer outputs\n  # To get the intermediate layer outputs, we can use the Model class in Keras. we need to specify the input and output layers, and then call the predict method on the model.\n\n  layer1_activations = layer1.predict(x, verbose= 0)\n  layer2_activations = layer2.predict(x, verbose= 0)\n  layer3_activations = layer3.predict(x, verbose= 0) \n\n  print('\\n\\n','*' * 60, classes[i].capitalize(), '*' * 60, '\\n\\n')\n  plt.figure(figsize=(20, 6))\n\n  # Visualize the intermediate activations\n  # Finally, you can visualize the intermediate activations using any visualization tool you prefer, such as Matplotlib. You can choose a layer to visualize and plot the activations as follows:\n\n  for i in range(10):\n    \n    plt.subplot(3, 10, i+1)\n    plt.imshow(layer2_activations[0, :, :, i], cmap='viridis')\n    plt.xticks([])\n    plt.yticks([])\n\n    plt.subplot(3, 10, i+11)\n    plt.imshow(layer1_activations[0, :, :, 10 + i], cmap='viridis')\n    plt.xticks([])\n    plt.yticks([])\n\n    plt.subplot(3, 10, i + 21)\n    plt.imshow(layer3_activations[0, :, :, 20 + i], cmap='viridis')\n    plt.xticks([])\n    plt.yticks([])\n  plt.show()","metadata":{"id":"iMCOYuLemHDD","outputId":"e0dce0fd-25e9-4920-ae45-a99cab30b0b8","execution":{"iopub.status.busy":"2023-04-15T21:05:47.524172Z","iopub.execute_input":"2023-04-15T21:05:47.524567Z","iopub.status.idle":"2023-04-15T21:06:02.795668Z","shell.execute_reply.started":"2023-04-15T21:05:47.524531Z","shell.execute_reply":"2023-04-15T21:06:02.794796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing heatmaps of class activation (Grad-CAM)\n","metadata":{"id":"i7yvObIMSnZS"}},{"cell_type":"code","source":"model.summary()","metadata":{"id":"5FFaxvdVLo5r","outputId":"4ad098e6-b29e-48fa-ee6a-7c86d6c89d25","execution":{"iopub.status.busy":"2023-04-15T21:06:10.443254Z","iopub.execute_input":"2023-04-15T21:06:10.443716Z","iopub.status.idle":"2023-04-15T21:06:10.492192Z","shell.execute_reply.started":"2023-04-15T21:06:10.443675Z","shell.execute_reply":"2023-04-15T21:06:10.491295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_path = f\"{train_dir}/c7/{random.sample(os.listdir(f'{train_dir}/c7'),1)[0]}\"\nimg = image.load_img(img_path, target_size=(224, 224))\nimg","metadata":{"id":"FAI0qQDsr2wy","outputId":"da93c9fc-2581-4251-da63-798a8d8b81ea","execution":{"iopub.status.busy":"2023-04-15T21:13:58.310493Z","iopub.execute_input":"2023-04-15T21:13:58.310892Z","iopub.status.idle":"2023-04-15T21:13:58.341789Z","shell.execute_reply.started":"2023-04-15T21:13:58.310856Z","shell.execute_reply":"2023-04-15T21:13:58.340707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = image.img_to_array(img)\nx = np.expand_dims(x, axis=0)\nx = preprocess_input(x)","metadata":{"id":"c4Tttx5PM5iG","execution":{"iopub.status.busy":"2023-04-15T21:14:01.062698Z","iopub.execute_input":"2023-04-15T21:14:01.063916Z","iopub.status.idle":"2023-04-15T21:14:01.071186Z","shell.execute_reply.started":"2023-04-15T21:14:01.063860Z","shell.execute_reply":"2023-04-15T21:14:01.070003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(x, verbose= 0)\nprint('Predicted:', classes[f'c{preds.argmax()}'], f', probability: {(preds.max()) * 100}%')","metadata":{"id":"GxXNkxz0sLDQ","outputId":"c0a47ba8-3d5e-4746-e8ed-39647e1fa866","execution":{"iopub.status.busy":"2023-04-15T21:14:02.141958Z","iopub.execute_input":"2023-04-15T21:14:02.143278Z","iopub.status.idle":"2023-04-15T21:14:02.212699Z","shell.execute_reply.started":"2023-04-15T21:14:02.143233Z","shell.execute_reply":"2023-04-15T21:14:02.211579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To visualize which parts of our image were the most related to the predicted class,  let's set up the Grad-CAM process.","metadata":{"id":"e8wfucvSwBkt"}},{"cell_type":"code","source":"def Heatmap(class_name, outputs):\n    fig = plt.figure(figsize=(20, 8))\n    print('\\n\\n','*' * 60, classes[class_name].capitalize(), '*' * 60, '\\n\\n')\n    for i in range(10):\n      plt.subplot(2, 5, i + 1)\n      plt.imshow(outputs[i])\n      plt.xticks([])\n      plt.yticks([])\n    plt.show()","metadata":{"id":"d0Tdz2_JQTe7","execution":{"iopub.status.busy":"2023-04-15T21:14:09.635551Z","iopub.execute_input":"2023-04-15T21:14:09.635932Z","iopub.status.idle":"2023-04-15T21:14:09.643491Z","shell.execute_reply.started":"2023-04-15T21:14:09.635898Z","shell.execute_reply":"2023-04-15T21:14:09.640896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, class_name in enumerate(classes.keys()):\n  output_data = []\n  for i in range(10):\n      # Load an image to test the model on\n      img_path = f\"{train_dir}/{class_name}/{random.sample(os.listdir(f'{train_dir}/{class_name}'),1)[0]}\"\n      img = cv2.imread(img_path)\n      img = cv2.resize(img, (224,224))\n      data = ([img], None)\n\n      # Use the GradCAM to generate the heatmap\n      explainer = GradCAM()\n      output = explainer.explain(validation_data = data, model = model, class_index = idx, layer_name = 'block5_conv4')\n      output_data.append(output)\n  Heatmap(class_name, output_data)","metadata":{"id":"xjErF4OLPoFT","outputId":"35c264be-92b8-4dc1-cef7-3ba72b3fb38b","execution":{"iopub.status.busy":"2023-04-15T21:16:44.104784Z","iopub.execute_input":"2023-04-15T21:16:44.105705Z","iopub.status.idle":"2023-04-15T21:17:05.290189Z","shell.execute_reply.started":"2023-04-15T21:16:44.105652Z","shell.execute_reply":"2023-04-15T21:17:05.289173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{"id":"IoPEyOP2bLe5"}},{"cell_type":"code","source":"df_test['img']","metadata":{"execution":{"iopub.status.busy":"2023-04-15T23:16:34.348565Z","iopub.execute_input":"2023-04-15T23:16:34.348944Z","iopub.status.idle":"2023-04-15T23:16:34.377404Z","shell.execute_reply.started":"2023-04-15T23:16:34.348912Z","shell.execute_reply":"2023-04-15T23:16:34.376265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['path'] = '/kaggle/input/state-farm-distracted-driver-detection/imgs/test' + '/' + df_test['img']","metadata":{"execution":{"iopub.status.busy":"2023-04-15T23:17:38.513712Z","iopub.execute_input":"2023-04-15T23:17:38.514441Z","iopub.status.idle":"2023-04-15T23:17:38.536222Z","shell.execute_reply.started":"2023-04-15T23:17:38.514403Z","shell.execute_reply":"2023-04-15T23:17:38.535067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64","metadata":{"execution":{"iopub.status.busy":"2023-04-15T23:38:03.877549Z","iopub.execute_input":"2023-04-15T23:38:03.878513Z","iopub.status.idle":"2023-04-15T23:38:03.883833Z","shell.execute_reply.started":"2023-04-15T23:38:03.878462Z","shell.execute_reply":"2023-04-15T23:38:03.882552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_datagen = ImageDataGenerator(preprocessing_function= preprocess_input)\ntest_generator = test_datagen.flow_from_dataframe(\n        dataframe=df_test,\n        x_col = 'path', \n        y_col = None,\n        class_mode=None,\n        # All images will be resized to 224*224\n        target_size=(224, 224),\n        batch_size=batch_size,\n        shuffle = False\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T23:33:30.924474Z","iopub.execute_input":"2023-04-15T23:33:30.925540Z","iopub.status.idle":"2023-04-15T23:36:17.748356Z","shell.execute_reply.started":"2023-04-15T23:33:30.925494Z","shell.execute_reply":"2023-04-15T23:36:17.747224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(test_generator, steps = len(df_test) // batch_size + 1)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T23:38:26.716340Z","iopub.execute_input":"2023-04-15T23:38:26.716832Z","iopub.status.idle":"2023-04-15T23:54:16.894972Z","shell.execute_reply.started":"2023-04-15T23:38:26.716784Z","shell.execute_reply":"2023-04-15T23:54:16.893909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-15T23:54:36.246379Z","iopub.execute_input":"2023-04-15T23:54:36.247576Z","iopub.status.idle":"2023-04-15T23:54:36.255376Z","shell.execute_reply.started":"2023-04-15T23:54:36.247524Z","shell.execute_reply":"2023-04-15T23:54:36.254091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.drop(columns = 'path', inplace = True)","metadata":{"id":"AFnnb77HA6Nu","execution":{"iopub.status.busy":"2023-04-15T23:55:15.203853Z","iopub.execute_input":"2023-04-15T23:55:15.204827Z","iopub.status.idle":"2023-04-15T23:55:15.225039Z","shell.execute_reply.started":"2023-04-15T23:55:15.204789Z","shell.execute_reply":"2023-04-15T23:55:15.224026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.iloc[:, 1:] = predictions","metadata":{"execution":{"iopub.status.busy":"2023-04-15T23:56:25.534855Z","iopub.execute_input":"2023-04-15T23:56:25.535478Z","iopub.status.idle":"2023-04-15T23:56:25.685973Z","shell.execute_reply.started":"2023-04-15T23:56:25.535440Z","shell.execute_reply":"2023-04-15T23:56:25.684917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-15T23:56:34.714905Z","iopub.execute_input":"2023-04-15T23:56:34.715457Z","iopub.status.idle":"2023-04-15T23:56:34.741058Z","shell.execute_reply.started":"2023-04-15T23:56:34.715408Z","shell.execute_reply":"2023-04-15T23:56:34.740003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T23:56:43.731393Z","iopub.execute_input":"2023-04-15T23:56:43.731754Z","iopub.status.idle":"2023-04-15T23:56:44.834926Z","shell.execute_reply.started":"2023-04-15T23:56:43.731723Z","shell.execute_reply":"2023-04-15T23:56:44.833890Z"},"trusted":true},"execution_count":null,"outputs":[]}]}