{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5048,"databundleVersionId":868335,"sourceType":"competition"}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import libraries","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\nimport os\nimport random\nimport pandas as pd\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-15T15:25:42.407537Z","iopub.execute_input":"2024-01-15T15:25:42.408401Z","iopub.status.idle":"2024-01-15T15:25:46.315500Z","shell.execute_reply.started":"2024-01-15T15:25:42.408362Z","shell.execute_reply":"2024-01-15T15:25:46.314724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize the data","metadata":{}},{"cell_type":"code","source":"TRAIN_DIR = \"/kaggle/input/state-farm-distracted-driver-detection/imgs/train/\"\nlabels = ['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9']\n\n#randomly selecting one image per class\n\nimg_c0_path = TRAIN_DIR + \"c0/\" + random.choice(os.listdir(TRAIN_DIR + \"c0/\"))\nimg_c1_path = TRAIN_DIR + \"c1/\" + random.choice(os.listdir(TRAIN_DIR + \"c1/\"))\nimg_c2_path = TRAIN_DIR + \"c2/\" + random.choice(os.listdir(TRAIN_DIR + \"c2/\"))\nimg_c3_path = TRAIN_DIR + \"c3/\" + random.choice(os.listdir(TRAIN_DIR + \"c3/\"))\nimg_c4_path = TRAIN_DIR + \"c4/\" + random.choice(os.listdir(TRAIN_DIR + \"c4/\"))\nimg_c5_path = TRAIN_DIR + \"c5/\" + random.choice(os.listdir(TRAIN_DIR + \"c5/\"))\nimg_c6_path = TRAIN_DIR + \"c6/\" + random.choice(os.listdir(TRAIN_DIR + \"c6/\"))\nimg_c7_path = TRAIN_DIR + \"c7/\" + random.choice(os.listdir(TRAIN_DIR + \"c7/\"))\nimg_c8_path = TRAIN_DIR + \"c8/\" + random.choice(os.listdir(TRAIN_DIR + \"c8/\"))\nimg_c9_path = TRAIN_DIR + \"c9/\" + random.choice(os.listdir(TRAIN_DIR + \"c9/\"))\n\n# reading the images\nimgs = {}\n\nimgs[\"c0\"] = cv2.cvtColor(cv2.imread(img_c0_path), cv2.COLOR_BGR2RGB)\nimgs[\"c1\"] = cv2.cvtColor(cv2.imread(img_c1_path), cv2.COLOR_BGR2RGB)\nimgs[\"c2\"] = cv2.cvtColor(cv2.imread(img_c2_path), cv2.COLOR_BGR2RGB)\nimgs[\"c3\"] = cv2.cvtColor(cv2.imread(img_c3_path), cv2.COLOR_BGR2RGB)\nimgs[\"c4\"] = cv2.cvtColor(cv2.imread(img_c4_path), cv2.COLOR_BGR2RGB)\nimgs[\"c5\"] = cv2.cvtColor(cv2.imread(img_c5_path), cv2.COLOR_BGR2RGB)\nimgs[\"c6\"] = cv2.cvtColor(cv2.imread(img_c6_path), cv2.COLOR_BGR2RGB)\nimgs[\"c7\"] = cv2.cvtColor(cv2.imread(img_c7_path), cv2.COLOR_BGR2RGB)\nimgs[\"c8\"] = cv2.cvtColor(cv2.imread(img_c8_path), cv2.COLOR_BGR2RGB)\nimgs[\"c9\"] = cv2.cvtColor(cv2.imread(img_c9_path), cv2.COLOR_BGR2RGB)\n\n# displaying the images\ndisplay_rows = 5\ndisplay_columns = 5\nfig = plt.figure(figsize=(20, 20))\n\nfor label, index in zip(labels, range(10)):\n    fig.add_subplot(display_rows, display_columns, index + 1)\n    plt.imshow(imgs[label])\n    plt.axis('off')\n    plt.title(label)\n\n#plt.savefig('data_visualization.png', bbox_inches='tight')\n","metadata":{"execution":{"iopub.status.busy":"2024-01-15T15:25:49.556070Z","iopub.execute_input":"2024-01-15T15:25:49.556701Z","iopub.status.idle":"2024-01-15T15:25:51.396120Z","shell.execute_reply.started":"2024-01-15T15:25:49.556667Z","shell.execute_reply":"2024-01-15T15:25:51.395229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Focus on head and arm","metadata":{}},{"cell_type":"code","source":"# adding focused details (head, arm) to the original image\nimg = cv2.cvtColor(cv2.imread(img_c5_path), cv2.COLOR_BGR2RGB)\n\nprint(\"original shape: \" + str(img.shape))\n\n# Split the image into two parts vertically\nimg_part1 = img[0:240, 0:320]\nimg_part2 = img[240:480, 320:640]\n\n# Check the dimensions of the parts\nprint(\"Part 1 shape:\", img_part1.shape)\nprint(\"Part 2 shape:\", img_part2.shape)\n\n# Check the data types of the parts\nprint(\"Part 1 dtype:\", img_part1.dtype)\nprint(\"Part 2 dtype:\", img_part2.dtype)\n\n# Ensure the dimensions and data types are consistent before concatenating\ndetail_concat = cv2.vconcat([img_part1, img_part2])\noriginal_resized = cv2.resize(img, (320, 480))\n\nfinal_image = cv2.hconcat([original_resized, detail_concat])\n\nplt.imshow(final_image)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T15:25:55.728768Z","iopub.execute_input":"2024-01-15T15:25:55.729146Z","iopub.status.idle":"2024-01-15T15:25:56.141370Z","shell.execute_reply.started":"2024-01-15T15:25:55.729113Z","shell.execute_reply":"2024-01-15T15:25:56.140421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FocusedDataGenerator(tf.keras.preprocessing.image.ImageDataGenerator):\n    def __init__(self, \n                **kwargs) -> None:\n\n      # Declare a processing functions\n      super().__init__(\n          preprocessing_function=self.actions,\n          **kwargs)\n    \n    def actions(self, image: np.ndarray) -> np.ndarray:\n        # Apply focus operation\n        image = self.focus(image)\n        \n        # Apply Keras preprocess_input function\n        image = tf.keras.applications.resnet50.preprocess_input(image)\n\n        return image\n\n    def focus(self, img: np.ndarray) -> np.ndarray:\n        height, width, _ = img.shape\n        img_part1 = img[0:(height // 2), 0:(width // 2)]\n        img_part2 = img[(height // 2):height, (width // 2):width]\n        detail_concat = cv2.vconcat([img_part1, img_part2])\n        original_resized = cv2.resize(img, ((width // 2), height))\n        final_image = cv2.hconcat([original_resized, detail_concat])\n        return final_image","metadata":{"execution":{"iopub.status.busy":"2024-01-15T15:26:04.194620Z","iopub.execute_input":"2024-01-15T15:26:04.195465Z","iopub.status.idle":"2024-01-15T15:26:04.203715Z","shell.execute_reply.started":"2024-01-15T15:26:04.195432Z","shell.execute_reply":"2024-01-15T15:26:04.202343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = cv2.imread(img_c1_path)\nimg = np.expand_dims(img, axis = 0)\nfocused_img = FocusedDataGenerator(rescale=1./255)\nfocused_img.fit(img)\noutput = focused_img.flow(img)[0]\nout = np.reshape(output, output.shape[1:4])\nplt.imshow(cv2.cvtColor(out, cv2.COLOR_BGR2RGB))","metadata":{"execution":{"iopub.status.busy":"2024-01-15T15:26:07.571154Z","iopub.execute_input":"2024-01-15T15:26:07.572010Z","iopub.status.idle":"2024-01-15T15:26:07.950160Z","shell.execute_reply.started":"2024-01-15T15:26:07.571975Z","shell.execute_reply":"2024-01-15T15:26:07.949266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data distribution","metadata":{}},{"cell_type":"code","source":"samples = {}\nfor label in labels:\n    samples[label] = [len(os.listdir(TRAIN_DIR + label + \"/\"))]\n\nsamples_df = pd.DataFrame.from_dict(samples)\nsns.barplot(data = samples_df)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T15:26:12.111726Z","iopub.execute_input":"2024-01-15T15:26:12.112503Z","iopub.status.idle":"2024-01-15T15:26:12.369152Z","shell.execute_reply.started":"2024-01-15T15:26:12.112472Z","shell.execute_reply":"2024-01-15T15:26:12.368047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameters","metadata":{}},{"cell_type":"code","source":"batch_size = 32\nbase_learning_rate = 0.001\nbase_momentum=0.9\ninitial_epochs = 30\nfine_tune_epochs = 40\nfine_tune_at = 4\ndrop_out_rate = 0.2\ninitial_patience = 3\nfine_tune_patience = 2","metadata":{"execution":{"iopub.status.busy":"2024-01-15T15:26:29.875835Z","iopub.execute_input":"2024-01-15T15:26:29.876789Z","iopub.status.idle":"2024-01-15T15:26:29.883435Z","shell.execute_reply.started":"2024-01-15T15:26:29.876756Z","shell.execute_reply":"2024-01-15T15:26:29.882606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data loading and augmentation","metadata":{}},{"cell_type":"code","source":"IMG_SIZE = (224, 224)\nIMG_SHAPE = IMG_SIZE + (3,)\nBASE_MODEL_LAYER = 1\n\npreprocess_input = tf.keras.applications.resnet50.preprocess_input\n\ndata_augmentation = FocusedDataGenerator(\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    fill_mode='nearest',\n    #preprocessing_function=preprocess_input, # I have to make it such that some of the augmented images are the focused versions. Not everyone. Now it would be probabilistically everyone as I am preprocessing it like this with probability = 0.5\n    validation_split=0.2\n)\n\ntraining_dataset = data_augmentation.flow_from_directory(TRAIN_DIR,\n                                                        subset=\"training\",\n                                                        seed=123,\n                                                        target_size=IMG_SIZE,\n                                                        batch_size=batch_size,\n                                                        class_mode='sparse',)\nvalidation_dataset = data_augmentation.flow_from_directory(TRAIN_DIR,\n                                                        subset=\"validation\",\n                                                        seed=123,\n                                                        target_size=IMG_SIZE,\n                                                        batch_size=batch_size,\n                                                        class_mode='sparse')","metadata":{"execution":{"iopub.status.busy":"2024-01-15T15:26:33.731012Z","iopub.execute_input":"2024-01-15T15:26:33.731791Z","iopub.status.idle":"2024-01-15T15:26:38.784197Z","shell.execute_reply.started":"2024-01-15T15:26:33.731757Z","shell.execute_reply":"2024-01-15T15:26:38.783406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transfer learning with Resnet50","metadata":{}},{"cell_type":"code","source":"base_model = tf.keras.applications.ResNet50(input_shape=IMG_SHAPE,\n                                                   include_top=False,\n                                                   weights='imagenet')\nbase_model.trainable = False\ninputs = tf.keras.Input(shape=IMG_SHAPE)\n\nx = base_model(inputs, training=False)\nx = tf.keras.layers.GlobalAveragePooling2D()(x)\nx = tf.keras.layers.Dropout(drop_out_rate)(x)\noutputs = tf.keras.layers.Dense(10, activation='softmax')(x)\nmodel = tf.keras.Model(inputs, outputs)\n\nloss_function = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n\ninitial_optimizer = tf.keras.optimizers.SGD(learning_rate=base_learning_rate,\n                                            momentum=base_momentum)\n\nmodel.compile(optimizer=initial_optimizer,\n              loss=loss_function,\n              metrics=['accuracy'])\n\nes = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=initial_patience, restore_best_weights=True)\n\nmodel.fit(training_dataset, validation_data=validation_dataset, epochs=initial_epochs, callbacks=[es])\n#model.save('tl_model.keras')","metadata":{"execution":{"iopub.status.busy":"2024-01-15T15:28:04.638071Z","iopub.execute_input":"2024-01-15T15:28:04.638935Z","iopub.status.idle":"2024-01-15T15:47:55.497594Z","shell.execute_reply.started":"2024-01-15T15:28:04.638898Z","shell.execute_reply":"2024-01-15T15:47:55.494843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fine tuning","metadata":{}},{"cell_type":"code","source":"base_model = model.layers[BASE_MODEL_LAYER]\nbase_model.trainable = True\n\n# Freeze all the layers before the `fine_tune_at` layer\nfor layer in base_model.layers[:fine_tune_at]:\n    layer.trainable = False\n\nfine_tuning_optimizer = tf.keras.optimizers.Adam(learning_rate=0.001 * base_learning_rate)\nmetrics=['accuracy']\n\nmodel.compile(loss=loss_function,\n              optimizer = fine_tuning_optimizer,\n              metrics=metrics)\n\nes = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=fine_tune_patience, restore_best_weights=True)\n\nhistory_fine = model.fit(training_dataset,\n                         epochs=fine_tune_epochs,\n                         validation_data=validation_dataset,\n                         callbacks=[es])\n\nmodel.save('fine_tuned_model.keras')","metadata":{"execution":{"iopub.status.busy":"2024-01-15T15:47:55.498368Z","iopub.status.idle":"2024-01-15T15:47:55.498737Z","shell.execute_reply.started":"2024-01-15T15:47:55.498536Z","shell.execute_reply":"2024-01-15T15:47:55.498552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Produce submission.csv","metadata":{}},{"cell_type":"code","source":"test_path = '/kaggle/input/state-farm-distracted-driver-detection/imgs/test/'\n\ntest_datagen = FocusedDataGenerator()\n\ntest_dataset = test_datagen.flow_from_directory(\n    directory='/kaggle/input/state-farm-distracted-driver-detection/imgs/',\n    target_size=IMG_SIZE,\n    batch_size=32,\n    classes=['test'],\n    class_mode=None,\n    shuffle=False\n)\n\n# Get filenames and predictions\nfile_names = sorted(os.listdir(test_path))\nlen(file_names)\n\npredictions = model.predict(test_dataset, verbose=1)\n\nfile_names, predictions = zip(*sorted(zip(file_names, predictions.tolist())))\n\n# Vectorized concatenation of file names with predictions\narr = np.column_stack([file_names, predictions])\n\n# Create a DataFrame with the desired columns and save to CSV\nsubmission_df = pd.DataFrame(arr, columns=['img', 'c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9'])\nsubmission_df.to_csv('./submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-15T15:47:55.500288Z","iopub.status.idle":"2024-01-15T15:47:55.500651Z","shell.execute_reply.started":"2024-01-15T15:47:55.500456Z","shell.execute_reply":"2024-01-15T15:47:55.500472Z"},"trusted":true},"execution_count":null,"outputs":[]}]}