{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Distracted driver detection\n\nAccording to the CDC motor vehicle safety division, one in five car accidents is caused by a distracted driver. Sadly, this translates to 425,000 people injured and 3,000 people killed by distracted driving every year.\n\n## Step 0: Defining the problem\n**Goal** : Predict the likelihood of what the driver is doing in each picture.  \n**Evaluation** : Submissions are evaluated using the multi-class logarithmic loss.  ","metadata":{}},{"cell_type":"markdown","source":"## Step 1: Setup helper functions and config variables","metadata":{}},{"cell_type":"code","source":"# Importing necessary functions and libraries\nfrom datetime import date\nimport typing as t\nimport warnings\nimport os\nimport pathlib\nimport random\nimport numpy as np\nimport numpy.typing as npt\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom sklearn import metrics\nimport tensorflow as tf\nfrom tensorflow.keras import (\n    models,\n    layers,\n    applications,\n    preprocessing,\n    optimizers,\n    callbacks,\n)\nfrom mlxtend import plotting\nfrom tqdm.auto import tqdm\n\nwarnings.simplefilter(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-07-05T23:22:29.031061Z","iopub.execute_input":"2023-07-05T23:22:29.032711Z","iopub.status.idle":"2023-07-05T23:22:29.040938Z","shell.execute_reply.started":"2023-07-05T23:22:29.032661Z","shell.execute_reply":"2023-07-05T23:22:29.039653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating utility helper functions\ndef show_random_samples(\n    directory_path: t.Union[pathlib.Path, str], label: str, num_of_samples: int = 5\n) -> None:\n    \"\"\"\n    Shows sample of images from a directory belonging to a class\n\n    Args:\n        directory_path (t.Union[pathlib.Path, str]): Path of the parent directory where `label` is located\n        label (str): Class label of the image\n        num_of_samples (int, optional): Number of samples to show. Defaults to 5.\n    \"\"\"\n    files = os.listdir(os.path.join(directory_path, label))\n    plt.figure(figsize=(10, 12))\n    plt.subplots_adjust(wspace=2)\n    for i in range(num_of_samples):\n        plt.subplot(1, num_of_samples, i + 1)\n        sample = random.choice(files)\n        sample_path = os.path.join(os.path.join(directory_path, label), sample)\n        img = mpimg.imread(sample_path)\n        imgplot = plt.imshow(img)\n        plt.axis(False)\n        plt.title(f\"{label.lower().capitalize()}\\n{img.shape}\")\n    plt.show()\n\n\ndef create_class_weight(label_count_dict: t.Dict[str, int]) -> t.Dict[int, float]:\n    \"\"\"\n    Creates class weights, optimally used for imbalanced data\n\n    Args:\n        label_count_dict (t.Dict[str, int]): Dictionary in the format {`label` : `count`}\n\n    Returns:\n        t.Dict[int, float]: Class weights dictionary in the format {`label` : `weight`}\n    \"\"\"\n    total_samples = np.sum(list(label_count_dict.values()))\n    labels = label_count_dict.keys()\n    class_weights = dict()\n\n    for idx, label in enumerate(labels):\n        class_weights[idx] = float(\n            total_samples / (len(list(labels)) * label_count_dict[label])\n        )\n\n    return class_weights\n\n\ndef create_model_checkpoint(\n    model_name: str, directory: str = \"models/\"\n) -> tf.keras.callbacks.ModelCheckpoint:\n    \"\"\"\n    Returns a model checkpoint callback ro save the best model while training\n\n    Args:\n        model_name (str): Name of the model, functions creates a directory for the model of this name\n        save_path (str, optional): Main directory to store all weights. Defaults to \"model_experiments\".\n\n    Returns:\n        tf.keras.callbacks.ModelCheckpoint: A callback to get the best model configuration according to the training.\n    \"\"\"\n    directory = pathlib.Path(directory)\n    directory.mkdir(parents=True, exist_ok=True)\n    filepath = directory / f\"{model_name}.h5\"\n    return callbacks.ModelCheckpoint(\n        filepath=filepath, verbose=1, save_best_only=True, save_weights_only=False\n    )\n\n\ndef create_early_stopping(\n    patience: int = 3, restore_best_weights: bool = True\n) -> tf.keras.callbacks.EarlyStopping:\n    \"\"\"\n    Returns a early stopping callback\n\n    Args:\n        patience (int, optional): Number of iterations to look for improvement. Defaults to 3.\n        restore_best_weights (bool, optional): Restore best weights. Defaults to True.\n\n    Returns:\n        tf.keras.callbacks.EarlyStoppin: A callback to stop training if the validation loss does not decrease.\n    \"\"\"\n    return callbacks.EarlyStopping(\n        monitor=\"val_loss\",\n        patience=patience,\n        restore_best_weights=restore_best_weights,\n    )\n\n\ndef plot_loss_curves(history: tf.keras.callbacks.History) -> None:\n    \"\"\"\n    Shows separate loss curves for training and validation metrics.\n\n    Args:\n        history: TensorFlow model History object (see: https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/History)\n    \"\"\"\n\n    loss = history.history[\"loss\"]\n    val_loss = history.history[\"val_loss\"]\n    accuracy = history.history[\"accuracy\"]\n    val_accuracy = history.history[\"val_accuracy\"]\n    epochs = range(len(history.history[\"loss\"]))\n\n    # Plot loss\n    plt.figure(figsize=(19, 7))\n    plt.subplot(1, 2, 1)\n    plt.plot(epochs, loss, label=\"training_loss\")\n    plt.plot(epochs, val_loss, label=\"val_loss\")\n    plt.title(\"Loss\")\n    plt.xlabel(\"Epochs\")\n    plt.xticks(epochs)\n    plt.legend(loc=\"upper right\")\n\n    # Plot accuracy\n    plt.subplot(1, 2, 2)\n    plt.plot(epochs, accuracy, label=\"training_accuracy\")\n    plt.plot(epochs, val_accuracy, label=\"val_accuracy\")\n    plt.title(\"Accuracy\")\n    plt.xlabel(\"Epochs\")\n    plt.xticks(epochs)\n    plt.legend(loc=\"upper left\")\n\n\ndef plot_conf_matrix(\n    y_true: npt.ArrayLike, y_pred: npt.ArrayLike, classes: list\n) -> None:\n    \"\"\"\n    Plots confusion matrix\n\n    Args:\n        y_true (npt.ArrayLike): True labels\n        y_pred (npt.ArrayLike): Pred labels\n        classes (list): List with names of classes\n    \"\"\"\n    cnf_matrix = metrics.confusion_matrix(y_true=y_true, y_pred=y_pred)\n    plotting.plot_confusion_matrix(\n        conf_mat=cnf_matrix, class_names=classes, figsize=(10, 7)\n    )\n\n\ndef get_metrics(\n    y_true: npt.ArrayLike,\n    y_pred: npt.ArrayLike,\n    model_path: pathlib.Path,\n    classes: list,\n    plot_confusion_matrix: bool = False,\n) -> t.Dict[str, float]:\n    \"\"\"\n    Generates classification metrics, optionally shows confusion matrix\n\n    Args:\n        y_true (npt.ArrayLike): True labels, make sure the arrays are flattened!\n        y_pred (npt.ArrayLike): Predicted labels, make sure the arrays are flattened!\n        plot_confusion_matrix (bool, optional): Shows confusion matrix. Defaults to False.\n\n    Raises:\n        ValueError: When shape of `y_test` and `y_pred` are not the same.\n\n    Returns:\n        t.Dict[str, float]: Classification metrics\n    \"\"\"\n    if y_true.shape != y_pred.shape:\n        raise ValueError(\"Shape mismatch error!\")\n\n    y_true, y_pred = y_true.flatten(), y_pred.flatten()\n\n    accuracy = metrics.accuracy_score(y_true, y_pred)\n    precision = metrics.precision_score(y_true, y_pred, average=\"micro\")\n    recall = metrics.recall_score(y_true, y_pred, average=\"micro\")\n    f1_score = metrics.f1_score(y_true, y_pred, average=\"micro\")\n\n    if plot_confusion_matrix:\n        plot_conf_matrix(y_true, y_pred, classes=classes)\n\n    # Model file size in MBs\n    model_size = model_path.stat().st_size // (1024**2)\n\n    return {\n        \"Accuracy\": accuracy,\n        \"Precision\": precision,\n        \"Recall\": recall,\n        \"F1-score\": f1_score,\n        \"Size (MB)\": model_size,\n    }\n\n\ndef load_and_prep_image(filename: pathlib.Path, img_shape: int = 224):\n    \"\"\"\n    Returns an image tensor by reading the image on `filepath`\n\n    Args:\n        filename (pathlib.Path): Filepath of a single image\n        img_shape (int, optional): Image dimension. This is considered as both height and width of image for resizing. Defaults to 224.\n\n    Returns:\n        tf.Tensor: Image as a tensor\n    \"\"\"\n    # Read in the image\n    img = tf.io.read_file(filename)\n    # Decode it into a tensor\n    img = tf.io.decode_jpeg(img, channels=3)\n    # Resize the image\n    img = tf.image.resize(img, [img_shape, img_shape])\n    return img\n\n\ndef create_submission_csv(\n    dataloader,\n    model: tf.keras.models.Model,\n    save_path: pathlib.Path,\n    submission_csv_path: pathlib.Path,\n) -> None:\n    \"\"\"\n    Creates a submission file `.csv` to submit to the competition\n\n    Args:\n        dataloader: dataloader to iterate over and predict\n        model (`tf.keras.models.Model`): Model to predict on\n        save_path (`pathlib.Path`): File/directory path to save the file\n        submission_csv_path(`pathlib.Path`): File path to the sample submission csv file\n\n    \"\"\"\n\n    today = date.today().strftime(\"%d_%m\")\n\n    preds = model.predict(dataloader, verbose=0)\n    print(\"[INFO] Predictions on dataloader completed.\")\n    sample_submission_file = pd.read_csv(submission_csv_path)\n    test_df = pd.DataFrame(\n        columns=[\"c0\", \"c1\", \"c2\", \"c3\", \"c4\", \"c5\", \"c6\", \"c7\", \"c8\", \"c9\"]\n    )\n\n    assert (\n        preds.shape[0] == sample_submission_file.shape[0]\n    ), f\"[ERROR] Shapes of predictions and csv mismatched\"\n\n    submission_csv = sample_submission_file.drop(\n        sample_submission_file.columns.difference([\"img\"]), axis=1\n    )\n\n    for idx in tqdm(range(preds.shape[0])):\n        test_df = test_df.append(\n            pd.Series(\n                preds[idx].tolist(), index=test_df.columns[: len(preds[idx].tolist())]\n            ),\n            ignore_index=True,\n        )\n\n    submission_csv = pd.concat([submission_csv, test_df], axis=1)\n\n    save_path.mkdir(parents=True, exist_ok=True)\n    save_file_path = save_path / f\"{model.name}_{today}.csv\"\n    print(\n        f\"[INFO] Saving submissions... File located at {save_path}/{model.name}_{today}.csv\"\n    )\n    submission_csv.to_csv(save_file_path, index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T23:22:29.044227Z","iopub.execute_input":"2023-07-05T23:22:29.045052Z","iopub.status.idle":"2023-07-05T23:22:29.092871Z","shell.execute_reply.started":"2023-07-05T23:22:29.045006Z","shell.execute_reply":"2023-07-05T23:22:29.091794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining config variables\nclass CONFIG:\n    train_path = pathlib.Path(\n        \"/kaggle/input/state-farm-distracted-driver-detection/imgs/train\"\n    )\n    test_path = pathlib.Path(\n        \"/kaggle/input/state-farm-distracted-driver-detection/imgs/test\"\n    )\n    classes = [\n        \"c0\",\n        \"c1\",\n        \"c2\",\n        \"c3\",\n        \"c4\",\n        \"c5\",\n        \"c6\",\n        \"c7\",\n        \"c8\",\n        \"c9\",\n    ]\n    num_classes = len(classes)\n    image_size = 224\n    verbose = 1\n    batch_size = 32\n    autotune = tf.data.AUTOTUNE\n    seed = 42\n    submission_dir = pathlib.Path(\"submissions\")\n    sample_submission_path = pathlib.Path(\n        \"/kaggle/input/state-farm-distracted-driver-detection/sample_submission.csv\"\n    )","metadata":{"execution":{"iopub.status.busy":"2023-07-05T23:22:29.095242Z","iopub.execute_input":"2023-07-05T23:22:29.095757Z","iopub.status.idle":"2023-07-05T23:22:29.106069Z","shell.execute_reply.started":"2023-07-05T23:22:29.095715Z","shell.execute_reply":"2023-07-05T23:22:29.105085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Step 2: Knowing more about data\n- Display an example of the data\n- Get the statistics of the data\n- Check for data imbalance for training dataset\n- (If required) Declare weights for model training, incase of data imbalance\n- (If required) Create a data augmentation pipeline\n- Make data processing faster for modelling using dataset generators","metadata":{}},{"cell_type":"code","source":"# Display random example of data\nfor idx, label in enumerate(CONFIG.classes):\n    show_random_samples(CONFIG.train_path, label)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T23:22:29.108913Z","iopub.execute_input":"2023-07-05T23:22:29.109291Z","iopub.status.idle":"2023-07-05T23:22:34.643954Z","shell.execute_reply.started":"2023-07-05T23:22:29.109248Z","shell.execute_reply":"2023-07-05T23:22:34.642949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Displaying a sample from image to get the statistics\nimage = mpimg.imread(\n    \"/kaggle/input/state-farm-distracted-driver-detection/imgs/train/c3/img_100006.jpg\"\n)\nprint(f\"Image size: {image.shape}\")\nprint(f\"Maximum pixel value in the image: {tf.math.reduce_max(image)}\")\nplt.imshow(image)\nplt.axis(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T23:22:34.645604Z","iopub.execute_input":"2023-07-05T23:22:34.646272Z","iopub.status.idle":"2023-07-05T23:22:37.35933Z","shell.execute_reply.started":"2023-07-05T23:22:34.646219Z","shell.execute_reply":"2023-07-05T23:22:37.358432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the statistics about the data\ndata_count_dict = {}\nfor label in CONFIG.classes:\n    data_count = len(os.listdir(os.path.join(CONFIG.train_path, label)))\n    data_count_dict[label] = data_count\n\ndata_count_df = pd.DataFrame.from_dict(data_count_dict, orient=\"index\").rename(\n    columns={0: \"No. of files\"}\n)\ndata_count_df.plot(kind=\"bar\");","metadata":{"execution":{"iopub.status.busy":"2023-07-05T23:22:37.362434Z","iopub.execute_input":"2023-07-05T23:22:37.363604Z","iopub.status.idle":"2023-07-05T23:22:37.64826Z","shell.execute_reply.started":"2023-07-05T23:22:37.363565Z","shell.execute_reply":"2023-07-05T23:22:37.647245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To make data processing faster\ndatagen = preprocessing.image.ImageDataGenerator(\n    zoom_range=0.2, rotation_range=20, horizontal_flip=True, validation_split=0.15\n)\n\ntrain_ds = datagen.flow_from_directory(\n    directory=CONFIG.train_path,\n    target_size=(CONFIG.image_size, CONFIG.image_size),\n    batch_size=CONFIG.batch_size,\n    class_mode=\"sparse\",\n    seed=CONFIG.seed,\n    subset=\"training\",\n)\n\nval_ds = datagen.flow_from_directory(\n    directory=CONFIG.train_path,\n    target_size=(CONFIG.image_size, CONFIG.image_size),\n    batch_size=CONFIG.batch_size,\n    class_mode=\"sparse\",\n    seed=CONFIG.seed,\n    subset=\"validation\",\n    shuffle=False,\n)\n\ntest_dataloader = tf.data.Dataset.list_files(str(CONFIG.test_path / \"*.jpg\"), shuffle=False)\ntest_dataloader = (\n    test_dataloader.map(\n        lambda x: load_and_prep_image(x, CONFIG.image_size),\n        num_parallel_calls=CONFIG.autotune,\n    )\n    .batch(CONFIG.batch_size)\n    .prefetch(1)\n)\n\ntest_filepaths = []\nfor filepath in test_dataloader.list_files(str(CONFIG.test_path / \"*.jpg\"), shuffle=False):\n    test_filepaths.append(filepath.numpy().decode(\"UTF-8\"))\n\ny_val_labels = val_ds.classes\nclass_indices = dict(map(reversed, val_ds.class_indices.items()))\nprint(f\"[INFO] Data loaders and artifacts processed\")","metadata":{"execution":{"iopub.status.busy":"2023-07-05T23:22:37.649586Z","iopub.execute_input":"2023-07-05T23:22:37.650324Z","iopub.status.idle":"2023-07-05T23:24:12.625176Z","shell.execute_reply.started":"2023-07-05T23:22:37.650292Z","shell.execute_reply":"2023-07-05T23:24:12.62408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Step 3: Describe modelling experiments\n\n#### Process to follow when making models:\n0. Set random seeds\n1. Create the model architecture\n2. Compile the model by defining loss function and optimizer\n3. Train the model with validation split of 15%, and add appropriate callbacks\n4. Plot the loss curves\n5. Load in the best weights\n6. Evaluate performance on whole validation data\n7. Get metrics for training data\n8. Create a submission file csv\n9. Submit the file to the competition\n\n\n#### Models to be made:\n- Model 1: Fine-tuned **ResNet50**, validated on 15% training data\n- Model 2: Fine-tuned **VGG16**, validated on 15% training data\n- Model 3: Fine-tuned **InceptionResNetV2** validated on 15% training data\n- Model 4: Fine-tuned **EfficientNetB0**, validated on 15% training data\n- Model 5: Fine-tuned **EfficientNetB5** validated on 15% training data","metadata":{}},{"cell_type":"markdown","source":"### Model 1\n***Architecture***: Fine-tuned ResNet50  \n***Optimizer***: Adam optimizer with 0.0001 learning rate  \n***Epochs*** : 50  \n***Callbacks*** : Early stopping, Model checkpointing","metadata":{}},{"cell_type":"code","source":"# 0. Set random seed\ntf.random.set_seed(CONFIG.seed)\nINPUT_SHAPE = (CONFIG.image_size, CONFIG.image_size, 3)\n\n# 1. Create the model\nbase_resnet_model = applications.resnet50.ResNet50(\n    include_top=False, weights=\"imagenet\"\n)\nbase_resnet_model.trainable = False\n\nLAYERS_TRAINALBLE = 10\nfor layer in base_resnet_model.layers[-LAYERS_TRAINALBLE:]:\n    layer.trainable = True\n\nmodel_1 = tf.keras.Sequential(\n    [\n        layers.Input(shape=INPUT_SHAPE),\n        base_resnet_model,\n        layers.GlobalAveragePooling2D(name=\"global_average_pooling_layer\"),\n        layers.Dense(1024, activation=\"relu\", name=\"fcl1\"),\n        layers.Dropout(0.2),\n        layers.Dense(256, activation=\"relu\", name=\"fcl2\"),\n        layers.Dropout(0.2),\n        layers.Dense(CONFIG.num_classes, activation=\"softmax\", name=\"output\"),\n    ],\n    name=\"model_1_resnet_ft\",\n)\n\n# 2. Compile the model\nLEARNING_RATE = 1e-3\noptim = optimizers.Adam(learning_rate=LEARNING_RATE)\nmodel_1.compile(\n    loss=\"sparse_categorical_crossentropy\", optimizer=optim, metrics=[\"accuracy\"]\n)\n\n# 3. Fit the model\nhistory_model_1 = model_1.fit(\n    train_ds,\n    epochs=10,\n    validation_data=val_ds,\n    verbose=CONFIG.verbose,\n    callbacks=[\n        create_model_checkpoint(model_name=model_1.name),\n        create_early_stopping(patience=3),\n    ],\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T23:24:12.626785Z","iopub.execute_input":"2023-07-05T23:24:12.627138Z","iopub.status.idle":"2023-07-06T00:22:45.880727Z","shell.execute_reply.started":"2023-07-05T23:24:12.627106Z","shell.execute_reply":"2023-07-06T00:22:45.879556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 4. Plot model training performance\nplot_loss_curves(history_model_1)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T00:22:45.882597Z","iopub.execute_input":"2023-07-06T00:22:45.882973Z","iopub.status.idle":"2023-07-06T00:22:46.342225Z","shell.execute_reply.started":"2023-07-06T00:22:45.882941Z","shell.execute_reply":"2023-07-06T00:22:46.341109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 5. Loading the best weights\nbest_weights_path_model_1 = \"/kaggle/working/models/model_1_resnet_ft.h5\"\nmodel_1 = models.load_model(best_weights_path_model_1)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T00:22:46.34421Z","iopub.execute_input":"2023-07-06T00:22:46.344635Z","iopub.status.idle":"2023-07-06T00:22:48.924195Z","shell.execute_reply.started":"2023-07-06T00:22:46.344594Z","shell.execute_reply":"2023-07-06T00:22:48.92312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 6. Evaluate on test data\nmodel_1.evaluate(val_ds)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T00:22:48.92626Z","iopub.execute_input":"2023-07-06T00:22:48.927049Z","iopub.status.idle":"2023-07-06T00:23:42.224997Z","shell.execute_reply.started":"2023-07-06T00:22:48.927005Z","shell.execute_reply":"2023-07-06T00:23:42.224068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 7. Get the model's performance metrics\nmodel_1_pred_probs = model_1.predict(val_ds, verbose=1)\nmodel_1_pred_classes = model_1_pred_probs.argmax(axis=1).reshape(-1).astype(\"int\")\n\nmodel_1_metrics = get_metrics(\n    y_true=model_1_pred_classes,\n    y_pred=y_val_labels,\n    model_path=pathlib.Path(best_weights_path_model_1),\n    plot_confusion_matrix=True,\n    classes=CONFIG.classes,\n)\nmodel_1_metrics","metadata":{"execution":{"iopub.status.busy":"2023-07-06T00:23:42.226461Z","iopub.execute_input":"2023-07-06T00:23:42.227804Z","iopub.status.idle":"2023-07-06T00:24:34.872542Z","shell.execute_reply.started":"2023-07-06T00:23:42.227762Z","shell.execute_reply":"2023-07-06T00:24:34.871457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 8. Create submissions file\ncreate_submission_csv(\n    dataloader=test_dataloader,\n    model=model_1,\n    save_path=CONFIG.submission_dir,\n    submission_csv_path=CONFIG.sample_submission_path,\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T00:24:34.874108Z","iopub.execute_input":"2023-07-06T00:24:34.875175Z","iopub.status.idle":"2023-07-06T00:33:58.038148Z","shell.execute_reply.started":"2023-07-06T00:24:34.875131Z","shell.execute_reply":"2023-07-06T00:33:58.036973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model 2\n***Architecture***: Fine-tuned VGG16  \n***Optimizer***: Adam optimizer with 0.0001 learning rate  \n***Epochs*** : 50  \n***Callbacks*** : Early stopping, Model checkpointing  ","metadata":{}},{"cell_type":"code","source":"# 0. Set random seed\ntf.random.set_seed(CONFIG.seed)\nINPUT_SHAPE = (CONFIG.image_size, CONFIG.image_size, 3)\n\n# 1. Create the model\nbase_vgg_model = applications.VGG16(include_top=False, weights=\"imagenet\")\nbase_vgg_model.trainable = False\n\nLAYERS_TRAINALBLE = 10\nfor layer in base_vgg_model.layers[-LAYERS_TRAINALBLE:]:\n    layer.trainable = True\n\nmodel_2 = tf.keras.Sequential(\n    [\n        layers.Input(shape=INPUT_SHAPE),\n        base_vgg_model,\n        layers.GlobalAveragePooling2D(name=\"global_average_pooling_layer\"),\n        layers.Dense(1024, activation=\"relu\", name=\"fcl1\"),\n        layers.Dropout(0.2),\n        layers.Dense(256, activation=\"relu\", name=\"fcl2\"),\n        layers.Dropout(0.2),\n        layers.Dense(CONFIG.num_classes, activation=\"softmax\", name=\"output\"),\n    ],\n    name=\"model_2_vgg_ft\",\n)\n\n# 2. Compile the model\nLEARNING_RATE = 1e-3\noptim = optimizers.Adam(learning_rate=LEARNING_RATE)\nmodel_2.compile(\n    loss=\"sparse_categorical_crossentropy\", optimizer=optim, metrics=[\"accuracy\"]\n)\n\n# 3. Fit the model\nhistory_model_2 = model_2.fit(\n    train_ds,\n    epochs=7,\n    validation_data=val_ds,\n    verbose=CONFIG.verbose,\n    callbacks=[\n        create_model_checkpoint(model_name=model_2.name),\n        create_early_stopping(patience=3),\n    ],\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T00:33:58.042087Z","iopub.execute_input":"2023-07-06T00:33:58.042441Z","iopub.status.idle":"2023-07-06T01:13:53.799561Z","shell.execute_reply.started":"2023-07-06T00:33:58.042378Z","shell.execute_reply":"2023-07-06T01:13:53.79844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 4. Plot model training performance\nplot_loss_curves(history_model_2)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:13:53.806737Z","iopub.execute_input":"2023-07-06T01:13:53.80712Z","iopub.status.idle":"2023-07-06T01:13:54.226442Z","shell.execute_reply.started":"2023-07-06T01:13:53.807081Z","shell.execute_reply":"2023-07-06T01:13:54.225363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 5. Loading the best weights\nbest_weights_path_model_2 = \"/kaggle/working/models/model_2_vgg_ft.h5\"\nmodel_2 = models.load_model(best_weights_path_model_2)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:13:54.228036Z","iopub.execute_input":"2023-07-06T01:13:54.22919Z","iopub.status.idle":"2023-07-06T01:13:54.665039Z","shell.execute_reply.started":"2023-07-06T01:13:54.229139Z","shell.execute_reply":"2023-07-06T01:13:54.663931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 6. Evaluate on test data\nmodel_2.evaluate(val_ds)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:13:54.666597Z","iopub.execute_input":"2023-07-06T01:13:54.666988Z","iopub.status.idle":"2023-07-06T01:14:47.220629Z","shell.execute_reply.started":"2023-07-06T01:13:54.666946Z","shell.execute_reply":"2023-07-06T01:14:47.219518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 7. Get the model's performance metrics\nmodel_2_pred_probs = model_2.predict(val_ds, verbose=1)\nmodel_2_pred_classes = model_2_pred_probs.argmax(axis=1).reshape(-1).astype(\"int\")\n\nmodel_2_metrics = get_metrics(\n    y_true=model_2_pred_classes,\n    y_pred=y_val_labels,\n    model_path=pathlib.Path(best_weights_path_model_2),\n    plot_confusion_matrix=True,\n    classes=CONFIG.classes,\n)\nmodel_2_metrics","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:14:47.222723Z","iopub.execute_input":"2023-07-06T01:14:47.223131Z","iopub.status.idle":"2023-07-06T01:15:39.734495Z","shell.execute_reply.started":"2023-07-06T01:14:47.22309Z","shell.execute_reply":"2023-07-06T01:15:39.733375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 8. Create submissions file\ncreate_submission_csv(\n    dataloader=test_dataloader,\n    model=model_2,\n    save_path=CONFIG.submission_dir,\n    submission_csv_path=CONFIG.sample_submission_path,\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:15:39.736066Z","iopub.execute_input":"2023-07-06T01:15:39.738206Z","iopub.status.idle":"2023-07-06T01:22:00.213249Z","shell.execute_reply.started":"2023-07-06T01:15:39.738152Z","shell.execute_reply":"2023-07-06T01:22:00.210861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model 3\n***Architecture***: Fine-tuned Inception-ResNet V2  \n***Optimizer***: Adam optimizer with 0.001 learning rate  \n***Epochs*** : 50  \n***Callbacks*** : Early stopping, Model checkpointing  ","metadata":{}},{"cell_type":"code","source":"# 0. Set random seed\ntf.random.set_seed(CONFIG.seed)\nINPUT_SHAPE = (CONFIG.image_size, CONFIG.image_size, 3)\n\n# 1. Create the model\nbase_incep_resnet_model = applications.inception_resnet_v2.InceptionResNetV2(\n    include_top=False, weights=\"imagenet\"\n)\nbase_incep_resnet_model.trainable = False\n\nLAYERS_TRAINALBLE = 10\nfor layer in base_incep_resnet_model.layers[-LAYERS_TRAINALBLE:]:\n    layer.trainable = True\n\nmodel_3 = tf.keras.Sequential(\n    [\n        layers.Input(shape=INPUT_SHAPE),\n        base_incep_resnet_model,\n        layers.GlobalAveragePooling2D(name=\"global_average_pooling_layer\"),\n        layers.Dense(1024, activation=\"relu\", name=\"fcl1\"),\n        layers.Dropout(0.2),\n        layers.Dense(256, activation=\"relu\", name=\"fcl2\"),\n        layers.Dropout(0.2),\n        layers.Dense(CONFIG.num_classes, activation=\"softmax\", name=\"output\"),\n    ],\n    name=\"model_3_incep_resnet_ft\",\n)\n\n# 2. Compile the model\nLEARNING_RATE = 1e-3\noptim = optimizers.Adam(learning_rate=LEARNING_RATE)\nmodel_3.compile(\n    loss=\"sparse_categorical_crossentropy\", optimizer=optim, metrics=[\"accuracy\"]\n)\n\n# 3. Fit the model\nhistory_model_3 = model_3.fit(\n    train_ds,\n    epochs=7,\n    validation_data=val_ds,\n    verbose=CONFIG.verbose,\n    callbacks=[\n        create_model_checkpoint(model_name=model_3.name),\n        create_early_stopping(patience=3),\n    ],\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:22:00.218862Z","iopub.execute_input":"2023-07-06T01:22:00.219212Z","iopub.status.idle":"2023-07-06T02:03:56.648051Z","shell.execute_reply.started":"2023-07-06T01:22:00.219178Z","shell.execute_reply":"2023-07-06T02:03:56.646933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 4. Plot model training performance\nplot_loss_curves(history_model_3)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:03:56.650006Z","iopub.execute_input":"2023-07-06T02:03:56.650338Z","iopub.status.idle":"2023-07-06T02:03:57.12147Z","shell.execute_reply.started":"2023-07-06T02:03:56.650306Z","shell.execute_reply":"2023-07-06T02:03:57.120354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 5. Loading the best weights\nbest_weights_path_model_3 = \"/kaggle/working/models/model_3_incep_resnet_ft.h5\"\nmodel_3 = models.load_model(best_weights_path_model_3)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:03:57.12318Z","iopub.execute_input":"2023-07-06T02:03:57.123931Z","iopub.status.idle":"2023-07-06T02:04:06.792968Z","shell.execute_reply.started":"2023-07-06T02:03:57.12389Z","shell.execute_reply":"2023-07-06T02:04:06.79188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 6. Evaluate on test data\nmodel_3.evaluate(val_ds)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:04:06.794513Z","iopub.execute_input":"2023-07-06T02:04:06.794864Z","iopub.status.idle":"2023-07-06T02:05:32.906266Z","shell.execute_reply.started":"2023-07-06T02:04:06.794826Z","shell.execute_reply":"2023-07-06T02:05:32.905221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 7. Get the model's performance metrics\nmodel_3_pred_probs = model_3.predict(val_ds, verbose=1)\nmodel_3_pred_classes = model_3_pred_probs.argmax(axis=1).reshape(-1).astype(\"int\")\n\nmodel_3_metrics = get_metrics(\n    y_true=model_3_pred_classes,\n    y_pred=y_val_labels,\n    model_path=pathlib.Path(best_weights_path_model_3),\n    plot_confusion_matrix=True,\n    classes=CONFIG.classes,\n)\nmodel_3_metrics","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:05:32.908122Z","iopub.execute_input":"2023-07-06T02:05:32.908529Z","iopub.status.idle":"2023-07-06T02:06:29.572444Z","shell.execute_reply.started":"2023-07-06T02:05:32.908488Z","shell.execute_reply":"2023-07-06T02:06:29.571368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 8. Create submissions file\ncreate_submission_csv(\n    dataloader=test_dataloader,\n    model=model_3,\n    save_path=CONFIG.submission_dir,\n    submission_csv_path=CONFIG.sample_submission_path,\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:06:29.57429Z","iopub.execute_input":"2023-07-06T02:06:29.575085Z","iopub.status.idle":"2023-07-06T02:12:55.631024Z","shell.execute_reply.started":"2023-07-06T02:06:29.57503Z","shell.execute_reply":"2023-07-06T02:12:55.629898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Step 4: Conclude modelling experiments\n- Evalaute all models' performance\n- Get the best model","metadata":{}},{"cell_type":"code","source":"# Dataframe of results-\nresults_df = pd.DataFrame({model_1.name: model_1_metrics,\n                           model_2.name: model_2_metrics,\n                           model_3.name: model_3_metrics,}).T\n\nresults_df[\"Size (MB)\"] = results_df[\"Size (MB)\"] / results_df[\"Size (MB)\"].max()\nresults_df","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:18:49.843581Z","iopub.execute_input":"2023-07-06T02:18:49.843993Z","iopub.status.idle":"2023-07-06T02:18:49.866998Z","shell.execute_reply.started":"2023-07-06T02:18:49.843957Z","shell.execute_reply":"2023-07-06T02:18:49.865369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df[[\"Accuracy\", \"Precision\", \"Recall\", \"F1-score\", \"Size (MB)\"]].plot(kind=\"bar\", figsize=(14,7), title=\"Metrics of various models\", ylabel=\"Metrics\", xlabel=\"Models\");","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:18:56.075597Z","iopub.execute_input":"2023-07-06T02:18:56.076298Z","iopub.status.idle":"2023-07-06T02:18:56.424519Z","shell.execute_reply.started":"2023-07-06T02:18:56.076259Z","shell.execute_reply":"2023-07-06T02:18:56.423401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## References\n- https://www.kaggle.com/competitions/state-farm-distracted-driver-detection/overview\n- https://www.tensorflow.org/tutorials/images/transfer_learning","metadata":{}}]}