{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from keras.models import model_from_json\nfrom sklearn.metrics import classification_report, accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, confusion_matrix\n\n# Load the model architecture from JSON file\nwith open('second_model.json', 'r') as json_file:\n    loaded_model_json = json_file.read()\nloaded_model = model_from_json(loaded_model_json)\n\n# Load the model weights\nloaded_model.load_weights(\"second_model_weights.h5\")\n\n# Compile the model\nloaded_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n# Load the test data using the same ImageDataGenerator\ntest_dir = \"/kaggle/input/final-masked/content/drive/Shareddrives/ML_project/hotel-id-to-combat-human-trafficking-2022/train_images\"\ntest_datagen = ImageDataGenerator()\n\ntest_generator = test_datagen.flow_from_directory(\n    test_dir,\n    target_size=(224, 224),\n    batch_size=32,\n    class_mode='categorical')\n\n# Predict the labels of the test data\nY_pred = loaded_model.predict_generator(test_generator)\n\n# Convert predicted labels to class labels\ny_pred = np.argmax(Y_pred, axis=1)\n\n# Get the true labels of the test data\ny_true = test_generator.classes\n\n# Compute and print evaluation metrics\nprint(\"Accuracy Score: \", accuracy_score(y_true, y_pred))\nprint(\"Precision Score: \", precision_score(y_true, y_pred, average='weighted'))\nprint(\"Recall Score: \", recall_score(y_true, y_pred, average='weighted'))\nprint(\"F1 Score: \", f1_score(y_true, y_pred, average='weighted'))\nprint(\"AUC-ROC Score: \", roc_auc_score(Y_pred, test_generator.labels, average='macro', multi_class='ovr'))\nprint(\"\\nConfusion Matrix:\\n\", confusion_matrix(y_true, y_pred))\nprint(\"\\nClassification Report:\\n\", classification_report(y_true, y_pred, target_names=test_generator.class_indices.keys()))\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**First model evaluation metrics**","metadata":{}},{"cell_type":"code","source":"import keras\nimport numpy as np\nfrom keras.models import model_from_json\nfrom keras.preprocessing import image\n\n# Load the model architecture from the JSON file\nwith open('/kaggle/input/cnn-two-models/first_model.json', 'r') as json_file:\n    json_savedModel= json_file.read()\nmodel = model_from_json(json_savedModel)\n\n# Load the saved weights into the model\nmodel.load_weights('/kaggle/input/cnn-two-models/first_model_weights.h5')\n\n# Load the entire dataset into memory\ntest_dir = \"/kaggle/input/hotel-id-to-combat-human-trafficking-2022-fgvc9/train_images\"\ndatagen = keras.preprocessing.image.ImageDataGenerator()\n\ntest_generator = datagen.flow_from_directory(\n    test_dir,\n    target_size=(224, 224),\n    batch_size=32,\n    class_mode='categorical',\n    shuffle=False)\n\n# Evaluate the model on the entire dataset\nmodel.compile(optimizer, loss)\nscores = model.evaluate(test_generator, verbose=1)\nprint(\"Test Loss:\", scores[0])\nprint(\"Test Accuracy:\", scores[1])\n\n# Generate predictions for the entire dataset\npredictions = model.predict(test_generator, verbose=1)\n\n# Convert predictions to class labels\npredicted_classes = np.argmax(predictions, axis=1)\n\n# Get the true class labels\ntrue_classes = test_generator.classes\n\n# Get the class names\nclass_names = list(test_generator.class_indices.keys())\n\n# Compute and print the confusion matrix\ncm = sklearn.metrics.confusion_matrix(true_classes, predicted_classes)\nprint(\"Confusion Matrix:\")\nprint(cm)\n\n# Compute and print classification report\ncr = sklearn.metrics.classification_report(true_classes, predicted_classes, target_names=class_names)\nprint(\"Classification Report:\")\nprint(cr)\n\n# Compute and print other evaluation metrics\nacc = sklearn.metrics.accuracy_score(true_classes, predicted_classes)\nprec = sklearn.metrics.precision_score(true_classes, predicted_classes, average='macro')\nrecall = sklearn.metrics.recall_score(true_classes, predicted_classes, average='macro')\nf1 = sklearn.metrics.f1_score(true_classes, predicted_classes, average='macro')\nprint(\"Accuracy:\", acc)\nprint(\"Precision:\", prec)\nprint(\"Recall:\", recall)\nprint(\"F1 Score:\", f1)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install keras --upgrade","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras,os\nfrom keras.models import load_model\nfrom keras.preprocessing import image\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom keras.models import Sequential #so that all layers are arranged in sequence\nfrom keras.layers import Dense, Conv2D, MaxPool2D , Flatten\nfrom keras.models import load_model\nimport numpy as np\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\n#from keras.preprocessing.image import ImageDataGenerator\nimport numpy as np\n\n# Load the saved model\nmodel = load_model('/kaggle/input/cnn-two-models/first_model.h5')\n\n# Define the directory containing all the images\ntest_dir = '/kaggle/input/hotel-id-to-combat-human-trafficking-2022-fgvc9/train_images'\n\n# Get all the images in the test directory\nimg_filenames = os.listdir(test_dir)\n\n# Create empty lists to store the predicted labels and true labels\npredicted_labels = []\ntrue_labels = []\n\n# Loop through all the images in the test directory\nfor img_filename in img_filenames:\n    # Load the image\n    img = load_img(os.path.join(test_dir, img_filename), target_size=(224, 224))\n    # Convert the image to a numpy array\n    img_array = image.img_to_array(img)\n    # Reshape the image to match the input shape of the model\n    img_array = img_array.reshape((1, img_array.shape[0], img_array.shape[1], img_array.shape[2]))\n    # Preprocess the image\n    img_array = preprocess_input(img_array)\n    # Predict the label of the image\n    predicted_label = model.predict(img_array)\n    # Get the index of the predicted label\n    predicted_index = np.argmax(predicted_label)\n    # Append the predicted label to the predicted labels list\n    predicted_labels.append(predicted_index)\n    # Get the true label from the image filename\n    true_label = int(img_filename.split('_')[0])\n    # Append the true label to the true labels list\n    true_labels.append(true_label)\n\n# Convert the predicted labels and true labels lists to numpy arrays\npredicted_labels = np.array(predicted_labels)\ntrue_labels = np.array(true_labels)\n\n# Print the classification report and confusion matrix\nprint(classification_report(true_labels, predicted_labels))\nprint(confusion_matrix(true_labels, predicted_labels))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nfrom keras.models import load_model\nfrom keras.preprocessing.image import ImageDataGenerator, img_to_array\nfrom sklearn.metrics import classification_report, confusion_matrix\n\n# Load the model\nmodel = load_model('/kaggle/input/cnn-two-models/second_model.h5')\n\n# Load the test data\ntest_dir = '/kaggle/input/hotel-id-to-combat-human-trafficking-2022-fgvc9/train_images'\ndatagen = ImageDataGenerator(rescale=1./255)\ntest_generator = datagen.flow_from_directory(\n    test_dir,\n    target_size=(224, 224),\n    batch_size=32,\n    class_mode='categorical',\n    shuffle=False)\n\n# Make predictions on the test data\nY_pred = model.predict_generator(test_generator, test_generator.n // test_generator.batch_size+1)\ny_pred = np.argmax(Y_pred, axis=1)\ny_true = test_generator.classes\n\n# Generate evaluation metrics\nprint('Confusion Matrix')\nprint(confusion_matrix(y_true, y_pred))\nprint('Classification Report')\ntarget_names = list(test_generator.class_indices.keys())\nprint(classification_report(y_true, y_pred, target_names=target_names))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array\nfrom keras.models import Sequential, load_model\nfrom keras.layers import Dropout, Flatten, Dense\nfrom keras import applications\nfrom PIL import Image\n\n# dimensions of our images\nimg_width, img_height = 150, 150\n\n# set the path to the training, validation, and testing directories\ntrain_dir = 'data/train'\nvalidation_dir = 'data/validation'\ntest_dir = 'data/test'\n\n# number of samples used for determining the samples_per_epoch and nb_val_samples\nnb_train_samples = 2000\nnb_validation_samples = 800\nbatch_size = 16\n\n# build the VGG16 network\nmodel = applications.VGG16(include_top=False, weights='imagenet')\n\ndatagen = ImageDataGenerator(rescale=1. / 255)\n\n# load the dataset but do not label them\ngenerator_train = datagen.flow_from_directory(\n    train_dir,\n    target_size=(img_width, img_height),\n    batch_size=batch_size,\n    class_mode=None,\n    shuffle=False)\n\ngenerator_validation = datagen.flow_from_directory(\n    validation_dir,\n    target_size=(img_width, img_height),\n    batch_size=batch_size,\n    class_mode=None,\n    shuffle=False)\n\n# predict features for the training, validation, and test sets\nbottleneck_features_train = model.predict(generator_train, nb_train_samples // batch_size)\nnp.save('bottleneck_features_train.npy', bottleneck_features_train)\n\nbottleneck_features_validation = model.predict(generator_validation, nb_validation_samples // batch_size)\nnp.save('bottleneck_features_validation.npy', bottleneck_features_validation)\n\ndatagen_top = ImageDataGenerator(rescale=1./255)\n\n# load the bottleneck features and train a model on top\ntrain_data = np.load('bottleneck_features_train.npy')\ntrain_labels = np.array([0] * (nb_train_samples // 2) + [1] * (nb_train_samples // 2))\n\nvalidation_data = np.load('bottleneck_features_validation.npy')\nvalidation_labels = np.array([0] * (nb_validation_samples // 2) + [1] * (nb_validation_samples // 2))\n\nmodel_top = Sequential()\nmodel_top.add(Flatten(input_shape=train_data.shape[1:]))\nmodel_top.add(Dense(256, activation='relu'))\nmodel_top.add(Dropout(0.5))\nmodel_top.add(Dense(1, activation='sigmoid'))\n\nmodel_top.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])\n\n# train the model\nmodel_top.fit(train_data, train_labels,\n          epochs=epochs,\n          batch_size=batch_size,\n          validation_data=(validation_data, validation_labels))\n\n# save the model\nmodel_top.save_weights('bottleneck_fc_model.h5')\n\n# test the model on new data\ntest_data = []\nfilenames = []\nfor img in os.listdir(test_dir):\n    img_path = os.path.join(test_dir, img)\n    img = Image.open(img_path)\n    img = img.resize((img_width, img_height), resample=Image.BILINEAR)\n    x = img_to_array(img)\n    x = np.expand_dims(x, axis=0)\n    x = x / 255.0\n    test_data.append(x)\n    filenames.append(img_path)\n\ntest_data = np.vstack(test_data)\npredictions = model_top.predict_classes(test_data, batch_size=batch_size)\n\n# print the predictions for each image\nfor i in range(len(filenames)):\n    print(filenames[i], predictions[i])\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom keras.models import model_from_json\nfrom sklearn.metrics import classification_report\n\n# Load the model from JSON file\nwith open('/kaggle/input/cnn-two-models/first_model.json', 'r') as json_file:\n    loaded_model_json = json_file.read()\n    model = model_from_json(loaded_model_json)\n\n# Load the weights into the model\nmodel.load_weights('/kaggle/input/cnn-two-models/first_model_weights.h5')\n\n# Compile the model with the same optimizer and loss function used during training\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n# Evaluate the model on the test set\ntest_dir = \"/kaggle/input/final-masked/content/drive/Shareddrives/ML_project/hotel-id-to-combat-human-trafficking-2022/test_set\"\ndatagen = ImageDataGenerator()\ntest_generator = datagen.flow_from_directory(\n    test_dir,\n    target_size=(224, 224),\n    batch_size=32,\n    class_mode='categorical',\n    shuffle=False)\n\nY_pred = model.predict_generator(test_generator)\ny_pred = np.argmax(Y_pred, axis=1)\n\n# Print classification report containing evaluation metrics\nprint(classification_report(test_generator.classes, y_pred))\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import load_model\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\nimport numpy as np\n\n# Load the saved model\nmodel = load_model('/kaggle/input/cnn-two-models/first_model.h5')\n\n# Load the validation data\nval_dir = \"/kaggle/input/hotel-id-to-combat-human-trafficking-2022-fgvc9/train_images\"\ndatagen = ImageDataGenerator()\nval_generator = datagen.flow_from_directory(\n    val_dir,\n    target_size=(224, 224),\n    batch_size=32,\n    class_mode='categorical')\n\n# Make predictions on the validation set\ny_pred = model.predict(val_generator)\ny_pred = np.argmax(y_pred, axis=1) # convert probabilities to class labels\n\n# Get the true labels of the validation set\ny_true = val_generator.classes\n\n# Calculate various evaluation metrics\naccuracy = accuracy_score(y_true, y_pred)\nprecision = precision_score(y_true, y_pred, average='weighted')\nrecall = recall_score(y_true, y_pred, average='weighted')\nf5 = f1_score(y_true, y_pred, average='weighted')\n\n# Print the evaluation metrics\nprint('Accuracy: {:.4f}'.format(accuracy))\nprint('Precision: {:.4f}'.format(precision))\nprint('Recall: {:.4f}'.format(recall))\nprint('F5 score: {:.4f}'.format(f5))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nimport os\nimport numpy as np\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, MaxPool2D, Flatten\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import load_model\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n\n# Set the random seed for reproducibility\nnp.random.seed(42)\n\n# Set your paths and parameters\ntrain_dir = \"/kaggle/input/hotel-id-to-combat-human-trafficking-2022-fgvc9/train_images\"\nbatch_size = 32\ntarget_size = (224, 224)\nepochs = 150\n\n# Load the data using the ImageDataGenerator\ndatagen = ImageDataGenerator(validation_split=0.2)\ntrain_generator = datagen.flow_from_directory(\n    train_dir,\n    target_size=target_size,\n    batch_size=batch_size,\n    class_mode='categorical',\n    subset='training')\nval_generator = datagen.flow_from_directory(\n    train_dir,\n    target_size=target_size,\n    batch_size=batch_size,\n    class_mode='categorical',\n    subset='validation')\n\n# Split the data into training and validation sets\ntrain_files, val_files = train_test_split(train_generator.filenames, test_size=0.2, random_state=42)\n\n# Load the model\nmodel = load_model(\"/kaggle/input/cnn-two-models/first_model.h5\")\n\n# Evaluate the model on the validation set\nval_steps = len(val_files) // batch_size\nval_preds = model.predict(val_generator, steps=val_steps)\nval_labels = np.concatenate([val_generator.next()[1] for i in range(val_steps)])\nval_preds = np.argmax(val_preds, axis=1)\nval_labels = np.argmax(val_labels, axis=1)\n\n# Compute evaluation metrics\naccuracy = accuracy_score(val_labels, val_preds)\nprecision = precision_score(val_labels, val_preds, average='weighted')\nrecall = recall_score(val_labels, val_preds, average='weighted')\nf1 = f1_score(val_labels, val_preds, average='weighted')\n\n# Print the evaluation metrics\nprint(\"Accuracy: {:.2f}%\".format(accuracy * 100))\nprint(\"Precision: {:.2f}%\".format(precision * 100))\nprint(\"Recall: {:.2f}%\".format(recall * 100))\nprint(\"F1-score: {:.2f}%\".format(f1 * 100))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model\npred_labels = model.predict(val_generator, steps=len(val_generator), verbose=1)\npred_labels = np.argmax(pred_labels, axis=1)\n\ntrue_labels = val_generator.classes\n\n# Exclude the incomplete batch\nnum_batches = len(val_generator)\nif num_batches * val_generator.batch_size < len(val_generator.filenames):\n    num_batches += 1\n    pred_labels = pred_labels[:num_batches*val_generator.batch_size]\n    true_labels = true_labels[:num_batches*val_generator.batch_size]\n\n# Calculate evaluation metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\nacc = accuracy_score(true_labels, pred_labels)\nprec = precision_score(true_labels, pred_labels, average='weighted', zero_division=1)\nrec = recall_score(true_labels, pred_labels, average='weighted')\nf1 = f1_score(true_labels, pred_labels, average='weighted')\n\nprint(\"Accuracy:\", acc)\nprint(\"Precision:\", prec)\nprint(\"Recall:\", rec)\nprint(\"F1 Score:\", f1)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prec = precision_score(true_labels, pred_labels, average='weighted', zero_division=1)\nprint(\"Precision:\", prec)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_data, val_labels = next(val_generator)\nprint(\"Shape of validation data:\", val_data.shape)\nprint(\"Shape of validation labels:\", val_labels.shape)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualisation for the first model**","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix\n\n# Predict class probabilities for validation data\ny_pred = model.predict(val_generator)\n\n# Get the predicted classes for each sample\ny_pred_classes = np.argmax(y_pred, axis=1)\n\n# Get the true classes for each sample\ny_true = val_generator.classes\n\n# Get the class labels\nclass_labels = list(val_generator.class_indices.keys())\n\n# Print classification report\nprint(classification_report(y_true, y_pred_classes, target_names=class_labels))\n\n# Plot confusion matrix\nconf_mat = confusion_matrix(y_true, y_pred_classes)\nplt.imshow(conf_mat, cmap=plt.cm.Blues)\nplt.colorbar()\ntick_marks = np.arange(len(class_labels))\nplt.xticks(tick_marks, class_labels, rotation=90)\nplt.yticks(tick_marks, class_labels)\nplt.xlabel('Predicted label')\nplt.ylabel('True label')\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Second model evaluation metrics**","metadata":{}},{"cell_type":"code","source":"import keras\nimport os\nimport numpy as np\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, MaxPool2D, Flatten\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import load_model\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n\n# Set the random seed for reproducibility\nnp.random.seed(42)\n\n# Set your paths and parameters\ntrain_dir = \"/kaggle/input/final-masked/content/drive/Shareddrives/ML_project/hotel-id-to-combat-human-trafficking-2022/extra_output\"\nbatch_size = 32\ntarget_size = (224, 224)\nepochs = 150\n\n# Load the data using the ImageDataGenerator\ndatagen = ImageDataGenerator(validation_split=0.2)\ntrain_generator = datagen.flow_from_directory(\n    train_dir,\n    target_size=target_size,\n    batch_size=batch_size,\n    class_mode='categorical',\n    subset='training')\nval_generator = datagen.flow_from_directory(\n    train_dir,\n    target_size=target_size,\n    batch_size=batch_size,\n    class_mode='categorical',\n    subset='validation')\n\n# Split the data into training and validation sets\ntrain_files, val_files = train_test_split(train_generator.filenames, test_size=0.2, random_state=42)\n\n# Load the model\nmodel = load_model(\"/kaggle/input/cnn-two-models/second_model.h5\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model\npred_labels = model.predict(val_generator, steps=len(val_generator), verbose=1)\npred_labels = np.argmax(pred_labels, axis=1)\n\ntrue_labels = val_generator.classes\n\n# Exclude the incomplete batch\nnum_batches = len(val_generator)\nif num_batches * val_generator.batch_size < len(val_generator.filenames):\n    num_batches += 1\n    pred_labels = pred_labels[:num_batches*val_generator.batch_size]\n    true_labels = true_labels[:num_batches*val_generator.batch_size]\n\n# Calculate evaluation metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\nacc = accuracy_score(true_labels, pred_labels)\nprec = precision_score(true_labels, pred_labels, average='weighted', zero_division=1)\nrec = recall_score(true_labels, pred_labels, average='weighted')\nf1 = f1_score(true_labels, pred_labels, average='weighted')\n\nprint(\"Accuracy:\", acc)\nprint(\"Precision:\", prec)\nprint(\"Recall:\", rec)\nprint(\"F1 Score:\", f1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualisation for the second model**","metadata":{}}]}