{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "7daa5a37-ea6b-2274-4bc1-fadb0a2fc41a"
      },
      "source": [
        "# Use keras to classify Sea Lions\n",
        "\n",
        "- I am using the first picture to extract Sea Lion coordinates using blob detection\n",
        "- I extract 32 by 32 images centered on the extracted coordinates\n",
        "- I train a simple keras model\n",
        "\n",
        "*** The test accuracy is for Sea Lions in the first image only and without negative examples** "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "35cabbd8-8e7b-5be7-bd00-61b1addc2d75"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import os\n",
        "import cv2\n",
        "import matplotlib.pyplot as plt\n",
        "import skimage.feature\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelBinarizer\n",
        "import keras\n",
        "from keras.models import Sequential, load_model\n",
        "from keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D, Lambda, Cropping2D\n",
        "from keras.utils import np_utils\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "39c70a8d-7645-f2f2-8e09-7d528eb48e57"
      },
      "source": [
        "### Initialize variables"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3a6329c7-aa3b-b576-b6e1-d675e6e8fbf2"
      },
      "outputs": [],
      "source": [
        "classes = [\"adult_males\", \"subadult_males\", \"adult_females\", \"juveniles\", \"pups\"]\n",
        "\n",
        "file_names = os.listdir(\"../input/Train/\")\n",
        "file_names = sorted(file_names, key=lambda \n",
        "                    item: (int(item.partition('.')[0]) if item[0].isdigit() else float('inf'), item)) \n",
        "\n",
        "# select a subset of files to run on\n",
        "file_names = file_names[0:1]\n",
        "\n",
        "# dataframe to store results in\n",
        "coordinates_df = pd.DataFrame(index=file_names, columns=classes)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "f6aecb87-1c65-5c28-12c8-d86c42ae223b"
      },
      "source": [
        "### Extract coordinates"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7755c681-04df-368a-aca2-f099dd9ce805"
      },
      "outputs": [],
      "source": [
        "for filename in file_names:\n",
        "    \n",
        "    # read the Train and Train Dotted images\n",
        "    image_1 = cv2.imread(\"../input/TrainDotted/\" + filename)\n",
        "    image_2 = cv2.imread(\"../input/Train/\" + filename)\n",
        "    \n",
        "    # absolute difference between Train and Train Dotted\n",
        "    image_3 = cv2.absdiff(image_1,image_2)\n",
        "    \n",
        "    # mask out blackened regions from Train Dotted\n",
        "    mask_1 = cv2.cvtColor(image_1, cv2.COLOR_BGR2GRAY)\n",
        "    mask_1[mask_1 < 20] = 0\n",
        "    mask_1[mask_1 > 0] = 255\n",
        "    \n",
        "    mask_2 = cv2.cvtColor(image_2, cv2.COLOR_BGR2GRAY)\n",
        "    mask_2[mask_2 < 20] = 0\n",
        "    mask_2[mask_2 > 0] = 255\n",
        "    \n",
        "    image_3 = cv2.bitwise_or(image_3, image_3, mask=mask_1)\n",
        "    image_3 = cv2.bitwise_or(image_3, image_3, mask=mask_2) \n",
        "    \n",
        "    # convert to grayscale to be accepted by skimage.feature.blob_log\n",
        "    image_3 = cv2.cvtColor(image_3, cv2.COLOR_BGR2GRAY)\n",
        "    \n",
        "    # detect blobs\n",
        "    blobs = skimage.feature.blob_log(image_3, min_sigma=3, max_sigma=4, num_sigma=1, threshold=0.02)\n",
        "    \n",
        "    adult_males = []\n",
        "    subadult_males = []\n",
        "    pups = []\n",
        "    juveniles = []\n",
        "    adult_females = [] \n",
        "    \n",
        "    for blob in blobs:\n",
        "        # get the coordinates for each blob\n",
        "        y, x, s = blob\n",
        "        # get the color of the pixel from Train Dotted in the center of the blob\n",
        "        g,b,r = image_1[int(y)][int(x)][:]\n",
        "        \n",
        "        # decision tree to pick the class of the blob by looking at the color in Train Dotted\n",
        "        if r > 200 and g < 50 and b < 50: # RED\n",
        "            adult_males.append((int(x),int(y)))        \n",
        "        elif r > 200 and g > 200 and b < 50: # MAGENTA\n",
        "            subadult_males.append((int(x),int(y)))         \n",
        "        elif r < 100 and g < 100 and 150 < b < 200: # GREEN\n",
        "            pups.append((int(x),int(y)))\n",
        "        elif r < 100 and  100 < g and b < 100: # BLUE\n",
        "            juveniles.append((int(x),int(y))) \n",
        "        elif r < 150 and g < 50 and b < 100:  # BROWN\n",
        "            adult_females.append((int(x),int(y)))\n",
        "            \n",
        "    coordinates_df[\"adult_males\"][filename] = adult_males\n",
        "    coordinates_df[\"subadult_males\"][filename] = subadult_males\n",
        "    coordinates_df[\"adult_females\"][filename] = adult_females\n",
        "    coordinates_df[\"juveniles\"][filename] = juveniles\n",
        "    coordinates_df[\"pups\"][filename] = pups"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "d03424c1-b12b-ae53-2fed-1dff86398164"
      },
      "source": [
        "### Extract 32 by 32 images"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c9549615-3a64-2ef2-2be0-1946e07c2ee2"
      },
      "outputs": [],
      "source": [
        "x = []\n",
        "y = []\n",
        "\n",
        "for filename in file_names:    \n",
        "    image = cv2.imread(\"../input/Train/\" + filename)\n",
        "    for lion_class in classes:\n",
        "        for coordinates in coordinates_df[lion_class][filename]:\n",
        "            thumb = image[coordinates[1]-16:coordinates[1]+16,coordinates[0]-16:coordinates[0]+16,:]\n",
        "            if np.shape(thumb) == (32, 32, 3):\n",
        "                x.append(thumb)\n",
        "                y.append(lion_class)\n",
        "x = np.array(x)\n",
        "y = np.array(y)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "75016379-58d6-1ff4-d411-71bb29eec895"
      },
      "source": [
        "### Plot examples"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "981bcb37-d353-dcd1-fc99-ff178605811c"
      },
      "outputs": [],
      "source": [
        "for lion_class in classes:\n",
        "    f, ax = plt.subplots(1,10,figsize=(12,1.5))\n",
        "    f.suptitle(lion_class)\n",
        "    axes = ax.flatten()\n",
        "    j = 0\n",
        "    for a in axes:\n",
        "        a.set_xticks([])\n",
        "        a.set_yticks([])\n",
        "        for i in range(j,len(x)):\n",
        "            if y[i] == lion_class:\n",
        "                j = i+1\n",
        "                a.imshow(cv2.cvtColor(x[i], cv2.COLOR_BGR2RGB))\n",
        "                break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "3ae5856f-c680-e4e3-459a-fa764559938a"
      },
      "source": [
        "### One hot encoding"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c1438c4d-e0d4-812a-adff-6f5db4c32a81"
      },
      "outputs": [],
      "source": [
        "encoder = LabelBinarizer()\n",
        "encoder.fit(y)\n",
        "y = encoder.transform(y).astype(float)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "7c0dfde1-a1b3-a181-bcaa-58455025138a"
      },
      "source": [
        "### Build Keras model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "25ecbbe7-cae5-a6fa-3ab0-943fe10f580a"
      },
      "outputs": [],
      "source": [
        "model = Sequential()\n",
        "\n",
        "model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=(32,32,3)))\n",
        "\n",
        "\n",
        "model.add(Conv2D(32, (5, 5), activation='relu', padding='same'))\n",
        "model.add(MaxPooling2D(pool_size=(2,2)))\n",
        "model.add(Conv2D(64, (5, 5), activation='relu', padding='same'))\n",
        "\n",
        "model.add(Flatten())\n",
        "\n",
        "model.add(Dense(512, activation='relu'))\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Dense(5, activation='softmax'))\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e9447e9b-92f7-aab7-bd03-03a3aa82b816"
      },
      "outputs": [],
      "source": [
        "history = model.fit(x, y, epochs=20, validation_split=0.2, verbose=0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "bf15c0bf-f00a-f80c-8520-43ef450feedb"
      },
      "source": [
        "### Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5782174e-3d6f-c3ce-be48-d8095a07c35e"
      },
      "outputs": [],
      "source": [
        "# http://machinelearningmastery.com/display-deep-learning-model-training-history-in-keras/\n",
        "# summarize history for accuracy\n",
        "plt.plot(history.history['acc'])\n",
        "plt.plot(history.history['val_acc'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'test'], loc='upper left')\n",
        "plt.show()\n",
        "# summarize history for loss\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'test'], loc='upper left')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cb53a383-7951-4ccd-8832-c6a73ec5829d"
      },
      "outputs": [],
      "source": ""
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "63078b8a-97e9-efe3-09bf-8740c1d50eb1"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}