{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "79c0b5d8-105f-5515-ed5d-fb05ae7b3125"
      },
      "source": [
        "First try to build a CNN for digit recognition"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0ef76dab-747d-e5a6-1eb5-a2922eb0eb16"
      },
      "outputs": [],
      "source": [
        "#import packages and functions\n",
        "import numpy as np \n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "from keras.utils import np_utils\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, Lambda, Flatten\n",
        "from keras.layers.convolutional import *\n",
        "from keras.optimizers import Adam ,RMSprop\n",
        "from sklearn.model_selection import train_test_split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cccff7c5-3e18-6fb1-b83d-52ce3fa648e5"
      },
      "outputs": [],
      "source": [
        "#import data\n",
        "train_file = pd.read_csv(\"../input/train.csv\")\n",
        "print (train_file.shape)\n",
        "test_images = pd.read_csv(\"../input/test.csv\")\n",
        "print (test_images.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f9fb0ff0-034d-1248-995b-11e8915babce"
      },
      "outputs": [],
      "source": [
        "#for CNN\n",
        "x_train = train_file.drop(['label'], axis=1).values.astype('float32')\n",
        "Y_train = train_file['label'].values\n",
        "x_valid = test_images.values.astype('float32')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e502bd8b-84d2-bba5-d1be-f4ed59fd9296"
      },
      "outputs": [],
      "source": [
        "#for FNN\n",
        "#remove labels and save it in a specific vector\n",
        "#Label is the first column, rest is 28*28=784 pixel-columns\n",
        "train_images = (train_file.ix[:,1:].values).astype('float32')\n",
        "print (train_images.shape)\n",
        "train_labels = train_file.ix[:,0].values.astype('int32')\n",
        "print (train_labels.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4210e800-c44f-7ba6-dcc5-6705eb3a37dd"
      },
      "outputs": [],
      "source": [
        "#for CNN\n",
        "#reshape train and test\n",
        "img_width, img_height = 28, 28\n",
        "n_train = x_train.shape[0]\n",
        "n_valid = x_valid.shape[0]\n",
        "n_classes = 10 \n",
        "x_train = x_train.reshape(n_train,1,img_width,img_height)\n",
        "x_valid = x_valid.reshape(n_valid,1,img_width,img_height)\n",
        "\n",
        "x_train = x_train/255 #normalize from [0,255] to [0,1]\n",
        "x_valid = x_valid/255 \n",
        "y_train = np_utils.to_categorical(Y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1b9624cd-4a7d-c3d2-02fe-dc2d6844a0bf"
      },
      "outputs": [],
      "source": [
        "#for FNN\n",
        "#Convert train dataset to (num_images, img_rows, img_cols) format \n",
        "train_images = train_images.reshape((42000, 28 * 28))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e782184c-7ba4-592e-db7f-3c790ce76986"
      },
      "outputs": [],
      "source": [
        "#we need to normelize the pixel value :\n",
        "train_images = train_images / 255\n",
        "test_images = test_images / 255\n",
        "#and transform label into categorie\n",
        "train_labels = np_utils.to_categorical(train_labels)\n",
        "num_classes = train_labels.shape[1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b515b86d-bee4-1868-df0c-388de79011a0"
      },
      "outputs": [],
      "source": [
        "#FNN !!!\n",
        "# fix random seed for reproducibility\n",
        "seed = 43\n",
        "np.random.seed(seed)\n",
        "#designing the network :\n",
        "model=Sequential() #configures the learning process for a sequential model\n",
        "#relu as activation function, first layer is input layer\n",
        "model.add(Dense(64, activation='relu',input_dim=(28 * 28)))\n",
        "#model.add(Dense(32,activation='relu',input_dim=(28 * 28)))\n",
        "#model.add(Dense(16,activation='relu'))\n",
        "model.add(Dense(128, activation='relu'))\n",
        "model.add(Dropout(0.15))\n",
        "model.add(Dense(64, activation='relu'))\n",
        "model.add(Dropout(0.15))\n",
        "model.add(Dense(32, activation='relu'))\n",
        "model.add(Dropout(0.15))\n",
        "#output layer, a 10 classes problem so output = 10!\n",
        "model.add(Dense(10,activation='softmax'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5bf485ea-daae-d6bd-71f3-8b93ef746351"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from keras.utils.np_utils import to_categorical\n",
        "from keras import backend as K\n",
        "\n",
        "K.set_image_dim_ordering('th') #input shape: (channels, height, width)\n",
        "\n",
        "train_df = pd.read_csv(\"../input/train.csv\")\n",
        "valid_df = pd.read_csv(\"../input/test.csv\")\n",
        "\n",
        "x_train = train_df.drop(['label'], axis=1).values.astype('float32')\n",
        "Y_train = train_df['label'].values\n",
        "x_valid = valid_df.values.astype('float32')\n",
        "\n",
        "img_width, img_height = 28, 28\n",
        "\n",
        "n_train = x_train.shape[0]\n",
        "n_valid = x_valid.shape[0]\n",
        "\n",
        "n_classes = 10 \n",
        "\n",
        "x_train = x_train.reshape(n_train,1,img_width,img_height)\n",
        "x_valid = x_valid.reshape(n_valid,1,img_width,img_height)\n",
        "\n",
        "x_train = x_train/255 #normalize from [0,255] to [0,1]\n",
        "x_valid = x_valid/255 \n",
        "\n",
        "y_train = to_categorical(Y_train)\n",
        "\n",
        "\n",
        "from keras.models import Sequential\n",
        "from keras.layers.convolutional import *\n",
        "from keras.layers.core import Dropout, Dense, Flatten, Activation\n",
        "\n",
        "n_filters = 64\n",
        "filter_size1 = 3\n",
        "filter_size2 = 2\n",
        "pool_size1 = 3\n",
        "pool_size2 = 1\n",
        "n_dense = 128\n",
        "\n",
        "model = Sequential()\n",
        "\n",
        "model.add(Convolution2D(n_filters, filter_size1, filter_size1, batch_input_shape=(None, 1, img_width, img_height), activation='relu', border_mode='valid'))\n",
        "\n",
        "model.add(MaxPooling2D(pool_size=(pool_size1, pool_size1)))\n",
        "\n",
        "model.add(Convolution2D(n_filters, filter_size2, filter_size2, activation='relu', border_mode='valid'))\n",
        "\n",
        "model.add(MaxPooling2D(pool_size=(pool_size2, pool_size2)))\n",
        "\n",
        "model.add(Dropout(0.25))\n",
        "\n",
        "model.add(Flatten())\n",
        "\n",
        "model.add(Dense(n_dense))\n",
        "\n",
        "model.add(Activation('relu'))\n",
        "\n",
        "model.add(Dropout(0.5))\n",
        "\n",
        "model.add(Dense(n_classes))\n",
        "\n",
        "model.add(Activation('softmax'))\n",
        "\n",
        "model.compile(optimizer='adam',\n",
        "              loss='categorical_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "\"\"\"\n",
        "#CNN !!\n",
        "n_filters = 64\n",
        "filter_size1 = 3\n",
        "filter_size2 = 2\n",
        "pool_size1 = 3\n",
        "pool_size2 = 1\n",
        "n_dense = 128\n",
        "n_classes =10\n",
        "img_width, img_height = 28, 28\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Convolution2D(n_filters, filter_size1, filter_size1, batch_input_shape=(None, 1, img_width, img_height), activation='relu', border_mode='valid'))\n",
        "model.add(MaxPooling2D(pool_size=(pool_size1, pool_size1)))\n",
        "model.add(Convolution2D(n_filters, filter_size2, filter_size2, activation='relu', border_mode='valid'))\n",
        "model.add(MaxPooling2D(pool_size=(pool_size2, pool_size2)))\n",
        "model.add(Dropout(0.25))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(64))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Dense(n_classes))\n",
        "model.add(Dense(n_classes,activation='softmax'))\n",
        "#model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "\"\"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5bed4ccc-a16c-30c3-3613-7c8339a4073c"
      },
      "outputs": [],
      "source": [
        "#defining metric, loss function and optimize\n",
        "model.compile(optimizer=RMSprop(lr=0.001), loss='categorical_crossentropy', metrics=['accuracy'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "96fb3150-9d97-66d4-1c5c-5e51f38c1779"
      },
      "outputs": [],
      "source": [
        "batch_size = 128\n",
        "n_epochs = 2\n",
        "\n",
        "model.fit(x_train,\n",
        "          y_train,\n",
        "          batch_size=batch_size,\n",
        "          epochs=n_epochs,verbose=2,\n",
        "          validation_split=.2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bfc6dd0f-6da9-d286-7524-1bb73b268bf9"
      },
      "outputs": [],
      "source": [
        "#fitting the model\n",
        "history=model.fit(train_images, train_labels, validation_split = 0.05, nb_epoch=25, batch_size=64)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7c7d8355-0ae0-40c1-7025-7544d7974cb0"
      },
      "outputs": [],
      "source": [
        "#print the network definition\n",
        "print(model.summary())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "fdd44a31-bc5f-9d56-0d83-4275b948447a"
      },
      "outputs": [],
      "source": [
        "test_images2 = (test_images.values).astype('float32')\n",
        "test_images2 = test_images2.reshape((28000, 28 * 28))\n",
        "predictions = model.predict(test_images2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b6aad7b5-307d-ee79-e661-71f674bd70dd"
      },
      "outputs": [],
      "source": [
        "#CNN\n",
        "predictions = model.predict(x_valid)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5de796e8-3012-1875-7ac1-033ce968696e"
      },
      "outputs": [],
      "source": [
        "predictions = np_utils.categorical_probas_to_classes(predictions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "dffec0f1-f3f7-a3f8-3ef7-7dc29c04b335"
      },
      "outputs": [],
      "source": [
        "np.savetxt('mnist_output.csv', np.c_[range(1,len(predictions)+1),predictions], delimiter=',', header = 'ImageId,Label', comments = '', fmt='%d')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "68b73525-5996-9c0a-df28-126b654df204"
      },
      "outputs": [],
      "source": [
        "output_file = \"submission.csv\"\n",
        "out = np.column_stack((range(1, predictions.shape[0]+1), predictions))\n",
        "np.savetxt(output_file, out, header=\"ImageId,Label\", comments=\"\", fmt=\"%d,%d\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "46efad0d-6d79-1c14-7c96-49cf733dc7f5"
      },
      "outputs": [],
      "source": [
        "history_dict = history.history\n",
        "history_dict.keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9e179034-c77f-9ae4-7e1f-fd03b66f6f46"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "loss_values = history_dict['loss']\n",
        "val_loss_values = history_dict['val_loss']\n",
        "epochs = range(1, len(loss_values) + 1)\n",
        "\n",
        "# \"bo\" is for \"blue dot\"\n",
        "plt.plot(epochs, loss_values, 'bo')\n",
        "# b+ is for \"blue crosses\"\n",
        "plt.plot(epochs, val_loss_values, 'b+')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a5f6d4ee-7d2d-141c-6ccd-b2254f20f87f"
      },
      "outputs": [],
      "source": [
        "plt.clf()   # clear figure\n",
        "acc_values = history_dict['acc']\n",
        "val_acc_values = history_dict['val_acc']\n",
        "\n",
        "plt.plot(epochs, acc_values, 'bo')\n",
        "plt.plot(epochs, val_acc_values, 'b+')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy')\n",
        "\n",
        "plt.show()"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}