{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9d6ea6fd-6278-170c-1e16-5290a10bd9ed"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense , Dropout , Lambda, Flatten\n",
        "from keras.optimizers import Adam ,RMSprop\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5f985377-a075-4831-fcaf-a0d425c059ba"
      },
      "outputs": [],
      "source": [
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output.\n",
        "# create the training & test sets, skipping the header row with [1:]\n",
        "train = pd.read_csv(\"../input/train.csv\")\n",
        "\n",
        "test_images = (pd.read_csv(\"../input/test.csv\").values).astype('float32')\n",
        "train_images = (train.ix[:,1:].values).astype('float32')\n",
        "train_labels = train.ix[:,0].values.astype('int32')\n",
        "train_images = train_images.reshape(train_images.shape[0],  28, 28)\n",
        "for i in range(6, 9):\n",
        "    plt.subplot(330 + (i+1))\n",
        "    plt.imshow(train_images[i], cmap=plt.get_cmap('gray'))\n",
        "    plt.title(train_labels[i]);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9625aaf5-d7e3-6288-e0c5-9c551fda1309"
      },
      "outputs": [],
      "source": [
        "\n",
        "train_images = train_images.reshape((42000, 28 * 28))\n",
        "\n",
        "test_images.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5be71942-6809-3973-2eac-ca3d4afb661d"
      },
      "outputs": [],
      "source": [
        "train_images = train_images / 255\n",
        "test_images = test_images / 255"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d08ada1f-2627-6333-8c21-0c8c90a2ad84"
      },
      "outputs": [],
      "source": [
        "from keras.utils.np_utils import to_categorical\n",
        "train_labels = to_categorical(train_labels)\n",
        "num_classes = train_labels.shape[1]\n",
        "num_classes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3fb27ac0-fa28-5b7b-47b4-f1a5176c25bb"
      },
      "outputs": [],
      "source": [
        "plt.title(train_labels[9])\n",
        "plt.plot(train_labels[9])\n",
        "plt.xticks(range(10));"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cb8d12a1-4348-0d18-70f7-e0aaf4aafcd4"
      },
      "outputs": [],
      "source": [
        "# fix random seed for reproducibility\n",
        "seed = 43\n",
        "np.random.seed(seed)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "198a91b8-40d9-0c5c-db5e-8bdeb703f747"
      },
      "outputs": [],
      "source": [
        "from keras.models import Sequential\n",
        "from keras.layers import Dense , Dropout\n",
        "\n",
        "model=Sequential()\n",
        "model.add(Dense(32,activation='relu',input_dim=(28 * 28)))\n",
        "model.add(Dense(16,activation='relu'))\n",
        "model.add(Dense(10,activation='softmax'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d04da183-644b-420f-e4b1-f662c28725cc"
      },
      "outputs": [],
      "source": [
        "from keras.optimizers import RMSprop\n",
        "model.compile(optimizer=RMSprop(lr=0.001),\n",
        " loss='categorical_crossentropy',\n",
        " metrics=['accuracy'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1d65ad70-4909-b0ac-87f8-e4f44e245cbb"
      },
      "outputs": [],
      "source": [
        "#history=model.fit(train_images, train_labels, validation_split = 0.05, \n",
        "            nb_epoch=25, batch_size=64)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a0eb5227-002e-2827-1c3b-fae984fc5dd1"
      },
      "outputs": [],
      "source": [
        "history_dict = history.history\n",
        "history_dict.keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "93e0f192-7628-e951-dfe2-751de125077f"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "loss_values = history_dict['loss']\n",
        "val_loss_values = history_dict['val_loss']\n",
        "epochs = range(1, len(loss_values) + 1)\n",
        "\n",
        "# \"bo\" is for \"blue dot\"\n",
        "plt.plot(epochs, loss_values, 'bo')\n",
        "# b+ is for \"blue crosses\"\n",
        "plt.plot(epochs, val_loss_values, 'b+')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "86b49a8e-c789-b71c-0591-5d7d0ae3c9c3"
      },
      "outputs": [],
      "source": [
        "plt.clf()   # clear figure\n",
        "acc_values = history_dict['acc']\n",
        "val_acc_values = history_dict['val_acc']\n",
        "\n",
        "plt.plot(epochs, acc_values, 'bo')\n",
        "plt.plot(epochs, val_acc_values, 'b+')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy')\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "85833b8e-2463-1dec-aab8-cc7fb3b7d1bf"
      },
      "outputs": [],
      "source": [
        "model = Sequential()\n",
        "model.add(Dense(64, activation='relu',input_dim=(28 * 28)))\n",
        "model.add(Dense(128, activation='relu'))\n",
        "model.add(Dropout(0.15))\n",
        "model.add(Dense(64, activation='relu'))\n",
        "model.add(Dropout(0.15))\n",
        "model.add(Dense(10, activation='softmax'))\n",
        "\n",
        "\n",
        "model.compile(optimizer=RMSprop(lr=0.0001), loss='categorical_crossentropy',\n",
        " metrics=['accuracy'])\n",
        "\n",
        "history=model.fit(train_images, train_labels, \n",
        "            nb_epoch=1, batch_size=64)\n",
        "history=model.fit(train_images, train_labels, \n",
        "            nb_epoch=2, batch_size=64)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bd419ffa-bd20-4ada-9063-93b3cb79594e"
      },
      "outputs": [],
      "source": [
        "for i in range(100):\n",
        "    history=model.fit(train_images, train_labels, \n",
        "            nb_epoch=20, batch_size=64)\n",
        "    predictions = model.predict_classes(test_images, verbose=0)\n",
        "\n",
        "    submissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n",
        "                         \"Label\": predictions})\n",
        "    submissions.to_csv(\"DR\"+str(i)+\".csv\", index=False, header=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "103be3eb-17e5-0e4c-9e95-5f2947b36321"
      },
      "outputs": [],
      "source": ""
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "73db388f-305c-2575-e8f9-799d8d91f1fc"
      },
      "outputs": [],
      "source": [
        "predictions = model.predict_classes(test_images, verbose=0)\n",
        "submissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n",
        "                         \"Label\": predictions})\n",
        "submissions.to_csv(\"DR\"+str(i)+\".csv\", index=False, header=True)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}