{"cells":[{"metadata":{"_uuid":"4ed6cbfad553f911d7414f5f1fb9d0bc45a809b5"},"cell_type":"markdown","source":"# VGG16 trained from scratch: 0.99600\n\nHere I use a VGG16 CNN from Keras. In a first attempt, I just removed the top Dense layer and added mine, keeping all the base model frozen, but results were poor. Then I unfrozen the last 4 layers but still the VGG16 performed around `0.98`, which was not enough to beat another model I made in the past weeks.\n\nSo I unfrozen all the network and trained it all for a few epochs to `0.99342` on validation set, reaching the same score on the leaderboard. I manually tuned the learning rate from `1e-02` to `1e-03` when it reached a plateau. I plan to add a Callback to automate this process.\n\nImportant:  I used Keras ImageDataGenerator to augment data with some basic manipulations and used 36x36 input shape for VGG16.\n\nAny advice is welcome!"},{"metadata":{"trusted":true,"_uuid":"50015a5a19402d4c5aad0f74d0488e087203784e"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix, mean_squared_error\nfrom sklearn.model_selection import train_test_split\nimport itertools\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, ZeroPadding2D, Input\nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications import VGG16\n\n%matplotlib inline\n\npd.set_option(\"display.max_rows\", 6)\n\nnp.random.seed(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddb7245d997f57d99254b91dc5277afa04d22a44"},"cell_type":"code","source":"!ls ../input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8569af5daca7397575d0b8aac95d4dedbc7c0e9b"},"cell_type":"code","source":"# Load the data\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\n\ntrain_X = (train.iloc[:,:-1]/255).values.copy()\ntrain_y = (train['label']).values.copy()\ntest = (test/255).values.copy()\n\nprint(train_X.shape)\nprint(train_y.shape)\nprint(test.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"303d87f213690502dd8e819cf12b643f5ab4b55e"},"cell_type":"code","source":"train_X = np.reshape(train_X, (42000, 28,28,1))\nnew_X = np.zeros((42000,28,28,3))\nfor i in range(len(new_X)):\n    tmp = np.stack((train_X[i],)*3, axis=-1)\n    new_X[i] = np.resize(tmp, (28,28,3))\ntrain_X = new_X\n\n\ntest = np.reshape(test, (28000, 28,28,1))\nnew_X = np.zeros((28000,28,28,3))\nfor i in range(len(new_X)):\n    tmp = np.stack((test[i],)*3, axis=-1)\n    new_X[i] = np.resize(tmp, (28,28,3))\ntest = new_X\n\n\nfrom keras.utils import to_categorical\ntrain_y = to_categorical(train_y, num_classes=10)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5356b2567c5541c80e028202fc21cde5a8810263"},"cell_type":"code","source":"batch_size = 4\ndatagen = ImageDataGenerator(\n    validation_split=.2,\n    rotation_range=10,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    fill_mode='nearest')\n\ntrain_generator = datagen.flow(\n        train_X,\n        train_y,\n        shuffle=True,\n        subset='training',\n        batch_size=batch_size,)\nval_generator = datagen.flow(\n        train_X,\n        train_y,\n        subset='validation',\n        batch_size=batch_size,)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf0d271f67b1aed1ac2e327c25b16ea71cb5fdf6"},"cell_type":"code","source":"\nplt.figure(figsize=(15,10)) \nfor X_batch, y_batch in train_generator:\n    for i in range(0, 4):\n        plt.subplot(220 + 1 + i)\n        plt.imshow(X_batch[i].reshape(28,28,3), cmap=plt.get_cmap('gray'))\n    plt.show()\n    break\n#g = plt.imshow(train_X[0][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"037ea2e41b67768f86247e2b18106da7390aa53c"},"cell_type":"code","source":"\nconv_base = VGG16(#weights='imagenet',\n                  include_top=False,\n                  input_shape=(36, 36, 3))\n\n#for layer in conv_base.layers[:-4]:\n#   layer.trainable = False\n#for layer in conv_base.layers:\n#    print(layer, layer.trainable)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8696bee9954b00f4678cd57233db96851248176"},"cell_type":"code","source":"\nfor layer in conv_base.layers:\n   layer.trainable = True\n\nmodel = Sequential()\nmodel.add(ZeroPadding2D(padding=(32-28, 32-28), input_shape=(28,28,3)))\nmodel.add(conv_base)\nmodel.add(Flatten())\nmodel.add(Dense(64, activation='relu'))\nmodel.add(Dense(10, activation='softmax'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e96b716827c81b11dda4b6602776269cba0a7e8e"},"cell_type":"code","source":"from keras.optimizers import SGD, Adam\n\n# I manually change the lr to 1e-3 when it gets stuck on .98\nmodel.compile(optimizer=SGD(lr=1e-4), loss='categorical_crossentropy', metrics=['accuracy'])\nmodel.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"594facc7b10d35cd0494424b0954306e1c12786b"},"cell_type":"code","source":"from keras.models import load_model\n\nmodel = load_model('vgg16_dense.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10ad7c4d1487076fedfaa437ab66631b8c585b8e"},"cell_type":"code","source":"epochs = 1 # change epochs to improve\n\nmodel.fit_generator(train_generator, \n                    validation_data=val_generator, \n                    validation_steps=(42000*.2)//batch_size, \n                    epochs=epochs, \n                    steps_per_epoch=(42000-42000*.2)//batch_size)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d7dde481daa3ad6f539256a8b30103c015aa8a8"},"cell_type":"code","source":"model.save('vgg16_dense.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6c087799b2a7b0d2611f84f9fab83915265b9e1"},"cell_type":"code","source":"pred = model.predict(test)\npred.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70db4b3872544888bca03d299631dd7383b0d77d"},"cell_type":"code","source":"plt.figure(figsize=(4,4)) \nplt.imshow(test[100].reshape(28,28,3), cmap=plt.get_cmap('gray'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b38ebb7f08aa38748ebeaf82eab99f6c395b412"},"cell_type":"code","source":"out_label = [ np.argmax(i) for i in pred]\nout_imageid = [ i+1 for i in range(len(test))]\nout = pd.DataFrame()\nout['ImageId'] = out_imageid\nout['Label'] = out_label\n\nout.to_csv('submission.csv', index=False)\nout","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}