{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#Adapted from https://www.kaggle.com/pestipeti/keras-cnn-starter\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# import warnings\nimport warnings\n# filter warnings\nwarnings.filterwarnings('ignore')\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.Id.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# put labels into y_train variable\ny_train = train[\"Id\"]\n# Drop 'Id' column\nX_train = train.drop(labels = [\"Id\"], axis = 1)\ny_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing import image\nfrom keras.applications.imagenet_utils import preprocess_input\n\ndef prepareImages(train, shape, path):\n    \n    x_train = np.zeros((shape, 100, 100, 3))\n    count = 0\n    \n    for fig in train['Image']:\n        \n        #load images into images of size 100x100x3\n        img = image.load_img(\"../input/\"+path+\"/\"+fig, target_size=(100, 100, 3))\n        x = image.img_to_array(img)\n        x = preprocess_input(x)\n\n        x_train[count] = x\n        if (count%500 == 0):\n            print(\"Processing image: \", count+1, \", \", fig)\n        count += 1\n    \n    return x_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = prepareImages(train, train.shape[0], \"train\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = x_train / 255.0\nprint(\"x_train shape: \",x_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Some examples(first one)\nplt.imshow(x_train[0][:,:,0], cmap=\"gray\")\nplt.title(plt.title(train.iloc[0,0]))\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nlabel_encoder = LabelEncoder()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = label_encoder.fit_transform(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train[0:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils.np_utils import to_categorical\ny_train = to_categorical(y_train, num_classes = 5005)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#converted\nprint(y_train.shape)\ny_train #let's look at vectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential # to create a cnn model\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop,Adam\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 16, kernel_size = (5,5), padding = 'Same', activation = 'relu', input_shape = (100,100,3)))\nmodel.add(Conv2D(filters = 16, kernel_size = (5,5), padding = 'Same', activation = 'relu'))\nmodel.add(MaxPool2D(pool_size = (2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 32, kernel_size = (3,3), padding = 'Same', activation = 'relu'))\nmodel.add(Conv2D(filters = 32, kernel_size = (3,3), padding = 'Same', activation = 'relu'))\nmodel.add(MaxPool2D(pool_size = (2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation = 'relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation = 'relu'))\nmodel.add(MaxPool2D(pool_size = (2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\n# fully connected\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = 'relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dense(y_train.shape[1], activation = \"softmax\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define the optimizer\noptimizer = Adam(lr = 0.001, beta_1 = 0.9, beta_2 = 0.999)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set a learning rate annealer\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer = optimizer, loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# With data augmentation to prevent overfitting\n# We added this part in order to perform data augmentation\ndatagen = ImageDataGenerator(\n         featurewise_center=False,  # set input mean to 0 over the dataset\n         samplewise_center=False,  # set each sample mean to 0\n         featurewise_std_normalization=False,  # divide inputs by std of the dataset\n         samplewise_std_normalization=False,  # divide each input by its std\n         zca_whitening=False,  # apply ZCA whitening\n         rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n         zoom_range = 0.1, # Randomly zoom image \n         width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n         height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n         horizontal_flip=False,  # randomly flip images\n         vertical_flip=False)  # randomly flip images\n\n\ndatagen.fit(x_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#needed to change this to ensure no memory issues\nepochs = 200  # for better result increase the epochs\nbatch_size = 500\nx_train.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#had to change this to model.fit_generator()\nhistory = model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\n                              epochs=epochs, verbose = 2, \n                              steps_per_epoch=x_train.shape[0] // batch_size,\n                              callbacks=[learning_rate_reduction]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the loss curve for training\nplt.plot(history.history['loss'], color='r', label=\"Train Loss\")\nplt.title(\"Train Loss\")\nplt.xlabel(\"Number of Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the accuracy curve for training\nplt.plot(history.history['acc'], color='g', label=\"Train Accuracy\")\nplt.title(\"Train Accuracy\")\nplt.xlabel(\"Number of Epochs\")\nplt.ylabel(\"Accuracy\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train accuracy of the model: ',history.history['acc'][-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train loss of the model: ',history.history['loss'][-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = os.listdir(\"../input/test/\")\nprint(len(test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col = ['Image']\ntest_data = pd.DataFrame(test, columns=col)\ntest_data['Id'] = ''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = prepareImages(test_data, test_data.shape[0], \"test\")\nx_test /= 255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict(np.array(x_test), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, pred in enumerate(predictions):\n    test_data.loc[i, 'Id'] = ' '.join(label_encoder.inverse_transform(pred.argsort()[-5:][::-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head(10)\ntest_data.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"nbformat":4,"nbformat_minor":1}