{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom random import sample\nimport statistics\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.preprocessing import MinMaxScaler\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras import Sequential\nfrom keras.layers import Dense, Conv2D, Flatten, BatchNormalization, Activation, MaxPooling2D\nfrom keras.utils.np_utils import to_categorical\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-04T14:24:50.642807Z","iopub.execute_input":"2022-07-04T14:24:50.643092Z","iopub.status.idle":"2022-07-04T14:24:56.380263Z","shell.execute_reply.started":"2022-07-04T14:24:50.643059Z","shell.execute_reply":"2022-07-04T14:24:56.379526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:24:56.381755Z","iopub.execute_input":"2022-07-04T14:24:56.382015Z","iopub.status.idle":"2022-07-04T14:24:57.100943Z","shell.execute_reply.started":"2022-07-04T14:24:56.381977Z","shell.execute_reply":"2022-07-04T14:24:57.100158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:24:57.103615Z","iopub.execute_input":"2022-07-04T14:24:57.103989Z","iopub.status.idle":"2022-07-04T14:25:02.242946Z","shell.execute_reply.started":"2022-07-04T14:24:57.103946Z","shell.execute_reply":"2022-07-04T14:25:02.242107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"============== Training Dataset ==============\\n\")\nprint(\"Total rows    :\", df_train.shape[0])\nprint(\"Total features:\", df_train.shape[1])\nprint()\nprint(\"============== Testing Dataset ==============\\n\")\nprint(\"Total rows    :\", df_test.shape[0])\nprint(\"Total features:\", df_test.shape[1])","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:02.245023Z","iopub.execute_input":"2022-07-04T14:25:02.247105Z","iopub.status.idle":"2022-07-04T14:25:02.255340Z","shell.execute_reply.started":"2022-07-04T14:25:02.247076Z","shell.execute_reply":"2022-07-04T14:25:02.254611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,7))\nsns.countplot(data=df_train, x='label', palette='viridis')\nplt.title('Frequency of Labels', fontdict={'fontsize':20})\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:02.256661Z","iopub.execute_input":"2022-07-04T14:25:02.256891Z","iopub.status.idle":"2022-07-04T14:25:02.617109Z","shell.execute_reply.started":"2022-07-04T14:25:02.256859Z","shell.execute_reply":"2022-07-04T14:25:02.616287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = df_train.drop('label', axis=1)\ny_train = df_train['label']\nX_test = df_test.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:02.621774Z","iopub.execute_input":"2022-07-04T14:25:02.622782Z","iopub.status.idle":"2022-07-04T14:25:02.781717Z","shell.execute_reply.started":"2022-07-04T14:25:02.622744Z","shell.execute_reply":"2022-07-04T14:25:02.780970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(18,18))\nindex = sample(range(0, X_train.shape[0]),16)\ncount = 0\n\nfor i in range(4):\n    for j in range(4):\n        plt.subplot(4, 4, count+1)\n        plt.imshow(X_train.iloc[index[count]].values.reshape((28,28)), cmap=plt.cm.binary)\n        plt.title(f\"Record: {index[count]} & Label: {y_train.iloc[index[count]]}\")\n        count+=1","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:02.783263Z","iopub.execute_input":"2022-07-04T14:25:02.783738Z","iopub.status.idle":"2022-07-04T14:25:04.604910Z","shell.execute_reply.started":"2022-07-04T14:25:02.783702Z","shell.execute_reply":"2022-07-04T14:25:04.604260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = MinMaxScaler(feature_range=(0,1))\n\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)\n\n\nX_train = X_train_scaled.reshape(-1,28,28,1)\nX_test = X_test_scaled.reshape(-1,28,28,1)\n\ny_train = to_categorical(y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:04.606117Z","iopub.execute_input":"2022-07-04T14:25:04.606460Z","iopub.status.idle":"2022-07-04T14:25:05.044748Z","shell.execute_reply.started":"2022-07-04T14:25:04.606428Z","shell.execute_reply":"2022-07-04T14:25:05.043928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"X train: {X_train.shape}\")\nprint(f\"X test : {X_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:05.045937Z","iopub.execute_input":"2022-07-04T14:25:05.046203Z","iopub.status.idle":"2022-07-04T14:25:05.051570Z","shell.execute_reply.started":"2022-07-04T14:25:05.046159Z","shell.execute_reply":"2022-07-04T14:25:05.050868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val= train_test_split(X_train, y_train, test_size=0.2, random_state=21)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:05.055109Z","iopub.execute_input":"2022-07-04T14:25:05.055884Z","iopub.status.idle":"2022-07-04T14:25:05.421098Z","shell.execute_reply.started":"2022-07-04T14:25:05.055842Z","shell.execute_reply":"2022-07-04T14:25:05.420380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 1: Kernel size=(3,3)","metadata":{}},{"cell_type":"code","source":"def model_3():\n    \n    cnn = Sequential(name=\"1\")\n\n    cnn.add(Conv2D(filters=25, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False, input_shape=(X_train.shape[1:])))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(Conv2D(filters=40, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(Conv2D(filters=75, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n    \n    cnn.add(Conv2D(filters=130, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(MaxPooling2D(pool_size=(2,2)))\n\n    cnn.add(Conv2D(filters=160, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(MaxPooling2D(pool_size=(2,2)))\n    \n    cnn.add(Flatten())\n\n    cnn.add(Dense(units=40))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(Dense(units=10))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('softmax'))\n\n    cnn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    cnn.summary()\n    \n    return cnn","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:05.422293Z","iopub.execute_input":"2022-07-04T14:25:05.422536Z","iopub.status.idle":"2022-07-04T14:25:05.435737Z","shell.execute_reply.started":"2022-07-04T14:25:05.422504Z","shell.execute_reply":"2022-07-04T14:25:05.435075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 2: Kernel size=(5,5)","metadata":{}},{"cell_type":"code","source":"def model_5():\n    \n    cnn = Sequential(name=\"2\")\n\n    cnn.add(Conv2D(filters=20, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False, input_shape=(X_train.shape[1:])))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(Conv2D(filters=50, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(Conv2D(filters=80, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(MaxPooling2D(pool_size=(2,2)))\n\n    cnn.add(Conv2D(filters=140, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(Flatten())\n\n    cnn.add(Dense(units=40))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(Dense(units=10))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('softmax'))\n\n    cnn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    cnn.summary()\n    \n    return cnn","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:05.437030Z","iopub.execute_input":"2022-07-04T14:25:05.437517Z","iopub.status.idle":"2022-07-04T14:25:05.450477Z","shell.execute_reply.started":"2022-07-04T14:25:05.437481Z","shell.execute_reply":"2022-07-04T14:25:05.449714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 3: Kernel size=(7,7)","metadata":{}},{"cell_type":"code","source":"def model_7():\n    \n    cnn = Sequential(name=\"3\")\n\n    cnn.add(Conv2D(filters=40, kernel_size=(7,7), strides=(1,1), padding='valid', activation=None, use_bias=False, input_shape=(X_train.shape[1:])))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(Conv2D(filters=100, kernel_size=(7,7), strides=(1,1), padding='valid', activation=None, use_bias=False))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(MaxPooling2D(pool_size=(2,2)))\n\n    cnn.add(Conv2D(filters=140, kernel_size=(7,7), strides=(1,1), padding='valid', activation=None, use_bias=False))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(Flatten())\n\n    cnn.add(Dense(units=30))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('relu'))\n\n    cnn.add(Dense(units=10))\n    cnn.add(BatchNormalization())\n    cnn.add(Activation('softmax'))\n\n    cnn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    cnn.summary()\n    \n    return cnn","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:05.451698Z","iopub.execute_input":"2022-07-04T14:25:05.451941Z","iopub.status.idle":"2022-07-04T14:25:05.465444Z","shell.execute_reply.started":"2022-07-04T14:25:05.451910Z","shell.execute_reply":"2022-07-04T14:25:05.464733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1 = model_3()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:05.466417Z","iopub.execute_input":"2022-07-04T14:25:05.466816Z","iopub.status.idle":"2022-07-04T14:25:08.415492Z","shell.execute_reply.started":"2022-07-04T14:25:05.466689Z","shell.execute_reply":"2022-07-04T14:25:08.414625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model2 = model_5()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:08.417187Z","iopub.execute_input":"2022-07-04T14:25:08.417452Z","iopub.status.idle":"2022-07-04T14:25:08.544561Z","shell.execute_reply.started":"2022-07-04T14:25:08.417415Z","shell.execute_reply":"2022-07-04T14:25:08.543842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model3 = model_7()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:08.545649Z","iopub.execute_input":"2022-07-04T14:25:08.545880Z","iopub.status.idle":"2022-07-04T14:25:08.657168Z","shell.execute_reply.started":"2022-07-04T14:25:08.545842Z","shell.execute_reply":"2022-07-04T14:25:08.656517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(rotation_range=20, zoom_range=0.20, width_shift_range=0.1, height_shift_range=0.1)\ntrain_generator = train_datagen.flow(X_train, y_train, batch_size=120, shuffle=True)\n\nval_datagen = ImageDataGenerator()\nval_generator = val_datagen.flow(X_val, y_val, batch_size=120, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:08.658496Z","iopub.execute_input":"2022-07-04T14:25:08.658746Z","iopub.status.idle":"2022-07-04T14:25:09.060830Z","shell.execute_reply.started":"2022-07-04T14:25:08.658712Z","shell.execute_reply":"2022-07-04T14:25:09.060083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduceLROnPlateau = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, factor=0.5, min_lr=0.00001)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:09.062029Z","iopub.execute_input":"2022-07-04T14:25:09.062600Z","iopub.status.idle":"2022-07-04T14:25:09.068968Z","shell.execute_reply.started":"2022-07-04T14:25:09.062559Z","shell.execute_reply":"2022-07-04T14:25:09.066190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history1 = model1.fit(train_generator, epochs=120, callbacks=[reduceLROnPlateau], validation_data=val_generator)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:25:09.072814Z","iopub.execute_input":"2022-07-04T14:25:09.073002Z","iopub.status.idle":"2022-07-04T14:45:16.426268Z","shell.execute_reply.started":"2022-07-04T14:25:09.072977Z","shell.execute_reply":"2022-07-04T14:45:16.425562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(history1.history)[['val_loss', 'val_accuracy']].plot(figsize=(20,8))\nplt.xlabel(\"Epoch\", fontdict={'size':18})\nplt.title(\"Model 1\", fontdict={'size':22})\nplt.ylim((0,1.05))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:45:16.427748Z","iopub.execute_input":"2022-07-04T14:45:16.427991Z","iopub.status.idle":"2022-07-04T14:45:16.669599Z","shell.execute_reply.started":"2022-07-04T14:45:16.427957Z","shell.execute_reply":"2022-07-04T14:45:16.668915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history2 = model2.fit(train_generator, epochs=95, callbacks=[reduceLROnPlateau], validation_data=val_generator)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T14:45:16.670934Z","iopub.execute_input":"2022-07-04T14:45:16.671186Z","iopub.status.idle":"2022-07-04T15:00:20.872128Z","shell.execute_reply.started":"2022-07-04T14:45:16.671141Z","shell.execute_reply":"2022-07-04T15:00:20.871427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(history2.history)[['val_loss', 'val_accuracy']].plot(figsize=(20,8))\nplt.xlabel(\"Epoch\", fontdict={'size':18})\nplt.title(\"Model 2\", fontdict={'size':22})\nplt.ylim((0,1.05))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T15:00:20.873623Z","iopub.execute_input":"2022-07-04T15:00:20.873894Z","iopub.status.idle":"2022-07-04T15:00:21.096737Z","shell.execute_reply.started":"2022-07-04T15:00:20.873857Z","shell.execute_reply":"2022-07-04T15:00:21.096056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history3 = model3.fit(train_generator, epochs=80, callbacks=[reduceLROnPlateau], validation_data=val_generator)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T15:00:21.097973Z","iopub.execute_input":"2022-07-04T15:00:21.098213Z","iopub.status.idle":"2022-07-04T15:12:52.749125Z","shell.execute_reply.started":"2022-07-04T15:00:21.098179Z","shell.execute_reply":"2022-07-04T15:12:52.748434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(history3.history)[['val_loss', 'val_accuracy']].plot(figsize=(20,8))\nplt.xlabel(\"Epoch\", fontdict={'size':18})\nplt.title(\"Model 3\", fontdict={'size':22})\nplt.ylim((0,1.05))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T15:12:52.750441Z","iopub.execute_input":"2022-07-04T15:12:52.750675Z","iopub.status.idle":"2022-07-04T15:12:52.990914Z","shell.execute_reply.started":"2022-07-04T15:12:52.750639Z","shell.execute_reply":"2022-07-04T15:12:52.990099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now let's make prediction\n\nfinal_pred = []\n\npredictions1 = np.argmax(model1.predict(X_test), axis=1)\npredictions2 = np.argmax(model2.predict(X_test), axis=1)\npredictions3 = np.argmax(model3.predict(X_test), axis=1)\n\nfor idx,(a,b,c) in enumerate(zip(predictions1, predictions2, predictions3)):\n    if a!=b and a!=c and b!=c:\n        final_pred.append(b)\n        \n    elif a!=b or a!=c or b!=c:\n        final_pred.append(statistics.mode([a,b,c]))\n        \n    else:\n        final_pred.append(a)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T15:12:52.992150Z","iopub.execute_input":"2022-07-04T15:12:52.992834Z","iopub.status.idle":"2022-07-04T15:12:59.117441Z","shell.execute_reply.started":"2022-07-04T15:12:52.992793Z","shell.execute_reply":"2022-07-04T15:12:59.116700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction on test data\n\nfig = plt.figure(figsize=(18,18))\nindex = sample(range(0, X_test.shape[0]),16)\ncount = 0\n\nfor i in range(4):\n    for j in range(4):\n        plt.subplot(4, 4, count+1)\n        plt.imshow(X_test[index[count]].reshape((28,28)), cmap=plt.cm.binary)\n        plt.title(f\"Prediction: {final_pred[index[count]]}\")\n        count+=1","metadata":{"execution":{"iopub.status.busy":"2022-07-04T15:12:59.118752Z","iopub.execute_input":"2022-07-04T15:12:59.118981Z","iopub.status.idle":"2022-07-04T15:13:01.115722Z","shell.execute_reply.started":"2022-07-04T15:12:59.118949Z","shell.execute_reply":"2022-07-04T15:13:01.114962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submit prediction\n\nsubmission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\nsubmission[\"Label\"] = final_pred\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T15:13:01.116846Z","iopub.execute_input":"2022-07-04T15:13:01.117383Z","iopub.status.idle":"2022-07-04T15:13:01.196281Z","shell.execute_reply.started":"2022-07-04T15:13:01.117338Z","shell.execute_reply":"2022-07-04T15:13:01.195494Z"},"trusted":true},"execution_count":null,"outputs":[]}]}