{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Input, MaxPooling2D, ZeroPadding2D, Conv2D, Flatten, BatchNormalization, AveragePooling2D, Activation\nfrom keras.losses import categorical_crossentropy\nfrom keras.callbacks import EarlyStopping\nfrom keras.applications import vgg16\nfrom keras.regularizers import L2\n\nimport tensorflow as tf\nfrom sklearn.preprocessing import MinMaxScaler #normalizer\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\n\nimport keras\nfrom keras import layers\n\nfrom keras.utils.np_utils import to_categorical #onehog encoder\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-20T03:33:17.422096Z","iopub.execute_input":"2022-07-20T03:33:17.422497Z","iopub.status.idle":"2022-07-20T03:33:17.433180Z","shell.execute_reply.started":"2022-07-20T03:33:17.422466Z","shell.execute_reply":"2022-07-20T03:33:17.431636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('../input/digit-recognizer/test.csv')\ntrain = pd.read_csv('../input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:17.709176Z","iopub.execute_input":"2022-07-20T03:33:17.709600Z","iopub.status.idle":"2022-07-20T03:33:22.724927Z","shell.execute_reply.started":"2022-07-20T03:33:17.709552Z","shell.execute_reply":"2022-07-20T03:33:22.723680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = train.iloc[:,1:]; train_y = train.iloc[:,0]\ntest_X = test","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:22.727390Z","iopub.execute_input":"2022-07-20T03:33:22.727958Z","iopub.status.idle":"2022-07-20T03:33:22.737225Z","shell.execute_reply.started":"2022-07-20T03:33:22.727875Z","shell.execute_reply":"2022-07-20T03:33:22.735486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#basic statistics and graph\nsns.countplot(train_y)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:22.739475Z","iopub.execute_input":"2022-07-20T03:33:22.740067Z","iopub.status.idle":"2022-07-20T03:33:22.978384Z","shell.execute_reply.started":"2022-07-20T03:33:22.740020Z","shell.execute_reply":"2022-07-20T03:33:22.976945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#counting numbers of each labels\ntrain_y.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:22.982902Z","iopub.execute_input":"2022-07-20T03:33:22.983276Z","iopub.status.idle":"2022-07-20T03:33:22.992358Z","shell.execute_reply.started":"2022-07-20T03:33:22.983246Z","shell.execute_reply":"2022-07-20T03:33:22.991119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:22.994532Z","iopub.execute_input":"2022-07-20T03:33:22.995412Z","iopub.status.idle":"2022-07-20T03:33:23.009851Z","shell.execute_reply.started":"2022-07-20T03:33:22.995366Z","shell.execute_reply":"2022-07-20T03:33:23.008624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:23.013940Z","iopub.execute_input":"2022-07-20T03:33:23.014592Z","iopub.status.idle":"2022-07-20T03:33:23.026174Z","shell.execute_reply.started":"2022-07-20T03:33:23.014560Z","shell.execute_reply":"2022-07-20T03:33:23.024898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#normalizer\nscaler = MinMaxScaler()#by default, feature_range=(0, 1)\nscaler.fit(train_X)\ntrain_X = scaler.transform(train_X)\ntest_X = scaler.transform(test_X)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:23.028058Z","iopub.execute_input":"2022-07-20T03:33:23.028537Z","iopub.status.idle":"2022-07-20T03:33:23.597485Z","shell.execute_reply.started":"2022-07-20T03:33:23.028493Z","shell.execute_reply":"2022-07-20T03:33:23.596131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#onehog encoding\ntrain_y = to_categorical(train_y, num_classes = 10)\ntrain_y","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:23.599488Z","iopub.execute_input":"2022-07-20T03:33:23.599921Z","iopub.status.idle":"2022-07-20T03:33:23.611376Z","shell.execute_reply.started":"2022-07-20T03:33:23.599844Z","shell.execute_reply":"2022-07-20T03:33:23.609973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = train_X.reshape(-1,28,28,1)\ntest_X = test_X.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:23.613366Z","iopub.execute_input":"2022-07-20T03:33:23.614211Z","iopub.status.idle":"2022-07-20T03:33:23.620567Z","shell.execute_reply.started":"2022-07-20T03:33:23.614154Z","shell.execute_reply":"2022-07-20T03:33:23.619097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# seperate 20% of training data for validation\n\nX_train, X_validation, y_train, y_validation = train_test_split(\n    train_X, train_y, test_size=0.2, random_state=1234)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:23.626736Z","iopub.execute_input":"2022-07-20T03:33:23.628005Z","iopub.status.idle":"2022-07-20T03:33:24.050573Z","shell.execute_reply.started":"2022-07-20T03:33:23.627945Z","shell.execute_reply":"2022-07-20T03:33:24.049399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train_X.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-19T21:50:46.133797Z","iopub.execute_input":"2022-07-19T21:50:46.134115Z","iopub.status.idle":"2022-07-19T21:50:46.141162Z","shell.execute_reply.started":"2022-07-19T21:50:46.134085Z","shell.execute_reply":"2022-07-19T21:50:46.140122Z"}}},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:24.052458Z","iopub.execute_input":"2022-07-20T03:33:24.052862Z","iopub.status.idle":"2022-07-20T03:33:24.062630Z","shell.execute_reply.started":"2022-07-20T03:33:24.052816Z","shell.execute_reply":"2022-07-20T03:33:24.061307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn = Sequential(\n    [\n        keras.Input(shape=(28, 28, 1)),\n        layers.Conv2D(32, kernel_size=(4, 4), activation=\"relu\"),\n        layers.Conv2D(32, kernel_size=(4, 4), activation=\"relu\"),\n        layers.MaxPooling2D(pool_size=(2, 2)),\n        layers.Dropout(0.125),\n        layers.Conv2D(32, kernel_size=(4, 4), activation=\"relu\"),\n        layers.Conv2D(32, kernel_size=(4, 4), activation=\"relu\"),\n        layers.MaxPooling2D(pool_size=(2, 2)),\n        layers.Dropout(0.125),\n        layers.Flatten(),\n        layers.Dropout(0.25),\n        layers.Dense(256, activation = \"relu\"),\n        layers.Dropout(0.5),\n        layers.Dense(10, activation=\"softmax\"),\n        \n    ]\n)\ncnn.compile(\n    loss = categorical_crossentropy,\n    metrics = [\"accuracy\"],\n    optimizer = \"adam\"\n)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:24.064386Z","iopub.execute_input":"2022-07-20T03:33:24.067443Z","iopub.status.idle":"2022-07-20T03:33:24.161546Z","shell.execute_reply.started":"2022-07-20T03:33:24.067381Z","shell.execute_reply":"2022-07-20T03:33:24.160433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:24.163201Z","iopub.execute_input":"2022-07-20T03:33:24.163616Z","iopub.status.idle":"2022-07-20T03:33:24.172465Z","shell.execute_reply.started":"2022-07-20T03:33:24.163575Z","shell.execute_reply":"2022-07-20T03:33:24.170754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn.fit(\n    X_train,\n    y_train,\n    batch_size = 2000,\n    validation_data = (X_validation, y_validation),\n    epochs = 94,\n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:33:24.174441Z","iopub.execute_input":"2022-07-20T03:33:24.175333Z","iopub.status.idle":"2022-07-20T03:34:25.423519Z","shell.execute_reply.started":"2022-07-20T03:33:24.175289Z","shell.execute_reply":"2022-07-20T03:34:25.422110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = cnn.predict(test_X)\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:34:25.429030Z","iopub.execute_input":"2022-07-20T03:34:25.433016Z","iopub.status.idle":"2022-07-20T03:34:28.470149Z","shell.execute_reply.started":"2022-07-20T03:34:25.432962Z","shell.execute_reply":"2022-07-20T03:34:28.468863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raise SystemExit()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:34:28.474043Z","iopub.execute_input":"2022-07-20T03:34:28.475074Z","iopub.status.idle":"2022-07-20T03:34:28.487051Z","shell.execute_reply.started":"2022-07-20T03:34:28.475015Z","shell.execute_reply":"2022-07-20T03:34:28.484673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# below shows other 2 settings of the model. They are more complex but may not have better solution.","metadata":{}},{"cell_type":"code","source":"del cnn\ncnn = Sequential()\n\ncnn.add(ZeroPadding2D(padding = (1, 1), input_shape=(28, 28, 1)))\ncnn.add(Conv2D(filters=5, kernel_size=(5, 5), activation=\"relu\"))\ncnn.add(Conv2D(filters=5, kernel_size=(5, 5), activation=\"relu\"))\ncnn.add(MaxPooling2D(pool_size=(2, 2)))\ncnn.add(BatchNormalization())\n\ncnn.add(ZeroPadding2D(padding = (1, 1)))\ncnn.add(Conv2D(filters=7, kernel_size=(5, 5), activation=\"relu\"))\ncnn.add(Conv2D(filters=7, kernel_size=(5, 5), activation=\"relu\"))\ncnn.add(MaxPooling2D(pool_size=(2, 2)))\ncnn.add(BatchNormalization())\n\ncnn.add(Flatten())\ncnn.add(Dense(49, activation=\"relu\"))\ncnn.add(Dropout(0.1))\n\ncnn.add(Dense(10, activation=\"softmax\"))\n\n\ncnn.compile(\n    loss = categorical_crossentropy,\n    metrics = [\"accuracy\"],\n    optimizer = \"adam\"\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:34:28.489240Z","iopub.status.idle":"2022-07-20T03:34:28.490562Z","shell.execute_reply.started":"2022-07-20T03:34:28.490233Z","shell.execute_reply":"2022-07-20T03:34:28.490264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:24:12.689922Z","iopub.execute_input":"2022-07-20T03:24:12.690817Z","iopub.status.idle":"2022-07-20T03:24:12.702082Z","shell.execute_reply.started":"2022-07-20T03:24:12.690766Z","shell.execute_reply":"2022-07-20T03:24:12.700504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_hist = cnn.fit(\n    X_train,\n    y_train,\n    batch_size = 2000,\n    validation_data = (X_validation, y_validation),\n    epochs = 100,\n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:24:12.704101Z","iopub.execute_input":"2022-07-20T03:24:12.705194Z","iopub.status.idle":"2022-07-20T03:24:47.014799Z","shell.execute_reply.started":"2022-07-20T03:24:12.705149Z","shell.execute_reply":"2022-07-20T03:24:47.013558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_rgb = tf.image.grayscale_to_rgb(tf.convert_to_tensor(X_train), name=None)\nX_validation_rgb = tf.image.grayscale_to_rgb(tf.convert_to_tensor(X_validation), name=None)\n\n#let the image becomes 56x56x3\nX_train_rgb = tf.keras.backend.resize_images(X_train_rgb, height_factor=2, width_factor=2, data_format='channels_last')\nX_validation_rgb = tf.keras.backend.resize_images(X_validation_rgb, height_factor=2, width_factor=2,data_format='channels_last')","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:10:12.648550Z","iopub.execute_input":"2022-07-20T02:10:12.649021Z","iopub.status.idle":"2022-07-20T02:10:13.273643Z","shell.execute_reply.started":"2022-07-20T02:10:12.648984Z","shell.execute_reply":"2022-07-20T02:10:13.272676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrained_vgg = vgg16.VGG16(weights='imagenet', include_top=False, input_shape=(56, 56, 3))\npretrained_vgg.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:10:13.275199Z","iopub.execute_input":"2022-07-20T02:10:13.275557Z","iopub.status.idle":"2022-07-20T02:10:14.170229Z","shell.execute_reply.started":"2022-07-20T02:10:13.275518Z","shell.execute_reply":"2022-07-20T02:10:14.169214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weight_decay = 0.0005\nfor layer in pretrained_vgg.layers:\n    layer.trainable = False\nmodel = Sequential()\nmodel.add(pretrained_vgg)\nmodel.add(Flatten())\nmodel.add(Dense(512, kernel_regularizer=L2(weight_decay)))\nmodel.add(Activation('relu'))\nmodel.add(BatchNormalization())\n\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10))\nmodel.add(Activation('softmax'))\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:10:14.171735Z","iopub.execute_input":"2022-07-20T02:10:14.172091Z","iopub.status.idle":"2022-07-20T02:10:14.266275Z","shell.execute_reply.started":"2022-07-20T02:10:14.172056Z","shell.execute_reply":"2022-07-20T02:10:14.265296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compile model\nmodel.compile(\n    optimizer='rmsprop',\n    loss='CategoricalCrossentropy',\n    metrics=['accuracy']\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:10:14.267771Z","iopub.execute_input":"2022-07-20T02:10:14.268133Z","iopub.status.idle":"2022-07-20T02:10:14.278543Z","shell.execute_reply.started":"2022-07-20T02:10:14.268097Z","shell.execute_reply":"2022-07-20T02:10:14.277580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callback = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=200)\n\nmodel_hist = model.fit(\n    X_train_rgb,\n    y_train,\n    validation_data = (X_validation_rgb, y_validation),\n    epochs = 30,\n    verbose = 1,\n    callbacks = [callback]\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:10:14.280940Z","iopub.execute_input":"2022-07-20T02:10:14.281222Z","iopub.status.idle":"2022-07-20T02:16:42.493027Z","shell.execute_reply.started":"2022-07-20T02:10:14.281197Z","shell.execute_reply":"2022-07-20T02:16:42.492089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1st approach: 0.9713\n\n2nd approach: 0.9787\n\n3rd approach: accuracy 0.9657","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}