{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Google - Isolated Sign Language Recognition\n#### In this notebook I am just trying to get familiar with the Sign Language Recognition by using exisiting dataset \"Sign language mnist\"\n\n![](https://www.researchgate.net/publication/328396430/figure/fig1/AS:683619848830976@1539999081795/The-26-letters-and-10-digits-of-American-Sign-Language-ASL_W640.jpg)","metadata":{}},{"cell_type":"markdown","source":"# 1. American Sign Language (ASL)","metadata":{}},{"cell_type":"code","source":"# Importing required libraries\n\nfrom sklearn.preprocessing import LabelBinarizer\nfrom tensorflow import keras\nfrom keras.utils import plot_model\n\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pickle","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-23T20:56:36.684017Z","iopub.execute_input":"2023-02-23T20:56:36.684919Z","iopub.status.idle":"2023-02-23T20:56:45.148867Z","shell.execute_reply.started":"2023-02-23T20:56:36.684810Z","shell.execute_reply":"2023-02-23T20:56:45.147840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loading the training data (X+y)\ntrain_df = pd.read_csv('/kaggle/input/asl-signs/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:45.150992Z","iopub.execute_input":"2023-02-23T20:56:45.151744Z","iopub.status.idle":"2023-02-23T20:56:45.357243Z","shell.execute_reply.started":"2023-02-23T20:56:45.151705Z","shell.execute_reply":"2023-02-23T20:56:45.356228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.sample(frac=1, random_state=42) # Shuffiling the entire dataset","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:45.358599Z","iopub.execute_input":"2023-02-23T20:56:45.358944Z","iopub.status.idle":"2023-02-23T20:56:45.382190Z","shell.execute_reply.started":"2023-02-23T20:56:45.358911Z","shell.execute_reply":"2023-02-23T20:56:45.381246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquet_file = '/kaggle/input/asl-signs/train_landmark_files/16069/100015657.parquet'\nnew_parquet_df = pd.read_parquet(parquet_file)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:45.388415Z","iopub.execute_input":"2023-02-23T20:56:45.390248Z","iopub.status.idle":"2023-02-23T20:56:45.578276Z","shell.execute_reply.started":"2023-02-23T20:56:45.390215Z","shell.execute_reply":"2023-02-23T20:56:45.577362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_parquet_df","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:45.581751Z","iopub.execute_input":"2023-02-23T20:56:45.582043Z","iopub.status.idle":"2023-02-23T20:56:45.606656Z","shell.execute_reply.started":"2023-02-23T20:56:45.582012Z","shell.execute_reply":"2023-02-23T20:56:45.605721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign_to_prediction_index_map = {\n                                  \"TV\": 0,\n                                  \"after\": 1,\n                                  \"airplane\": 2,\n                                  \"all\": 3,\n                                  \"alligator\": 4,\n                                  \"animal\": 5,\n                                  \"another\": 6,\n                                  \"any\": 7,\n                                  \"apple\": 8,\n                                  \"arm\": 9,\n                                  \"aunt\": 10,\n                                  \"awake\": 11,\n                                  \"backyard\": 12,\n                                  \"bad\": 13,\n                                  \"balloon\": 14,\n                                  \"bath\": 15,\n                                  \"because\": 16,\n                                  \"bed\": 17,\n                                  \"bedroom\": 18,\n                                  \"bee\": 19,\n                                  \"before\": 20,\n                                  \"beside\": 21,\n                                  \"better\": 22,\n                                  \"bird\": 23,\n                                  \"black\": 24,\n                                  \"blow\": 25,\n                                  \"blue\": 26,\n                                  \"boat\": 27,\n                                  \"book\": 28,\n                                  \"boy\": 29,\n                                  \"brother\": 30,\n                                  \"brown\": 31,\n                                  \"bug\": 32,\n                                  \"bye\": 33,\n                                  \"callonphone\": 34,\n                                  \"can\": 35,\n                                  \"car\": 36,\n                                  \"carrot\": 37,\n                                  \"cat\": 38,\n                                  \"cereal\": 39,\n                                  \"chair\": 40,\n                                  \"cheek\": 41,\n                                  \"child\": 42,\n                                  \"chin\": 43,\n                                  \"chocolate\": 44,\n                                  \"clean\": 45,\n                                  \"close\": 46,\n                                  \"closet\": 47,\n                                  \"cloud\": 48,\n                                  \"clown\": 49,\n                                  \"cow\": 50,\n                                  \"cowboy\": 51,\n                                  \"cry\": 52,\n                                  \"cut\": 53,\n                                  \"cute\": 54,\n                                  \"dad\": 55,\n                                  \"dance\": 56,\n                                  \"dirty\": 57,\n                                  \"dog\": 58,\n                                  \"doll\": 59,\n                                  \"donkey\": 60,\n                                  \"down\": 61,\n                                  \"drawer\": 62,\n                                  \"drink\": 63,\n                                  \"drop\": 64,\n                                  \"dry\": 65,\n                                  \"dryer\": 66,\n                                  \"duck\": 67,\n                                  \"ear\": 68,\n                                  \"elephant\": 69,\n                                  \"empty\": 70,\n                                  \"every\": 71,\n                                  \"eye\": 72,\n                                  \"face\": 73,\n                                  \"fall\": 74,\n                                  \"farm\": 75,\n                                  \"fast\": 76,\n                                  \"feet\": 77,\n                                  \"find\": 78,\n                                  \"fine\": 79,\n                                  \"finger\": 80,\n                                  \"finish\": 81,\n                                  \"fireman\": 82,\n                                  \"first\": 83,\n                                  \"fish\": 84,\n                                  \"flag\": 85,\n                                  \"flower\": 86,\n                                  \"food\": 87,\n                                  \"for\": 88,\n                                  \"frenchfries\": 89,\n                                  \"frog\": 90,\n                                  \"garbage\": 91,\n                                  \"gift\": 92,\n                                  \"giraffe\": 93,\n                                  \"girl\": 94,\n                                  \"give\": 95,\n                                  \"glasswindow\": 96,\n                                  \"go\": 97,\n                                  \"goose\": 98,\n                                  \"grandma\": 99,\n                                  \"grandpa\": 100,\n                                  \"grass\": 101,\n                                  \"green\": 102,\n                                  \"gum\": 103,\n                                  \"hair\": 104,\n                                  \"happy\": 105,\n                                  \"hat\": 106,\n                                  \"hate\": 107,\n                                  \"have\": 108,\n                                  \"haveto\": 109,\n                                  \"head\": 110,\n                                  \"hear\": 111,\n                                  \"helicopter\": 112,\n                                  \"hello\": 113,\n                                  \"hen\": 114,\n                                  \"hesheit\": 115,\n                                  \"hide\": 116,\n                                  \"high\": 117,\n                                  \"home\": 118,\n                                  \"horse\": 119,\n                                  \"hot\": 120,\n                                  \"hungry\": 121,\n                                  \"icecream\": 122,\n                                  \"if\": 123,\n                                  \"into\": 124,\n                                  \"jacket\": 125,\n                                  \"jeans\": 126,\n                                  \"jump\": 127,\n                                  \"kiss\": 128,\n                                  \"kitty\": 129,\n                                  \"lamp\": 130,\n                                  \"later\": 131,\n                                  \"like\": 132,\n                                  \"lion\": 133,\n                                  \"lips\": 134,\n                                  \"listen\": 135,\n                                  \"look\": 136,\n                                  \"loud\": 137,\n                                  \"mad\": 138,\n                                  \"make\": 139,\n                                  \"man\": 140,\n                                  \"many\": 141,\n                                  \"milk\": 142,\n                                  \"minemy\": 143,\n                                  \"mitten\": 144,\n                                  \"mom\": 145,\n                                  \"moon\": 146,\n                                  \"morning\": 147,\n                                  \"mouse\": 148,\n                                  \"mouth\": 149,\n                                  \"nap\": 150,\n                                  \"napkin\": 151,\n                                  \"night\": 152,\n                                  \"no\": 153,\n                                  \"noisy\": 154,\n                                  \"nose\": 155,\n                                  \"not\": 156,\n                                  \"now\": 157,\n                                  \"nuts\": 158,\n                                  \"old\": 159,\n                                  \"on\": 160,\n                                  \"open\": 161,\n                                  \"orange\": 162,\n                                  \"outside\": 163,\n                                  \"owie\": 164,\n                                  \"owl\": 165,\n                                  \"pajamas\": 166,\n                                  \"pen\": 167,\n                                  \"pencil\": 168,\n                                  \"penny\": 169,\n                                  \"person\": 170,\n                                  \"pig\": 171,\n                                  \"pizza\": 172,\n                                  \"please\": 173,\n                                  \"police\": 174,\n                                  \"pool\": 175,\n                                  \"potty\": 176,\n                                  \"pretend\": 177,\n                                  \"pretty\": 178,\n                                  \"puppy\": 179,\n                                  \"puzzle\": 180,\n                                  \"quiet\": 181,\n                                  \"radio\": 182,\n                                  \"rain\": 183,\n                                  \"read\": 184,\n                                  \"red\": 185,\n                                  \"refrigerator\": 186,\n                                  \"ride\": 187,\n                                  \"room\": 188,\n                                  \"sad\": 189,\n                                  \"same\": 190,\n                                  \"say\": 191,\n                                  \"scissors\": 192,\n                                  \"see\": 193,\n                                  \"shhh\": 194,\n                                  \"shirt\": 195,\n                                  \"shoe\": 196,\n                                  \"shower\": 197,\n                                  \"sick\": 198,\n                                  \"sleep\": 199,\n                                  \"sleepy\": 200,\n                                  \"smile\": 201,\n                                  \"snack\": 202,\n                                  \"snow\": 203,\n                                  \"stairs\": 204,\n                                  \"stay\": 205,\n                                  \"sticky\": 206,\n                                  \"store\": 207,\n                                  \"story\": 208,\n                                  \"stuck\": 209,\n                                  \"sun\": 210,\n                                  \"table\": 211,\n                                  \"talk\": 212,\n                                  \"taste\": 213,\n                                  \"thankyou\": 214,\n                                  \"that\": 215,\n                                  \"there\": 216,\n                                  \"think\": 217,\n                                  \"thirsty\": 218,\n                                  \"tiger\": 219,\n                                  \"time\": 220,\n                                  \"tomorrow\": 221,\n                                  \"tongue\": 222,\n                                  \"tooth\": 223,\n                                  \"toothbrush\": 224,\n                                  \"touch\": 225,\n                                  \"toy\": 226,\n                                  \"tree\": 227,\n                                  \"uncle\": 228,\n                                  \"underwear\": 229,\n                                  \"up\": 230,\n                                  \"vacuum\": 231,\n                                  \"wait\": 232,\n                                  \"wake\": 233,\n                                  \"water\": 234,\n                                  \"wet\": 235,\n                                  \"weus\": 236,\n                                  \"where\": 237,\n                                  \"white\": 238,\n                                  \"who\": 239,\n                                  \"why\": 240,\n                                  \"will\": 241,\n                                  \"wolf\": 242,\n                                  \"yellow\": 243,\n                                  \"yes\": 244,\n                                  \"yesterday\": 245,\n                                  \"yourself\": 246,\n                                  \"yucky\": 247,\n                                  \"zebra\": 248,\n                                  \"zipper\": 249\n}","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:45.608229Z","iopub.execute_input":"2023-02-23T20:56:45.608824Z","iopub.status.idle":"2023-02-23T20:56:45.637528Z","shell.execute_reply.started":"2023-02-23T20:56:45.608787Z","shell.execute_reply":"2023-02-23T20:56:45.636301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loading the training data (X+y)\ntrain_df = pd.read_csv('/kaggle/input/sign-language-mnist/sign_mnist_train/sign_mnist_train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:45.639083Z","iopub.execute_input":"2023-02-23T20:56:45.639569Z","iopub.status.idle":"2023-02-23T20:56:48.983595Z","shell.execute_reply.started":"2023-02-23T20:56:45.639531Z","shell.execute_reply":"2023-02-23T20:56:48.982615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.sample(frac=1, random_state=42) # Shuffiling the entire dataset","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:48.984914Z","iopub.execute_input":"2023-02-23T20:56:48.985609Z","iopub.status.idle":"2023-02-23T20:56:49.124195Z","shell.execute_reply.started":"2023-02-23T20:56:48.985555Z","shell.execute_reply":"2023-02-23T20:56:49.123161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y = train_df.drop('label', axis=1), train_df['label'] # Split the dataset into X, y","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:49.125774Z","iopub.execute_input":"2023-02-23T20:56:49.126229Z","iopub.status.idle":"2023-02-23T20:56:49.185766Z","shell.execute_reply.started":"2023-02-23T20:56:49.126195Z","shell.execute_reply":"2023-02-23T20:56:49.184646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, y.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:49.190889Z","iopub.execute_input":"2023-02-23T20:56:49.191241Z","iopub.status.idle":"2023-02-23T20:56:49.198911Z","shell.execute_reply.started":"2023-02-23T20:56:49.191198Z","shell.execute_reply":"2023-02-23T20:56:49.197571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(X.dtypes), y.dtype","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:49.200291Z","iopub.execute_input":"2023-02-23T20:56:49.200806Z","iopub.status.idle":"2023-02-23T20:56:49.211800Z","shell.execute_reply.started":"2023-02-23T20:56:49.200771Z","shell.execute_reply":"2023-02-23T20:56:49.210365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_binarizer = LabelBinarizer() # Binarize labels in a one-vs-all fashion (return one-hot encoded vectors)\ny = label_binarizer.fit_transform(y)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:49.213867Z","iopub.execute_input":"2023-02-23T20:56:49.214126Z","iopub.status.idle":"2023-02-23T20:56:49.228330Z","shell.execute_reply.started":"2023-02-23T20:56:49.214102Z","shell.execute_reply":"2023-02-23T20:56:49.227370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X/255.0 # Normalizing the training data and converting the data type to float","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:49.229767Z","iopub.execute_input":"2023-02-23T20:56:49.230796Z","iopub.status.idle":"2023-02-23T20:56:49.303252Z","shell.execute_reply.started":"2023-02-23T20:56:49.230752Z","shell.execute_reply":"2023-02-23T20:56:49.302296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(X.dtypes)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:49.304647Z","iopub.execute_input":"2023-02-23T20:56:49.305010Z","iopub.status.idle":"2023-02-23T20:56:49.313522Z","shell.execute_reply.started":"2023-02-23T20:56:49.304973Z","shell.execute_reply":"2023-02-23T20:56:49.312400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Converting the 1-D array of 784 pixels to (28, 28, 1) Image\n# (28, 28) represents the spatial dimensions of the image & 1 specifies that the image is grayscale\nX = tf.reshape(X, [-1, 28, 28, 1])","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:49.315149Z","iopub.execute_input":"2023-02-23T20:56:49.315693Z","iopub.status.idle":"2023-02-23T20:56:51.993469Z","shell.execute_reply.started":"2023-02-23T20:56:49.315657Z","shell.execute_reply":"2023-02-23T20:56:51.992462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, y.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:51.995563Z","iopub.execute_input":"2023-02-23T20:56:51.996297Z","iopub.status.idle":"2023-02-23T20:56:52.002998Z","shell.execute_reply.started":"2023-02-23T20:56:51.996259Z","shell.execute_reply":"2023-02-23T20:56:52.002002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generating a validation set\n\nX_train, X_valid = X[:25000], X[25000:]\ny_train, y_valid = y[:25000], y[25000:]","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:52.004772Z","iopub.execute_input":"2023-02-23T20:56:52.005561Z","iopub.status.idle":"2023-02-23T20:56:52.020291Z","shell.execute_reply.started":"2023-02-23T20:56:52.005484Z","shell.execute_reply":"2023-02-23T20:56:52.019191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[0].dtype","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:52.021320Z","iopub.execute_input":"2023-02-23T20:56:52.021593Z","iopub.status.idle":"2023-02-23T20:56:52.032758Z","shell.execute_reply.started":"2023-02-23T20:56:52.021554Z","shell.execute_reply":"2023-02-23T20:56:52.031861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[0].shape","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:52.034089Z","iopub.execute_input":"2023-02-23T20:56:52.035198Z","iopub.status.idle":"2023-02-23T20:56:52.053954Z","shell.execute_reply.started":"2023-02-23T20:56:52.035164Z","shell.execute_reply":"2023-02-23T20:56:52.052814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X[0], cmap='gray'), y[0]","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:52.055249Z","iopub.execute_input":"2023-02-23T20:56:52.056366Z","iopub.status.idle":"2023-02-23T20:56:52.298371Z","shell.execute_reply.started":"2023-02-23T20:56:52.056329Z","shell.execute_reply":"2023-02-23T20:56:52.297438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the CNN\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Conv2D(32, (5, 5), padding='same', activation='relu', input_shape=(28, 28, 1)))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Conv2D(64, (5, 5), padding='same', activation='relu'))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Conv2D(128, (5, 5), padding='same', activation='relu'))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Flatten())\nmodel.add(keras.layers.Dense(128, activation='relu'))\nmodel.add(keras.layers.Dense(24, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:52.299993Z","iopub.execute_input":"2023-02-23T20:56:52.300667Z","iopub.status.idle":"2023-02-23T20:56:52.402201Z","shell.execute_reply.started":"2023-02-23T20:56:52.300631Z","shell.execute_reply":"2023-02-23T20:56:52.401214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:52.403811Z","iopub.execute_input":"2023-02-23T20:56:52.404146Z","iopub.status.idle":"2023-02-23T20:56:52.435064Z","shell.execute_reply.started":"2023-02-23T20:56:52.404112Z","shell.execute_reply":"2023-02-23T20:56:52.434296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:52.436049Z","iopub.execute_input":"2023-02-23T20:56:52.436397Z","iopub.status.idle":"2023-02-23T20:56:52.461176Z","shell.execute_reply.started":"2023-02-23T20:56:52.436363Z","shell.execute_reply":"2023-02-23T20:56:52.460154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_best_cb = keras.callbacks.ModelCheckpoint('models/initial-end-to-end', save_best_only=True) # Saves the best model so far\nearly_stopping_cb = keras.callbacks.EarlyStopping(patience=5) # Interupts training when there is no progress","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:52.462764Z","iopub.execute_input":"2023-02-23T20:56:52.463384Z","iopub.status.idle":"2023-02-23T20:56:52.469188Z","shell.execute_reply.started":"2023-02-23T20:56:52.463348Z","shell.execute_reply":"2023-02-23T20:56:52.467511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The model is same is 'models/initial-end-to-end'\n# The history object is 'models/initial-end-to-end-history'\n\nhistory = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid), callbacks=[save_best_cb, early_stopping_cb])","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:56:52.472057Z","iopub.execute_input":"2023-02-23T20:56:52.472994Z","iopub.status.idle":"2023-02-23T20:58:10.914495Z","shell.execute_reply.started":"2023-02-23T20:56:52.472960Z","shell.execute_reply":"2023-02-23T20:58:10.913510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history.history # Contains the training related information for each epoch","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:58:10.918988Z","iopub.execute_input":"2023-02-23T20:58:10.919297Z","iopub.status.idle":"2023-02-23T20:58:10.929326Z","shell.execute_reply.started":"2023-02-23T20:58:10.919269Z","shell.execute_reply":"2023-02-23T20:58:10.928308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Saving the history object\n\nwith open('models/intial-end-to-end-history', 'wb') as history_file:\n    pickle.dump(history.history, history_file)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T20:59:42.548673Z","iopub.execute_input":"2023-02-23T20:59:42.549047Z","iopub.status.idle":"2023-02-23T20:59:42.554273Z","shell.execute_reply.started":"2023-02-23T20:59:42.549017Z","shell.execute_reply":"2023-02-23T20:59:42.552994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h = np.load('models/intial-end-to-end-history', allow_pickle=True)\nh","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:00:38.596435Z","iopub.execute_input":"2023-02-23T21:00:38.597414Z","iopub.status.idle":"2023-02-23T21:00:38.605602Z","shell.execute_reply.started":"2023-02-23T21:00:38.597378Z","shell.execute_reply":"2023-02-23T21:00:38.604369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = keras.models.load_model('models/initial-end-to-end')","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:01:05.384888Z","iopub.execute_input":"2023-02-23T21:01:05.385260Z","iopub.status.idle":"2023-02-23T21:01:15.878301Z","shell.execute_reply.started":"2023-02-23T21:01:05.385228Z","shell.execute_reply":"2023-02-23T21:01:15.877357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10, 5))\nn_epochs = len(h['loss'])\nax.plot(range(1, n_epochs+1), h['loss'], color='b', label='train_loss')\nax.plot(range(1, n_epochs+1), h['val_loss'], color='c', label='val_loss')\nax.plot(range(1, n_epochs+1), h['accuracy'], color='b', label='train_accuracy', linestyle='--')\nax.plot(range(1, n_epochs+1), h['val_accuracy'], color='c', label='val_accuracy', linestyle='--')\nax.set_xticks(range(1, n_epochs+1))\nax.legend()","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:01:23.509300Z","iopub.execute_input":"2023-02-23T21:01:23.509697Z","iopub.status.idle":"2023-02-23T21:01:23.820493Z","shell.execute_reply.started":"2023-02-23T21:01:23.509662Z","shell.execute_reply":"2023-02-23T21:01:23.819531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Loss Correction\n\nfig, ax = plt.subplots(figsize=(10, 5))\nn_epochs = len(h['loss'])\n\n# Shift training loss by 0.5 as training loss is measured during the epoch and validation loss is measured after the epoch\n\nx_loss = np.arange(n_epochs+1)-0.5\nax.plot(x_loss[x_loss >= 0], h['loss'], color='b', label='train_loss')\nax.plot(range(1, n_epochs+1), h['val_loss'], color='r', label='val_loss')\nax.plot(range(1, n_epochs+1), h['accuracy'], color='b', label='train_accuracy', linestyle='--')\nax.plot(range(1, n_epochs+1), h['val_accuracy'], color='r', label='val_accuracy', linestyle='--')\nax.set_xlim(0, n_epochs)\nax.set_xticks(range(1, n_epochs+1))\nax.legend()","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:01:40.044320Z","iopub.execute_input":"2023-02-23T21:01:40.045028Z","iopub.status.idle":"2023-02-23T21:01:40.353619Z","shell.execute_reply.started":"2023-02-23T21:01:40.044992Z","shell.execute_reply":"2023-02-23T21:01:40.352630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining a function the get the training and validation plots representing the accuracy and loss at each epoch\n\ndef get_train_val_plots(h, yticks=None, figsize=(10, 5)):\n    # h: Any dictionary like the history.history\n    \n    fig, ax = plt.subplots(figsize=figsize)\n    n_epochs = len(h['loss'])\n    x_loss = np.arange(n_epochs+1)-0.5\n    \n    ax.plot(x_loss[x_loss >= 0], h['loss'], color='b', label='train_loss')\n    ax.plot(range(1, n_epochs+1), h['val_loss'], color='r', label='val_loss')\n    ax.plot(range(1, n_epochs+1), h['accuracy'], color='b', label='train_accuracy', linestyle='--')\n    ax.plot(range(1, n_epochs+1), h['val_accuracy'], color='r', label='val_accuracy', linestyle='--')\n    ax.set_xlim(0, n_epochs)\n    ax.set_xticks(range(1, n_epochs+1))\n    if yticks is not None:\n        ax.set_yticks(yticks)\n    ax.legend()","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:01:55.706648Z","iopub.execute_input":"2023-02-23T21:01:55.707342Z","iopub.status.idle":"2023-02-23T21:01:55.715950Z","shell.execute_reply.started":"2023-02-23T21:01:55.707307Z","shell.execute_reply":"2023-02-23T21:01:55.714760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_train_val_plots(h)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:02:11.270320Z","iopub.execute_input":"2023-02-23T21:02:11.270760Z","iopub.status.idle":"2023-02-23T21:02:11.576539Z","shell.execute_reply.started":"2023-02-23T21:02:11.270718Z","shell.execute_reply":"2023-02-23T21:02:11.575536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/sign-language-mnist/sign_mnist_test/sign_mnist_test.csv') # Load the test data","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:02:48.121092Z","iopub.execute_input":"2023-02-23T21:02:48.121495Z","iopub.status.idle":"2023-02-23T21:02:49.323597Z","shell.execute_reply.started":"2023-02-23T21:02:48.121462Z","shell.execute_reply":"2023-02-23T21:02:49.322530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test, y_test = test_df.drop('label', axis=1), test_df['label']","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:02:57.771337Z","iopub.execute_input":"2023-02-23T21:02:57.771758Z","iopub.status.idle":"2023-02-23T21:02:57.790876Z","shell.execute_reply.started":"2023-02-23T21:02:57.771716Z","shell.execute_reply":"2023-02-23T21:02:57.789765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = tf.reshape(X_test, [-1, 28, 28, 1])","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:03:08.620088Z","iopub.execute_input":"2023-02-23T21:03:08.620453Z","iopub.status.idle":"2023-02-23T21:03:08.698064Z","shell.execute_reply.started":"2023-02-23T21:03:08.620420Z","shell.execute_reply":"2023-02-23T21:03:08.697066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = label_binarizer.transform(y_test)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:03:16.847235Z","iopub.execute_input":"2023-02-23T21:03:16.847637Z","iopub.status.idle":"2023-02-23T21:03:16.855015Z","shell.execute_reply.started":"2023-02-23T21:03:16.847600Z","shell.execute_reply":"2023-02-23T21:03:16.853936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model.evaluate(X_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:03:51.294544Z","iopub.execute_input":"2023-02-23T21:03:51.295002Z","iopub.status.idle":"2023-02-23T21:03:52.666109Z","shell.execute_reply.started":"2023-02-23T21:03:51.294963Z","shell.execute_reply":"2023-02-23T21:03:52.665145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocesses the input and evaluates the model\n\ndef evaluate_model(model, X_test, y_test, label_binarizer):\n    X_test_reshape = tf.reshape(X_test, [-1, 28, 28, 1])\n    y_test_labels = label_binarizer.transform(y_test)\n    results = model.evaluate(X_test_reshape, y_test_labels)\n    print(f'Loss: {results[0]:.3f} Accuracy: {results[1]:.3f}')","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:04:03.580611Z","iopub.execute_input":"2023-02-23T21:04:03.581299Z","iopub.status.idle":"2023-02-23T21:04:03.587265Z","shell.execute_reply.started":"2023-02-23T21:04:03.581262Z","shell.execute_reply":"2023-02-23T21:04:03.586034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = evaluate_model(best_model, test_df.drop('label', axis=1), test_df['label'], label_binarizer)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:04:24.227042Z","iopub.execute_input":"2023-02-23T21:04:24.227419Z","iopub.status.idle":"2023-02-23T21:04:27.069799Z","shell.execute_reply.started":"2023-02-23T21:04:24.227388Z","shell.execute_reply":"2023-02-23T21:04:27.068608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/sign-language-mnist/sign_mnist_test/sign_mnist_test.csv') # Load the test data","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:04:54.681927Z","iopub.execute_input":"2023-02-23T21:04:54.682310Z","iopub.status.idle":"2023-02-23T21:04:55.266760Z","shell.execute_reply.started":"2023-02-23T21:04:54.682277Z","shell.execute_reply":"2023-02-23T21:04:55.265766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test, y_test = test_df.drop('label', axis=1), test_df['label']","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:05:17.570869Z","iopub.execute_input":"2023-02-23T21:05:17.571229Z","iopub.status.idle":"2023-02-23T21:05:17.593062Z","shell.execute_reply.started":"2023-02-23T21:05:17.571191Z","shell.execute_reply":"2023-02-23T21:05:17.592029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = tf.reshape(X_test, [-1, 28, 28, 1])","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:05:27.617740Z","iopub.execute_input":"2023-02-23T21:05:27.618168Z","iopub.status.idle":"2023-02-23T21:05:27.706994Z","shell.execute_reply.started":"2023-02-23T21:05:27.618132Z","shell.execute_reply":"2023-02-23T21:05:27.706010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = {chr(ord('a') + i):i for i in range(26)}\nd_rev = {i:chr(ord('a') + i) for i in range(26)}\nd[' '] = d_rev[' '] = ' '","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:05:37.340395Z","iopub.execute_input":"2023-02-23T21:05:37.340785Z","iopub.status.idle":"2023-02-23T21:05:37.346472Z","shell.execute_reply.started":"2023-02-23T21:05:37.340753Z","shell.execute_reply":"2023-02-23T21:05:37.345327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence = 'sign language'\n\nfor i in sentence:\n    print(d[i], end=' ')","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:05:50.782567Z","iopub.execute_input":"2023-02-23T21:05:50.783299Z","iopub.status.idle":"2023-02-23T21:05:50.789276Z","shell.execute_reply.started":"2023-02-23T21:05:50.783262Z","shell.execute_reply":"2023-02-23T21:05:50.788270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model.predict(tf.reshape(X_test[0], [-1, 28, 28, 1]))","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:06:03.396113Z","iopub.execute_input":"2023-02-23T21:06:03.396568Z","iopub.status.idle":"2023-02-23T21:06:03.977294Z","shell.execute_reply.started":"2023-02-23T21:06:03.396526Z","shell.execute_reply":"2023-02-23T21:06:03.976372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_taken = []\nresult = ''\n\nfor i in sentence:\n    if i != ' ':\n        char_index = np.random.choice(y_test[y_test==ord(i)-ord('a')].index)\n        images_taken.append(char_index)\n        y_pred = best_model.predict(tf.reshape(X_test[char_index], [-1, 28, 28, 1]))\n        result += d_rev[label_binarizer.inverse_transform(y_pred)[0]]\n    else:\n        result += ' '\nprint(result)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:06:20.644902Z","iopub.execute_input":"2023-02-23T21:06:20.645270Z","iopub.status.idle":"2023-02-23T21:06:21.264251Z","shell.execute_reply.started":"2023-02-23T21:06:20.645238Z","shell.execute_reply":"2023-02-23T21:06:21.263253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualizing the test images\nimages_taken_dup = list(reversed(images_taken))\nfor word in sentence.split():\n    fig, ax = plt.subplots(1, len(word), figsize=(20, 20))\n    for i in range(len(word)):\n        ax[i].imshow(X_test[images_taken_dup.pop()], cmap='gray')\n        ax[i].set_title(word[i])","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:06:38.222372Z","iopub.execute_input":"2023-02-23T21:06:38.222788Z","iopub.status.idle":"2023-02-23T21:06:39.520707Z","shell.execute_reply.started":"2023-02-23T21:06:38.222754Z","shell.execute_reply":"2023-02-23T21:06:39.519572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_on_sentence(model, sentence, X_test, y_test, label_binarizer, figsize=(20, 20)):\n    # Random images are taken from X_test along with the corresponding labels in y_test\n    # based on the letters in the sentence.\n    # These images are fed to the model and its output is printed\n    \n    sentence = sentence.lower()\n    \n    d = {chr(ord('a') + i):i for i in range(26)}\n    d_rev = {i:chr(ord('a') + i) for i in range(26)}\n    d[' '] = d_rev[' '] = ' '\n    \n    \n    images_taken = []\n    result = ''\n    \n    X_test_reshape = tf.reshape(X_test, [-1, 28, 28, 1])\n    \n\n    for i in sentence:\n        if i != ' ':\n            char_index = np.random.choice(y_test[y_test==ord(i)-ord('a')].index)\n            images_taken.append(char_index)\n            y_pred = model.predict(tf.reshape(X_test_reshape[char_index], [1, 28, 28, 1]))\n            result += d_rev[label_binarizer.inverse_transform(y_pred)[0]]\n        else:\n            result += ' '\n            \n    print(f'The actual sentence is \"{sentence}\"')\n    print(f'The predicted sentence is \"{result}\"')\n        \n    images_taken.reverse()\n    for word in sentence.split():\n        fig, ax = plt.subplots(1, len(word), figsize=figsize)\n        for i in range(len(word)):\n            ax[i].imshow(X_test_reshape[images_taken.pop()], cmap='gray')\n            ax[i].set_title(word[i])","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:06:58.443326Z","iopub.execute_input":"2023-02-23T21:06:58.443784Z","iopub.status.idle":"2023-02-23T21:06:58.465284Z","shell.execute_reply.started":"2023-02-23T21:06:58.443744Z","shell.execute_reply":"2023-02-23T21:06:58.464027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_on_sentence(best_model, 'sign language', test_df.drop('label', axis=1), test_df['label'], label_binarizer)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:07:17.921024Z","iopub.execute_input":"2023-02-23T21:07:17.921405Z","iopub.status.idle":"2023-02-23T21:07:19.906629Z","shell.execute_reply.started":"2023-02-23T21:07:17.921371Z","shell.execute_reply":"2023-02-23T21:07:19.905551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initial End-to-End Workflow (End)\n### Hyperparameter Tuning\n1. Convolution and Max Pooling Pairs\n2. Filters in the convolution layers\n3. Filter Shape\n4. Dropout\n\n### Convolution and Max Pooling Pairs\n###### Before flattening\n\n1. For pair = 1 -> Output to the dense layer will be of the shape (None, 14, 14, 32)\n2. For pair = 2 -> Output to the dense layer will be of the shape (None, 7, 7, 64)\n3. For pair = 3 -> Output to the dense layer will be of the shape (None, 3, 3, 96)\n4. For pair = 4 -> Output to the dense layer will be of the shape (None, 1, 1, 128)\n###### As the output shape rapidly decreases for the pair = 4 it is better to choose among the pair = 1, 2 or 3","metadata":{}},{"cell_type":"code","source":"# 20% Validation Split\n\nX_train, X_valid = X[:19500], X[19500:]\ny_train, y_valid = y[:19500], y[19500:]","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:09:57.811663Z","iopub.execute_input":"2023-02-23T21:09:57.812041Z","iopub.status.idle":"2023-02-23T21:09:57.832066Z","shell.execute_reply.started":"2023-02-23T21:09:57.812010Z","shell.execute_reply":"2023-02-23T21:09:57.830978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_pairs = 3\nmodels_pairs = [keras.models.Sequential() for i in range(n_pairs)]\nearly_stopping_cb = keras.callbacks.EarlyStopping(patience=5)\n\nfor n in range(1, n_pairs+1):\n    models_pairs[n-1].add(keras.layers.Conv2D(32, (5, 5), padding='same', activation='relu', input_shape=(28, 28, 1)))\n    models_pairs[n-1].add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n    for i in range(1, n):\n        models_pairs[n-1].add(keras.layers.Conv2D(32*(i+1), (5, 5), padding='same', activation='relu'))\n        models_pairs[n-1].add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n    models_pairs[n-1].add(keras.layers.Flatten())\n    models_pairs[n-1].add(keras.layers.Dense(128, activation='relu'))\n    models_pairs[n-1].add(keras.layers.Dense(24, activation='softmax'))\n    models_pairs[n-1].compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    save_best_cb = keras.callbacks.ModelCheckpoint(f'models/experiment-1-{n}', save_best_only=True)\n    history = models_pairs[n-1].fit(X_train, y_train, epochs=15, validation_data=(X_valid, y_valid), callbacks=[save_best_cb, early_stopping_cb])\n    with open(f'models/experiment-1-{n}-history', 'wb') as history_file:\n        pickle.dump(history.history, history_file)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:10:33.732225Z","iopub.execute_input":"2023-02-23T21:10:33.732611Z","iopub.status.idle":"2023-02-23T21:14:14.390682Z","shell.execute_reply.started":"2023-02-23T21:10:33.732557Z","shell.execute_reply":"2023-02-23T21:14:14.389542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_pairs[0].summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:14:14.392733Z","iopub.execute_input":"2023-02-23T21:14:14.393418Z","iopub.status.idle":"2023-02-23T21:14:14.416737Z","shell.execute_reply.started":"2023-02-23T21:14:14.393378Z","shell.execute_reply":"2023-02-23T21:14:14.415929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_pairs[1].summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:14:14.417789Z","iopub.execute_input":"2023-02-23T21:14:14.418298Z","iopub.status.idle":"2023-02-23T21:14:14.443433Z","shell.execute_reply.started":"2023-02-23T21:14:14.418249Z","shell.execute_reply":"2023-02-23T21:14:14.442643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_pairs[2].summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:14:14.445742Z","iopub.execute_input":"2023-02-23T21:14:14.446216Z","iopub.status.idle":"2023-02-23T21:14:14.474667Z","shell.execute_reply.started":"2023-02-23T21:14:14.446178Z","shell.execute_reply":"2023-02-23T21:14:14.473885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index in range(n_pairs):\n    model = keras.models.load_model(f'models/experiment-1-{index+1}')\n    model.evaluate(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:14:14.475679Z","iopub.execute_input":"2023-02-23T21:14:14.476132Z","iopub.status.idle":"2023-02-23T21:14:20.438559Z","shell.execute_reply.started":"2023-02-23T21:14:14.476096Z","shell.execute_reply":"2023-02-23T21:14:20.437460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h_1_1 = np.load('models/experiment-1-1-history', allow_pickle=True)\nh_1_2 = np.load('models/experiment-1-2-history', allow_pickle=True)\nh_1_3 = np.load('models/experiment-1-3-history', allow_pickle=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:14:20.440210Z","iopub.execute_input":"2023-02-23T21:14:20.440608Z","iopub.status.idle":"2023-02-23T21:14:20.449616Z","shell.execute_reply.started":"2023-02-23T21:14:20.440556Z","shell.execute_reply":"2023-02-23T21:14:20.448630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_train_val_plots(h_1_1, yticks=np.arange(0, 1.2, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:14:20.450963Z","iopub.execute_input":"2023-02-23T21:14:20.451290Z","iopub.status.idle":"2023-02-23T21:14:20.756426Z","shell.execute_reply.started":"2023-02-23T21:14:20.451262Z","shell.execute_reply":"2023-02-23T21:14:20.755448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_train_val_plots(h_1_2, yticks=np.arange(0, 1.2, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:14:20.758177Z","iopub.execute_input":"2023-02-23T21:14:20.758595Z","iopub.status.idle":"2023-02-23T21:14:21.054486Z","shell.execute_reply.started":"2023-02-23T21:14:20.758543Z","shell.execute_reply":"2023-02-23T21:14:21.053548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_train_val_plots(h_1_3, yticks=np.arange(0, 1.2, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:14:21.056167Z","iopub.execute_input":"2023-02-23T21:14:21.056832Z","iopub.status.idle":"2023-02-23T21:14:21.347225Z","shell.execute_reply.started":"2023-02-23T21:14:21.056794Z","shell.execute_reply":"2023-02-23T21:14:21.346279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Model 3 has the least loss on the validation data\n\n### Filters\n##### Our Models now contains 3 pairs of Convolution and Pooling layers\n\n##### Number of filters maps can be\n\n1. 8 - 16 - 32\n2. 16 - 32 - 64\n3. 24 - 48 - 96","metadata":{}},{"cell_type":"code","source":"n_tests = 3\n\nmodels = []\nearly_stopping_cb = keras.callbacks.EarlyStopping(patience=5)\n\n\nfor i in range(n_tests):\n    model = keras.models.Sequential()\n    models.append(model)\n    model.add(keras.layers.Input(shape=(28, 28, 1)))\n    for pairs in range(3):\n        model.add(keras.layers.Conv2D((8*(i+1))*(2**pairs), (5, 5), padding='same', activation='relu'))\n        model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n    model.add(keras.layers.Flatten())\n    model.add(keras.layers.Dense(128, activation='relu'))\n    model.add(keras.layers.Dense(24, activation='softmax'))\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    save_best_cb = keras.callbacks.ModelCheckpoint(f'models/experiment-fmaps-{i+1}', save_best_only=True)\n    history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid), callbacks=[save_best_cb, early_stopping_cb])\n    with open(f'models/experiment-fmaps-{i+1}-history', 'wb') as history_file:\n        pickle.dump(history.history, history_file)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:18:25.949075Z","iopub.execute_input":"2023-02-23T21:18:25.949660Z","iopub.status.idle":"2023-02-23T21:20:28.505667Z","shell.execute_reply.started":"2023-02-23T21:18:25.949619Z","shell.execute_reply":"2023-02-23T21:20:28.504616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models[0].summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:20:28.508942Z","iopub.execute_input":"2023-02-23T21:20:28.509356Z","iopub.status.idle":"2023-02-23T21:20:28.538935Z","shell.execute_reply.started":"2023-02-23T21:20:28.509317Z","shell.execute_reply":"2023-02-23T21:20:28.538153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models[1].summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:20:28.539929Z","iopub.execute_input":"2023-02-23T21:20:28.540286Z","iopub.status.idle":"2023-02-23T21:20:28.570467Z","shell.execute_reply.started":"2023-02-23T21:20:28.540251Z","shell.execute_reply":"2023-02-23T21:20:28.569690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models[2].summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:20:28.572704Z","iopub.execute_input":"2023-02-23T21:20:28.573042Z","iopub.status.idle":"2023-02-23T21:20:28.600599Z","shell.execute_reply.started":"2023-02-23T21:20:28.573008Z","shell.execute_reply":"2023-02-23T21:20:28.599882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index in range(n_tests):\n    model = keras.models.load_model(f'models/experiment-fmaps-{index+1}')\n    model.evaluate(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:20:28.601527Z","iopub.execute_input":"2023-02-23T21:20:28.601873Z","iopub.status.idle":"2023-02-23T21:20:33.547847Z","shell.execute_reply.started":"2023-02-23T21:20:28.601839Z","shell.execute_reply":"2023-02-23T21:20:33.546841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h_2_1 = np.load('models/experiment-fmaps-1-history', allow_pickle=True)\nh_2_2 = np.load('models/experiment-fmaps-2-history', allow_pickle=True)\nh_2_3 = np.load('models/experiment-fmaps-3-history', allow_pickle=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:20:33.552562Z","iopub.execute_input":"2023-02-23T21:20:33.553244Z","iopub.status.idle":"2023-02-23T21:20:33.563192Z","shell.execute_reply.started":"2023-02-23T21:20:33.553196Z","shell.execute_reply":"2023-02-23T21:20:33.562348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_train_val_plots(h_2_1, yticks=np.arange(0, 1.2, 0.1))\nget_train_val_plots(h_2_2, yticks=np.arange(0, 1.2, 0.1))\nget_train_val_plots(h_2_3, yticks=np.arange(0, 1.2, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:20:33.565137Z","iopub.execute_input":"2023-02-23T21:20:33.565921Z","iopub.status.idle":"2023-02-23T21:20:34.465008Z","shell.execute_reply.started":"2023-02-23T21:20:33.565886Z","shell.execute_reply":"2023-02-23T21:20:34.463946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Models\n# 'models/experiment-fiters-1'\n\n# History objects\n# 'models/experiment-filters-1-history'\n\nsave_best_cb = keras.callbacks.ModelCheckpoint(f'models/experiment-fiters-1', save_best_only=True)\nearly_stopping_cb = keras.callbacks.EarlyStopping(patience=5)\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Conv2D(24, (3, 3), padding='same', activation='relu'))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Conv2D(48, (3, 3), padding='same', activation='relu'))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Conv2D(96, (3, 3), padding='same', activation='relu'))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Flatten())\nmodel.add(keras.layers.Dense(128, activation='relu'))\nmodel.add(keras.layers.Dense(24, activation='softmax'))\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nhistory = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid), callbacks=[save_best_cb, early_stopping_cb])\nwith open(f'models/experiment-filters-1-history', 'wb') as history_file:\n    pickle.dump(history.history, history_file)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:20:34.466699Z","iopub.execute_input":"2023-02-23T21:20:34.467315Z","iopub.status.idle":"2023-02-23T21:21:15.971810Z","shell.execute_reply.started":"2023-02-23T21:20:34.467276Z","shell.execute_reply":"2023-02-23T21:21:15.970654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.models.load_model('models/experiment-fiters-1/')\nmodel.evaluate(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:21:15.974480Z","iopub.execute_input":"2023-02-23T21:21:15.974843Z","iopub.status.idle":"2023-02-23T21:21:17.793380Z","shell.execute_reply.started":"2023-02-23T21:21:15.974811Z","shell.execute_reply":"2023-02-23T21:21:17.792445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h_2_3 = np.load('models/experiment-fmaps-3-history', allow_pickle=True)\nh = np.load('models/experiment-filters-1-history', allow_pickle=True)\nget_train_val_plots(h, yticks=np.arange(0, 1.2, 0.1))\nget_train_val_plots(h_2_3, yticks=np.arange(0, 1.2, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:21:17.794958Z","iopub.execute_input":"2023-02-23T21:21:17.795342Z","iopub.status.idle":"2023-02-23T21:21:18.338856Z","shell.execute_reply.started":"2023-02-23T21:21:17.795307Z","shell.execute_reply":"2023-02-23T21:21:18.337913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stopping_cb = keras.callbacks.EarlyStopping(patience=5)\n\ndropout_rates = [0.3, 0.4, 0.5]\n\nfor index, i in enumerate(dropout_rates):\n    model = keras.models.Sequential()\n    model.add(keras.layers.Conv2D(24, (5, 5), padding='same', activation='relu'))\n    model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n    model.add(keras.layers.Dropout(i))\n    model.add(keras.layers.Conv2D(48, (5, 5), padding='same', activation='relu'))\n    model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n    model.add(keras.layers.Dropout(i))\n    model.add(keras.layers.Conv2D(96, (5, 5), padding='same', activation='relu'))\n    model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n    model.add(keras.layers.Dropout(i))\n    model.add(keras.layers.Flatten())\n    model.add(keras.layers.Dense(128, activation='relu'))\n    model.add(keras.layers.Dropout(i))\n    model.add(keras.layers.Dense(24, activation='softmax'))\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    save_best_cb = keras.callbacks.ModelCheckpoint(f'models/experiment-dropout-{index}', save_best_only=True)\n    history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid), callbacks=[save_best_cb, early_stopping_cb])\n    with open(f'models/experiment-dropout-{index}-history', 'wb') as history_file:\n        pickle.dump(history.history, history_file)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:21:20.575811Z","iopub.execute_input":"2023-02-23T21:21:20.576180Z","iopub.status.idle":"2023-02-23T21:24:12.411736Z","shell.execute_reply.started":"2023-02-23T21:21:20.576150Z","shell.execute_reply":"2023-02-23T21:24:12.410652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index in range(3):\n    model = keras.models.load_model(f'models/experiment-dropout-{index}')\n    model.evaluate(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:24:12.415384Z","iopub.execute_input":"2023-02-23T21:24:12.415706Z","iopub.status.idle":"2023-02-23T21:24:18.069349Z","shell.execute_reply.started":"2023-02-23T21:24:12.415676Z","shell.execute_reply":"2023-02-23T21:24:18.068322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h_0 = np.load('models/experiment-dropout-0-history', allow_pickle=True)\nh_1 = np.load('models/experiment-dropout-1-history', allow_pickle=True)\nh_2 = np.load('models/experiment-dropout-2-history', allow_pickle=True)\n\nget_train_val_plots(h_0)\nget_train_val_plots(h_1)\nget_train_val_plots(h_2)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:24:18.071205Z","iopub.execute_input":"2023-02-23T21:24:18.071570Z","iopub.status.idle":"2023-02-23T21:24:18.807834Z","shell.execute_reply.started":"2023-02-23T21:24:18.071534Z","shell.execute_reply":"2023-02-23T21:24:18.806655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sample Code\n\ndata_augmentation = keras.models.Sequential()\ndata_augmentation.add(keras.layers.RandomRotation(0.1, fill_mode='nearest', input_shape=(28, 28, 1)))\ndata_augmentation.add(keras.layers.RandomZoom((0.15, 0.2), fill_mode='nearest'))\ndata_augmentation.add(keras.layers.RandomTranslation(0.1, 0.1, fill_mode='nearest'))\n\nmodel = keras.models.Sequential()\n# Add the layers from the above model","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:24:18.809996Z","iopub.execute_input":"2023-02-23T21:24:18.810656Z","iopub.status.idle":"2023-02-23T21:24:18.840458Z","shell.execute_reply.started":"2023-02-23T21:24:18.810615Z","shell.execute_reply":"2023-02-23T21:24:18.839538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Improves the models performance by decreasing the tendency to overfit\n\n## Other Hyperparameters to try\n1. Batch Normalization - It normalizes the layer inputs\n2. Deeper networks work well - Replacing the single convolution layer of filter size (5X5) with two successive consecutive convolution layers of filter size (3X3)\n3. Number of units in the dense layer and number of dense layers\n4. Replacing the MaxPooling Layer with a convolution layer having a stride > 1\n5. Optimizers\n6. Learning rate of the optimizer\n## Visualising the final model","metadata":{}},{"cell_type":"code","source":"best_model = keras.models.load_model('models/experiment-dropout-0/')\nplot_model(best_model, to_file='model.png', show_shapes=True, show_layer_names=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:24:48.627165Z","iopub.execute_input":"2023-02-23T21:24:48.627540Z","iopub.status.idle":"2023-02-23T21:24:50.079774Z","shell.execute_reply.started":"2023-02-23T21:24:48.627507Z","shell.execute_reply":"2023-02-23T21:24:50.078622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Performance on the Test Set","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/sign-language-mnist/sign_mnist_test/sign_mnist_test.csv')\nX_test, y_test = test_df.drop('label', axis=1), test_df['label']","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:25:54.753098Z","iopub.execute_input":"2023-02-23T21:25:54.753487Z","iopub.status.idle":"2023-02-23T21:25:55.418386Z","shell.execute_reply.started":"2023-02-23T21:25:54.753450Z","shell.execute_reply":"2023-02-23T21:25:55.416766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = keras.models.load_model('models/experiment-dropout-0/')","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:26:09.931874Z","iopub.execute_input":"2023-02-23T21:26:09.932239Z","iopub.status.idle":"2023-02-23T21:26:10.479959Z","shell.execute_reply.started":"2023-02-23T21:26:09.932209Z","shell.execute_reply":"2023-02-23T21:26:10.478972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_model(best_model, X_test, y_test, label_binarizer)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:26:17.607494Z","iopub.execute_input":"2023-02-23T21:26:17.610664Z","iopub.status.idle":"2023-02-23T21:26:18.921350Z","shell.execute_reply.started":"2023-02-23T21:26:17.610588Z","shell.execute_reply":"2023-02-23T21:26:18.920432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/sign-language-mnist/sign_mnist_test/sign_mnist_test.csv')\nX_test, y_test = test_df.drop('label', axis=1), test_df['label']","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:26:40.460518Z","iopub.execute_input":"2023-02-23T21:26:40.461748Z","iopub.status.idle":"2023-02-23T21:26:41.156170Z","shell.execute_reply.started":"2023-02-23T21:26:40.461704Z","shell.execute_reply":"2023-02-23T21:26:41.155161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Applying normalisation which is applied for X_train\nX_test /= 255.0","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:26:48.843925Z","iopub.execute_input":"2023-02-23T21:26:48.844399Z","iopub.status.idle":"2023-02-23T21:26:48.888861Z","shell.execute_reply.started":"2023-02-23T21:26:48.844360Z","shell.execute_reply":"2023-02-23T21:26:48.887303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = keras.models.load_model('models/experiment-dropout-0/')","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:26:59.047348Z","iopub.execute_input":"2023-02-23T21:26:59.047729Z","iopub.status.idle":"2023-02-23T21:26:59.567990Z","shell.execute_reply.started":"2023-02-23T21:26:59.047695Z","shell.execute_reply":"2023-02-23T21:26:59.566986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_model(best_model, X_test, y_test, label_binarizer)","metadata":{"execution":{"iopub.status.busy":"2023-02-23T21:27:07.624299Z","iopub.execute_input":"2023-02-23T21:27:07.624683Z","iopub.status.idle":"2023-02-23T21:27:09.188916Z","shell.execute_reply.started":"2023-02-23T21:27:07.624650Z","shell.execute_reply":"2023-02-23T21:27:09.187642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reference: [LINK](https://github.com/Sathwick-Reddy-M/Sign-Language-Recognition)","metadata":{}}]}