{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelBinarizer\nfrom tensorflow import keras\nfrom keras.utils import plot_model\n\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pickle","metadata":{"execution":{"iopub.status.busy":"2023-03-17T06:45:51.942393Z","iopub.execute_input":"2023-03-17T06:45:51.942676Z","iopub.status.idle":"2023-03-17T06:46:00.209108Z","shell.execute_reply.started":"2023-03-17T06:45:51.942644Z","shell.execute_reply":"2023-03-17T06:46:00.208002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/asl-signs/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T06:48:29.736472Z","iopub.execute_input":"2023-03-17T06:48:29.738021Z","iopub.status.idle":"2023-03-17T06:48:29.912818Z","shell.execute_reply.started":"2023-03-17T06:48:29.737975Z","shell.execute_reply":"2023-03-17T06:48:29.911675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.sample(frac=1, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T06:48:33.010585Z","iopub.execute_input":"2023-03-17T06:48:33.011409Z","iopub.status.idle":"2023-03-17T06:48:33.035587Z","shell.execute_reply.started":"2023-03-17T06:48:33.011369Z","shell.execute_reply":"2023-03-17T06:48:33.034611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquet_file = '/kaggle/input/asl-signs/train_landmark_files/16069/100015657.parquet'\nnew_parquet_df = pd.read_parquet(parquet_file)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T06:48:36.945626Z","iopub.execute_input":"2023-03-17T06:48:36.946426Z","iopub.status.idle":"2023-03-17T06:48:37.115335Z","shell.execute_reply.started":"2023-03-17T06:48:36.946386Z","shell.execute_reply":"2023-03-17T06:48:37.114491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_parquet_df","metadata":{"execution":{"iopub.status.busy":"2023-03-17T06:48:48.043609Z","iopub.execute_input":"2023-03-17T06:48:48.044102Z","iopub.status.idle":"2023-03-17T06:48:48.074413Z","shell.execute_reply.started":"2023-03-17T06:48:48.044060Z","shell.execute_reply":"2023-03-17T06:48:48.073298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign_to_prediction_index_map = {\n \"TV\": 0,\n \"after\": 1,\n \"airplane\": 2,\n \"all\": 3,\n \"alligator\": 4,\n \"animal\": 5,\n \"another\": 6,\n \"any\": 7,\n \"apple\": 8,\n \"arm\": 9,\n \"aunt\": 10,\n \"awake\": 11,\n \"backyard\": 12,\n \"bad\": 13,\n \"balloon\": 14,\n \"bath\": 15,\n \"because\": 16,\n \"bed\": 17,\n \"bedroom\": 18,\n \"bee\": 19,\n \"before\": 20,\n \"beside\": 21,\n \"better\": 22,\n \"bird\": 23,\n \"black\": 24,\n \"blow\": 25,\n \"blue\": 26,\n \"boat\": 27,\n \"book\": 28,\n \"boy\": 29,\n \"brother\": 30,\n \"brown\": 31,\n \"bug\": 32,\n \"bye\": 33,\n \"callonphone\": 34,\n \"can\": 35,\n \"car\": 36,\n \"carrot\": 37,\n \"cat\": 38,\n \"cereal\": 39,\n \"chair\": 40,\n \"cheek\": 41,\n \"child\": 42,\n \"chin\": 43,\n \"chocolate\": 44,\n \"clean\": 45,\n \"close\": 46,\n \"closet\": 47,\n \"cloud\": 48,\n \"clown\": 49,\n \"cow\": 50,\n \"cowboy\": 51,\n \"cry\": 52,\n \"cut\": 53,\n \"cute\": 54,\n \"dad\": 55,\n \"dance\": 56,\n \"dirty\": 57,\n \"dog\": 58,\n \"doll\": 59,\n \"donkey\": 60,\n \"down\": 61,\n \"drawer\": 62,\n \"drink\": 63,\n \"drop\": 64,\n \"dry\": 65,\n \"dryer\": 66,\n \"duck\": 67,\n \"ear\": 68,\n \"elephant\": 69,\n \"empty\": 70,\n \"every\": 71,\n \"eye\": 72,\n \"face\": 73,\n \"fall\": 74,\n \"farm\": 75,\n \"fast\": 76,\n \"feet\": 77,\n \"find\": 78,\n \"fine\": 79,\n \"finger\": 80,\n \"finish\": 81,\n \"fireman\": 82,\n \"first\": 83,\n \"fish\": 84,\n \"flag\": 85,\n \"flower\": 86,\n \"food\": 87,\n \"for\": 88,\n \"frenchfries\": 89,\n \"frog\": 90,\n \"garbage\": 91,\n \"gift\": 92,\n \"giraffe\": 93,\n \"girl\": 94,\n \"give\": 95,\n \"glasswindow\": 96,\n \"go\": 97,\n \"goose\": 98,\n \"grandma\": 99,\n \"grandpa\": 100,\n \"grass\": 101,\n \"green\": 102,\n \"gum\": 103,\n \"hair\": 104,\n \"happy\": 105,\n \"hat\": 106,\n \"hate\": 107,\n \"have\": 108,\n \"haveto\": 109,\n \"head\": 110,\n \"hear\": 111,\n \"helicopter\": 112,\n \"hello\": 113,\n \"hen\": 114,\n \"hesheit\": 115,\n \"hide\": 116,\n \"high\": 117,\n \"home\": 118,\n \"horse\": 119,\n \"hot\": 120,\n \"hungry\": 121,\n \"icecream\": 122,\n \"if\": 123,\n \"into\": 124,\n \"jacket\": 125,\n \"jeans\": 126,\n \"jump\": 127,\n \"kiss\": 128,\n \"kitty\": 129,\n \"lamp\": 130,\n \"later\": 131,\n \"like\": 132,\n \"lion\": 133,\n \"lips\": 134,\n \"listen\": 135,\n \"look\": 136,\n \"loud\": 137,\n \"mad\": 138,\n \"make\": 139,\n \"man\": 140,\n \"many\": 141,\n \"milk\": 142,\n \"minemy\": 143,\n \"mitten\": 144,\n \"mom\": 145,\n \"moon\": 146,\n \"morning\": 147,\n \"mouse\": 148,\n \"mouth\": 149,\n \"nap\": 150,\n \"napkin\": 151,\n \"night\": 152,\n \"no\": 153,\n \"noisy\": 154,\n \"nose\": 155,\n \"not\": 156,\n \"now\": 157,\n \"nuts\": 158,\n \"old\": 159,\n \"on\": 160,\n \"open\": 161,\n \"orange\": 162,\n \"outside\": 163,\n \"owie\": 164,\n \"owl\": 165,\n \"pajamas\": 166,\n \"pen\": 167,\n \"pencil\": 168,\n \"penny\": 169,\n \"person\": 170,\n \"pig\": 171,\n \"pizza\": 172,\n \"please\": 173,\n \"police\": 174,\n \"pool\": 175,\n \"potty\": 176,\n \"pretend\": 177,\n \"pretty\": 178,\n \"puppy\": 179,\n \"puzzle\": 180,\n \"quiet\": 181,\n \"radio\": 182,\n \"rain\": 183,\n \"read\": 184,\n \"red\": 185,\n \"refrigerator\": 186,\n \"ride\": 187,\n \"room\": 188,\n \"sad\": 189,\n \"same\": 190,\n \"say\": 191,\n \"scissors\": 192,\n \"see\": 193,\n \"shhh\": 194,\n \"shirt\": 195,\n \"shoe\": 196,\n \"shower\": 197,\n \"sick\": 198,\n \"sleep\": 199,\n \"sleepy\": 200,\n \"smile\": 201,\n \"snack\": 202,\n \"snow\": 203,\n \"stairs\": 204,\n \"stay\": 205,\n \"sticky\": 206,\n \"store\": 207,\n \"story\": 208,\n \"stuck\": 209,\n \"sun\": 210,\n \"table\": 211,\n \"talk\": 212,\n \"taste\": 213,\n \"thankyou\": 214,\n \"that\": 215,\n \"there\": 216,\n \"think\": 217,\n \"thirsty\": 218,\n \"tiger\": 219,\n \"time\": 220,\n \"tomorrow\": 221,\n \"tongue\": 222,\n \"tooth\": 223,\n \"toothbrush\": 224,\n \"touch\": 225,\n \"toy\": 226,\n \"tree\": 227,\n \"uncle\": 228,\n \"underwear\": 229,\n \"up\": 230,\n \"vacuum\": 231,\n \"wait\": 232,\n \"wake\": 233,\n \"water\": 234,\n \"wet\": 235,\n \"weus\": 236,\n \"where\": 237,\n \"white\": 238,\n \"who\": 239,\n \"why\": 240,\n \"will\": 241,\n \"wolf\": 242,\n \"yellow\": 243,\n \"yes\": 244,\n \"yesterday\": 245,\n \"yourself\": 246,\n \"yucky\": 247,\n \"zebra\": 248,\n \"zipper\": 249\n}","metadata":{"execution":{"iopub.status.busy":"2023-03-17T06:59:44.077394Z","iopub.execute_input":"2023-03-17T06:59:44.078287Z","iopub.status.idle":"2023-03-17T06:59:44.104467Z","shell.execute_reply.started":"2023-03-17T06:59:44.078248Z","shell.execute_reply":"2023-03-17T06:59:44.102961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/sign-language-mnist/sign_mnist_train/sign_mnist_train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T06:59:57.820126Z","iopub.execute_input":"2023-03-17T06:59:57.821042Z","iopub.status.idle":"2023-03-17T06:59:59.615914Z","shell.execute_reply.started":"2023-03-17T06:59:57.820992Z","shell.execute_reply":"2023-03-17T06:59:59.614703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.sample(frac=1, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:06.173526Z","iopub.execute_input":"2023-03-17T07:00:06.174248Z","iopub.status.idle":"2023-03-17T07:00:06.311753Z","shell.execute_reply.started":"2023-03-17T07:00:06.174210Z","shell.execute_reply":"2023-03-17T07:00:06.310694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y = train_df.drop('label', axis=1), train_df['label']","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:08.777574Z","iopub.execute_input":"2023-03-17T07:00:08.778274Z","iopub.status.idle":"2023-03-17T07:00:08.836335Z","shell.execute_reply.started":"2023-03-17T07:00:08.778237Z","shell.execute_reply":"2023-03-17T07:00:08.835260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, y.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:11.027550Z","iopub.execute_input":"2023-03-17T07:00:11.028247Z","iopub.status.idle":"2023-03-17T07:00:11.035250Z","shell.execute_reply.started":"2023-03-17T07:00:11.028209Z","shell.execute_reply":"2023-03-17T07:00:11.034115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(X.dtypes), y.dtype","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:13.601682Z","iopub.execute_input":"2023-03-17T07:00:13.602425Z","iopub.status.idle":"2023-03-17T07:00:13.609732Z","shell.execute_reply.started":"2023-03-17T07:00:13.602388Z","shell.execute_reply":"2023-03-17T07:00:13.608724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_binarizer = LabelBinarizer() # Binarize labels in a one-vs-all fashion (return one-hot encoded vectors)\ny = label_binarizer.fit_transform(y)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:16.684084Z","iopub.execute_input":"2023-03-17T07:00:16.684495Z","iopub.status.idle":"2023-03-17T07:00:16.705117Z","shell.execute_reply.started":"2023-03-17T07:00:16.684460Z","shell.execute_reply":"2023-03-17T07:00:16.704193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X/255.0","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:23.656748Z","iopub.execute_input":"2023-03-17T07:00:23.657321Z","iopub.status.idle":"2023-03-17T07:00:23.728105Z","shell.execute_reply.started":"2023-03-17T07:00:23.657280Z","shell.execute_reply":"2023-03-17T07:00:23.727065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(X.dtypes)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:25.967192Z","iopub.execute_input":"2023-03-17T07:00:25.967549Z","iopub.status.idle":"2023-03-17T07:00:25.975664Z","shell.execute_reply.started":"2023-03-17T07:00:25.967516Z","shell.execute_reply":"2023-03-17T07:00:25.974480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = tf.reshape(X, [-1, 28, 28, 1])","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:30.737250Z","iopub.execute_input":"2023-03-17T07:00:30.738016Z","iopub.status.idle":"2023-03-17T07:00:33.164921Z","shell.execute_reply.started":"2023-03-17T07:00:30.737976Z","shell.execute_reply":"2023-03-17T07:00:33.163895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, y.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:33.486021Z","iopub.execute_input":"2023-03-17T07:00:33.486596Z","iopub.status.idle":"2023-03-17T07:00:33.493089Z","shell.execute_reply.started":"2023-03-17T07:00:33.486562Z","shell.execute_reply":"2023-03-17T07:00:33.492075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nX_train, X_valid = X[:25000], X[25000:]\ny_train, y_valid = y[:25000], y[25000:]","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:38.031616Z","iopub.execute_input":"2023-03-17T07:00:38.032321Z","iopub.status.idle":"2023-03-17T07:00:38.046735Z","shell.execute_reply.started":"2023-03-17T07:00:38.032283Z","shell.execute_reply":"2023-03-17T07:00:38.045806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[0].dtype","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:44.626146Z","iopub.execute_input":"2023-03-17T07:00:44.626787Z","iopub.status.idle":"2023-03-17T07:00:44.637302Z","shell.execute_reply.started":"2023-03-17T07:00:44.626729Z","shell.execute_reply":"2023-03-17T07:00:44.636142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[0].shape","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:46.845449Z","iopub.execute_input":"2023-03-17T07:00:46.846559Z","iopub.status.idle":"2023-03-17T07:00:46.854789Z","shell.execute_reply.started":"2023-03-17T07:00:46.846510Z","shell.execute_reply":"2023-03-17T07:00:46.853628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X[0], cmap='gray'), y[0]","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:52.858887Z","iopub.execute_input":"2023-03-17T07:00:52.859363Z","iopub.status.idle":"2023-03-17T07:00:53.097778Z","shell.execute_reply.started":"2023-03-17T07:00:52.859328Z","shell.execute_reply":"2023-03-17T07:00:53.096733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the CNN\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Conv2D(32, (5, 5), padding='same', activation='relu', input_shape=(28, 28, 1)))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Conv2D(64, (5, 5), padding='same', activation='relu'))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Conv2D(128, (5, 5), padding='same', activation='relu'))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Flatten())\nmodel.add(keras.layers.Dense(128, activation='relu'))\nmodel.add(keras.layers.Dense(24, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:00:57.062860Z","iopub.execute_input":"2023-03-17T07:00:57.063223Z","iopub.status.idle":"2023-03-17T07:00:57.372792Z","shell.execute_reply.started":"2023-03-17T07:00:57.063192Z","shell.execute_reply":"2023-03-17T07:00:57.371811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:01:01.678502Z","iopub.execute_input":"2023-03-17T07:01:01.678973Z","iopub.status.idle":"2023-03-17T07:01:01.706358Z","shell.execute_reply.started":"2023-03-17T07:01:01.678935Z","shell.execute_reply":"2023-03-17T07:01:01.705589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:01:05.193562Z","iopub.execute_input":"2023-03-17T07:01:05.193936Z","iopub.status.idle":"2023-03-17T07:01:05.212357Z","shell.execute_reply.started":"2023-03-17T07:01:05.193903Z","shell.execute_reply":"2023-03-17T07:01:05.211433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_best_cb = keras.callbacks.ModelCheckpoint('models/initial-end-to-end', save_best_only=True) # Saves the best model so far\nearly_stopping_cb = keras.callbacks.EarlyStopping(patience=5)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:01:08.058253Z","iopub.execute_input":"2023-03-17T07:01:08.058616Z","iopub.status.idle":"2023-03-17T07:01:08.063897Z","shell.execute_reply.started":"2023-03-17T07:01:08.058582Z","shell.execute_reply":"2023-03-17T07:01:08.062608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid), callbacks=[save_best_cb, early_stopping_cb])","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:01:13.413313Z","iopub.execute_input":"2023-03-17T07:01:13.413914Z","iopub.status.idle":"2023-03-17T07:03:36.481917Z","shell.execute_reply.started":"2023-03-17T07:01:13.413877Z","shell.execute_reply":"2023-03-17T07:03:36.480807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history.history","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:06:24.852732Z","iopub.execute_input":"2023-03-17T07:06:24.853883Z","iopub.status.idle":"2023-03-17T07:06:24.863531Z","shell.execute_reply.started":"2023-03-17T07:06:24.853828Z","shell.execute_reply":"2023-03-17T07:06:24.862286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('models/intial-end-to-end-history', 'wb') as history_file:\n pickle.dump(history.history, history_file)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:06:31.557406Z","iopub.execute_input":"2023-03-17T07:06:31.558126Z","iopub.status.idle":"2023-03-17T07:06:31.563652Z","shell.execute_reply.started":"2023-03-17T07:06:31.558087Z","shell.execute_reply":"2023-03-17T07:06:31.562530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h = np.load('models/intial-end-to-end-history', allow_pickle=True)\nh","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:06:34.426031Z","iopub.execute_input":"2023-03-17T07:06:34.426627Z","iopub.status.idle":"2023-03-17T07:06:34.436395Z","shell.execute_reply.started":"2023-03-17T07:06:34.426590Z","shell.execute_reply":"2023-03-17T07:06:34.435284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = keras.models.load_model('models/initial-end-to-end')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:06:38.071596Z","iopub.execute_input":"2023-03-17T07:06:38.072440Z","iopub.status.idle":"2023-03-17T07:06:38.504158Z","shell.execute_reply.started":"2023-03-17T07:06:38.072384Z","shell.execute_reply":"2023-03-17T07:06:38.502640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10, 5))\nn_epochs = len(h['loss'])\nax.plot(range(1, n_epochs+1), h['loss'], color='b', label='train_loss')\nax.plot(range(1, n_epochs+1), h['val_loss'], color='c', label='val_loss')\nax.plot(range(1, n_epochs+1), h['accuracy'], color='b', label='train_accuracy', linestyle='--')\nax.plot(range(1, n_epochs+1), h['val_accuracy'], color='c', label='val_accuracy', linestyle='--')\nax.set_xticks(range(1, n_epochs+1))\nax.legend()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:06:40.897258Z","iopub.execute_input":"2023-03-17T07:06:40.897972Z","iopub.status.idle":"2023-03-17T07:06:41.211792Z","shell.execute_reply.started":"2023-03-17T07:06:40.897932Z","shell.execute_reply":"2023-03-17T07:06:41.210710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10, 5))\nn_epochs = len(h['loss'])\n\n# Shift training loss by 0.5 as training loss is measured during the epoch and validation loss is measured after the epoch\n\nx_loss = np.arange(n_epochs+1)-0.5\nax.plot(x_loss[x_loss >= 0], h['loss'], color='b', label='train_loss')\nax.plot(range(1, n_epochs+1), h['val_loss'], color='r', label='val_loss')\nax.plot(range(1, n_epochs+1), h['accuracy'], color='b', label='train_accuracy', linestyle='--')\nax.plot(range(1, n_epochs+1), h['val_accuracy'], color='r', label='val_accuracy', linestyle='--')\nax.set_xlim(0, n_epochs)\nax.set_xticks(range(1, n_epochs+1))\nax.legend()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:06:45.519672Z","iopub.execute_input":"2023-03-17T07:06:45.520401Z","iopub.status.idle":"2023-03-17T07:06:45.826858Z","shell.execute_reply.started":"2023-03-17T07:06:45.520363Z","shell.execute_reply":"2023-03-17T07:06:45.825776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_val_plots(h, yticks=None, figsize=(10, 5)):\n # h: Any dictionary like the history.history\n \n fig, ax = plt.subplots(figsize=figsize)\n n_epochs = len(h['loss'])\n x_loss = np.arange(n_epochs+1)-0.5\n \n ax.plot(x_loss[x_loss >= 0], h['loss'], color='b', label='train_loss')\n ax.plot(range(1, n_epochs+1), h['val_loss'], color='r', label='val_loss')\n ax.plot(range(1, n_epochs+1), h['accuracy'], color='b', label='train_accuracy', linestyle='--')\n ax.plot(range(1, n_epochs+1), h['val_accuracy'], color='r', label='val_accuracy', linestyle='--')\n ax.set_xlim(0, n_epochs)\n ax.set_xticks(range(1, n_epochs+1))\n if yticks is not None:\n ax.set_yticks(yticks)\n ax.legend()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:06:48.439645Z","iopub.execute_input":"2023-03-17T07:06:48.440674Z","iopub.status.idle":"2023-03-17T07:06:48.450734Z","shell.execute_reply.started":"2023-03-17T07:06:48.440625Z","shell.execute_reply":"2023-03-17T07:06:48.449582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_train_val_plots(h)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:06:52.262723Z","iopub.execute_input":"2023-03-17T07:06:52.263447Z","iopub.status.idle":"2023-03-17T07:06:52.665610Z","shell.execute_reply.started":"2023-03-17T07:06:52.263406Z","shell.execute_reply":"2023-03-17T07:06:52.664440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/sign-language-mnist/sign_mnist_test/sign_mnist_test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:06:56.775092Z","iopub.execute_input":"2023-03-17T07:06:56.775455Z","iopub.status.idle":"2023-03-17T07:06:57.580974Z","shell.execute_reply.started":"2023-03-17T07:06:56.775425Z","shell.execute_reply":"2023-03-17T07:06:57.579925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test, y_test = test_df.drop('label', axis=1), test_df['label']","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:06:58.771492Z","iopub.execute_input":"2023-03-17T07:06:58.772198Z","iopub.status.idle":"2023-03-17T07:06:58.793783Z","shell.execute_reply.started":"2023-03-17T07:06:58.772160Z","shell.execute_reply":"2023-03-17T07:06:58.792451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = tf.reshape(X_test, [-1, 28, 28, 1])","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:07:04.039009Z","iopub.execute_input":"2023-03-17T07:07:04.039373Z","iopub.status.idle":"2023-03-17T07:07:04.115729Z","shell.execute_reply.started":"2023-03-17T07:07:04.039339Z","shell.execute_reply":"2023-03-17T07:07:04.114685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = label_binarizer.transform(y_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:07:06.843672Z","iopub.execute_input":"2023-03-17T07:07:06.844603Z","iopub.status.idle":"2023-03-17T07:07:06.852121Z","shell.execute_reply.started":"2023-03-17T07:07:06.844567Z","shell.execute_reply":"2023-03-17T07:07:06.851019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model.evaluate(X_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:07:14.793403Z","iopub.execute_input":"2023-03-17T07:07:14.793935Z","iopub.status.idle":"2023-03-17T07:07:16.126247Z","shell.execute_reply.started":"2023-03-17T07:07:14.793896Z","shell.execute_reply":"2023-03-17T07:07:16.125246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef evaluate_model(model, X_test, y_test, label_binarizer):\n X_test_reshape = tf.reshape(X_test, [-1, 28, 28, 1])\n y_test_labels = label_binarizer.transform(y_test)\n results = model.evaluate(X_test_reshape, y_test_labels)\n print(f'Loss: {results[0]:.3f} Accuracy: {results[1]:.3f}')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:07:19.620611Z","iopub.execute_input":"2023-03-17T07:07:19.621345Z","iopub.status.idle":"2023-03-17T07:07:19.626999Z","shell.execute_reply.started":"2023-03-17T07:07:19.621306Z","shell.execute_reply":"2023-03-17T07:07:19.625834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = evaluate_model(best_model, test_df.drop('label', axis=1), test_df['label'], label_binarizer)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:07:25.691591Z","iopub.execute_input":"2023-03-17T07:07:25.692071Z","iopub.status.idle":"2023-03-17T07:07:26.741298Z","shell.execute_reply.started":"2023-03-17T07:07:25.692015Z","shell.execute_reply":"2023-03-17T07:07:26.740346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/sign-language-mnist/sign_mnist_test/sign_mnist_test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:07:40.827809Z","iopub.execute_input":"2023-03-17T07:07:40.828493Z","iopub.status.idle":"2023-03-17T07:07:41.450236Z","shell.execute_reply.started":"2023-03-17T07:07:40.828456Z","shell.execute_reply":"2023-03-17T07:07:41.449130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test, y_test = test_df.drop('label', axis=1), test_df['label']","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:08:00.670628Z","iopub.execute_input":"2023-03-17T07:08:00.671355Z","iopub.status.idle":"2023-03-17T07:08:00.692874Z","shell.execute_reply.started":"2023-03-17T07:08:00.671315Z","shell.execute_reply":"2023-03-17T07:08:00.691834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = tf.reshape(X_test, [-1, 28, 28, 1])","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:08:05.322352Z","iopub.execute_input":"2023-03-17T07:08:05.323019Z","iopub.status.idle":"2023-03-17T07:08:05.399142Z","shell.execute_reply.started":"2023-03-17T07:08:05.322966Z","shell.execute_reply":"2023-03-17T07:08:05.398108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = {chr(ord('a') + i):i for i in range(26)}\nd_rev = {i:chr(ord('a') + i) for i in range(26)}\nd[' '] = d_rev[' '] = ' '","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:08:08.159090Z","iopub.execute_input":"2023-03-17T07:08:08.160049Z","iopub.status.idle":"2023-03-17T07:08:08.167839Z","shell.execute_reply.started":"2023-03-17T07:08:08.159997Z","shell.execute_reply":"2023-03-17T07:08:08.166711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence = 'sign language'\n\nfor i in sentence:\n print(d[i], end=' ')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:08:11.492072Z","iopub.execute_input":"2023-03-17T07:08:11.493002Z","iopub.status.idle":"2023-03-17T07:08:11.499457Z","shell.execute_reply.started":"2023-03-17T07:08:11.492948Z","shell.execute_reply":"2023-03-17T07:08:11.498311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model.predict(tf.reshape(X_test[0], [-1, 28, 28, 1]))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_taken = []\nresult = ''\n\nfor i in sentence:\n if i != ' ':\n char_index = np.random.choice(y_test[y_test==ord(i)-ord('a')].index)\n images_taken.append(char_index)\n y_pred = best_model.predict(tf.reshape(X_test[char_index], [-1, 28, 28, 1]))\n result += d_rev[label_binarizer.inverse_transform(y_pred)[0]]\n else:\n result += ' '\nprint(result)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:08:16.087819Z","iopub.execute_input":"2023-03-17T07:08:16.088916Z","iopub.status.idle":"2023-03-17T07:08:16.858371Z","shell.execute_reply.started":"2023-03-17T07:08:16.088856Z","shell.execute_reply":"2023-03-17T07:08:16.857280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_taken_dup = list(reversed(images_taken))\nfor word in sentence.split():\n fig, ax = plt.subplots(1, len(word), figsize=(20, 20))\n for i in range(len(word)):\n ax[i].imshow(X_test[images_taken_dup.pop()], cmap='gray')\n ax[i].set_title(word[i])","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:08:20.160958Z","iopub.execute_input":"2023-03-17T07:08:20.161965Z","iopub.status.idle":"2023-03-17T07:08:21.655248Z","shell.execute_reply.started":"2023-03-17T07:08:20.161918Z","shell.execute_reply":"2023-03-17T07:08:21.654094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_on_sentence(model, sentence, X_test, y_test, label_binarizer, figsize=(20, 20)):\n # Random images are taken from X_test along with the corresponding labels in y_test\n # based on the letters in the sentence.\n # These images are fed to the model and its output is printed\n \n sentence = sentence.lower()\n \n d = {chr(ord('a') + i):i for i in range(26)}\n d_rev = {i:chr(ord('a') + i) for i in range(26)}\n d[' '] = d_rev[' '] = ' '\n \n \n images_taken = []\n result = ''\n \n X_test_reshape = tf.reshape(X_test, [-1, 28, 28, 1])\n \n\n for i in sentence:\n if i != ' ':\n char_index = np.random.choice(y_test[y_test==ord(i)-ord('a')].index)\n images_taken.append(char_index)\n y_pred = model.predict(tf.reshape(X_test_reshape[char_index], [1, 28, 28, 1]))\n result += d_rev[label_binarizer.inverse_transform(y_pred)[0]]\n else:\n result += ' '\n \n print(f'The actual sentence is \"{sentence}\"')\n print(f'The predicted sentence is \"{result}\"')\n \n images_taken.reverse()\n for word in sentence.split():\n fig, ax = plt.subplots(1, len(word), figsize=figsize)\n for i in range(len(word)):\n ax[i].imshow(X_test_reshape[images_taken.pop()], cmap='gray')\n ax[i].set_title(word[i])","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:08:25.497551Z","iopub.execute_input":"2023-03-17T07:08:25.498271Z","iopub.status.idle":"2023-03-17T07:08:25.510128Z","shell.execute_reply.started":"2023-03-17T07:08:25.498230Z","shell.execute_reply":"2023-03-17T07:08:25.508157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_on_sentence(best_model, 'sign language', test_df.drop('label', axis=1), test_df['label'], label_binarizer)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:08:31.282150Z","iopub.execute_input":"2023-03-17T07:08:31.282743Z","iopub.status.idle":"2023-03-17T07:08:33.841622Z","shell.execute_reply.started":"2023-03-17T07:08:31.282705Z","shell.execute_reply":"2023-03-17T07:08:33.840479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 20% Validation Split\n\nX_train, X_valid = X[:19500], X[19500:]\ny_train, y_valid = y[:19500], y[19500:]","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:08:37.243293Z","iopub.execute_input":"2023-03-17T07:08:37.244079Z","iopub.status.idle":"2023-03-17T07:08:37.262236Z","shell.execute_reply.started":"2023-03-17T07:08:37.244031Z","shell.execute_reply":"2023-03-17T07:08:37.261250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_pairs = 3\nmodels_pairs = [keras.models.Sequential() for i in range(n_pairs)]\nearly_stopping_cb = keras.callbacks.EarlyStopping(patience=5)\n\nfor n in range(1, n_pairs+1):\n models_pairs[n-1].add(keras.layers.Conv2D(32, (5, 5), padding='same', activation='relu', input_shape=(28, 28, 1)))\n models_pairs[n-1].add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n for i in range(1, n):\n models_pairs[n-1].add(keras.layers.Conv2D(32*(i+1), (5, 5), padding='same', activation='relu'))\n models_pairs[n-1].add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n models_pairs[n-1].add(keras.layers.Flatten())\n models_pairs[n-1].add(keras.layers.Dense(128, activation='relu'))\n models_pairs[n-1].add(keras.layers.Dense(24, activation='softmax'))\n models_pairs[n-1].compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n save_best_cb = keras.callbacks.ModelCheckpoint(f'models/experiment-1-{n}', save_best_only=True)\n history = models_pairs[n-1].fit(X_train, y_train, epochs=15, validation_data=(X_valid, y_valid), callbacks=[save_best_cb, early_stopping_cb])\n with open(f'models/experiment-1-{n}-history', 'wb') as history_file:\n pickle.dump(history.history, history_file)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:08:41.580798Z","iopub.execute_input":"2023-03-17T07:08:41.581177Z","iopub.status.idle":"2023-03-17T07:12:20.656878Z","shell.execute_reply.started":"2023-03-17T07:08:41.581144Z","shell.execute_reply":"2023-03-17T07:12:20.655885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_pairs[0].summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:22:52.929501Z","iopub.execute_input":"2023-03-17T07:22:52.930467Z","iopub.status.idle":"2023-03-17T07:22:52.954799Z","shell.execute_reply.started":"2023-03-17T07:22:52.930416Z","shell.execute_reply":"2023-03-17T07:22:52.953984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_pairs[1].summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:22:59.697957Z","iopub.execute_input":"2023-03-17T07:22:59.698557Z","iopub.status.idle":"2023-03-17T07:22:59.724297Z","shell.execute_reply.started":"2023-03-17T07:22:59.698521Z","shell.execute_reply":"2023-03-17T07:22:59.723538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_pairs[2].summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:25:37.999351Z","iopub.execute_input":"2023-03-17T07:25:38.000274Z","iopub.status.idle":"2023-03-17T07:25:38.027517Z","shell.execute_reply.started":"2023-03-17T07:25:38.000218Z","shell.execute_reply":"2023-03-17T07:25:38.026713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index in range(n_pairs):\n model = keras.models.load_model(f'models/experiment-1-{index+1}')\n model.evaluate(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:25:42.795237Z","iopub.execute_input":"2023-03-17T07:25:42.796279Z","iopub.status.idle":"2023-03-17T07:25:47.359980Z","shell.execute_reply.started":"2023-03-17T07:25:42.796228Z","shell.execute_reply":"2023-03-17T07:25:47.358986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h_1_1 = np.load('models/experiment-1-1-history', allow_pickle=True)\nh_1_2 = np.load('models/experiment-1-2-history', allow_pickle=True)\nh_1_3 = np.load('models/experiment-1-3-history', allow_pickle=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:25:52.310820Z","iopub.execute_input":"2023-03-17T07:25:52.311792Z","iopub.status.idle":"2023-03-17T07:25:52.318632Z","shell.execute_reply.started":"2023-03-17T07:25:52.311717Z","shell.execute_reply":"2023-03-17T07:25:52.317471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_train_val_plots(h_1_1, yticks=np.arange(0, 1.2, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:25:56.254513Z","iopub.execute_input":"2023-03-17T07:25:56.255522Z","iopub.status.idle":"2023-03-17T07:25:56.604819Z","shell.execute_reply.started":"2023-03-17T07:25:56.255479Z","shell.execute_reply":"2023-03-17T07:25:56.603805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_train_val_plots(h_1_2, yticks=np.arange(0, 1.2, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:26:01.350150Z","iopub.execute_input":"2023-03-17T07:26:01.350590Z","iopub.status.idle":"2023-03-17T07:26:01.683898Z","shell.execute_reply.started":"2023-03-17T07:26:01.350549Z","shell.execute_reply":"2023-03-17T07:26:01.682913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nget_train_val_plots(h_1_3, yticks=np.arange(0, 1.2, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:26:05.651073Z","iopub.execute_input":"2023-03-17T07:26:05.652258Z","iopub.status.idle":"2023-03-17T07:26:05.942700Z","shell.execute_reply.started":"2023-03-17T07:26:05.652207Z","shell.execute_reply":"2023-03-17T07:26:05.941711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_tests = 3\n\nmodels = []\nearly_stopping_cb = keras.callbacks.EarlyStopping(patience=5)\n\n\nfor i in range(n_tests):\n model = keras.models.Sequential()\n models.append(model)\n model.add(keras.layers.Input(shape=(28, 28, 1)))\n for pairs in range(3):\n model.add(keras.layers.Conv2D((8*(i+1))*(2**pairs), (5, 5), padding='same', activation='relu'))\n model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n model.add(keras.layers.Flatten())\n model.add(keras.layers.Dense(128, activation='relu'))\n model.add(keras.layers.Dense(24, activation='softmax'))\n model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n save_best_cb = keras.callbacks.ModelCheckpoint(f'models/experiment-fmaps-{i+1}', save_best_only=True)\n history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid), callbacks=[save_best_cb, early_stopping_cb])\n with open(f'models/experiment-fmaps-{i+1}-history', 'wb') as history_file:\n pickle.dump(history.history, history_file)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:26:10.083450Z","iopub.execute_input":"2023-03-17T07:26:10.083847Z","iopub.status.idle":"2023-03-17T07:29:44.182436Z","shell.execute_reply.started":"2023-03-17T07:26:10.083810Z","shell.execute_reply":"2023-03-17T07:29:44.181414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models[0].summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:29:44.273453Z","iopub.execute_input":"2023-03-17T07:29:44.273803Z","iopub.status.idle":"2023-03-17T07:29:44.303363Z","shell.execute_reply.started":"2023-03-17T07:29:44.273737Z","shell.execute_reply":"2023-03-17T07:29:44.302606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models[1].summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:29:44.304233Z","iopub.execute_input":"2023-03-17T07:29:44.304547Z","iopub.status.idle":"2023-03-17T07:29:44.334253Z","shell.execute_reply.started":"2023-03-17T07:29:44.304514Z","shell.execute_reply":"2023-03-17T07:29:44.333519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index in range(n_tests):\n model = keras.models.load_model(f'models/experiment-fmaps-{index+1}')\n model.evaluate(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:29:55.626111Z","iopub.execute_input":"2023-03-17T07:29:55.627104Z","iopub.status.idle":"2023-03-17T07:29:59.721205Z","shell.execute_reply.started":"2023-03-17T07:29:55.627051Z","shell.execute_reply":"2023-03-17T07:29:59.720223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h_2_1 = np.load('models/experiment-fmaps-1-history', allow_pickle=True)\nh_2_2 = np.load('models/experiment-fmaps-2-history', allow_pickle=True)\nh_2_3 = np.load('models/experiment-fmaps-3-history', allow_pickle=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:30:08.232592Z","iopub.execute_input":"2023-03-17T07:30:08.233303Z","iopub.status.idle":"2023-03-17T07:30:08.239537Z","shell.execute_reply.started":"2023-03-17T07:30:08.233265Z","shell.execute_reply":"2023-03-17T07:30:08.238362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_train_val_plots(h_2_1, yticks=np.arange(0, 1.2, 0.1))\nget_train_val_plots(h_2_2, yticks=np.arange(0, 1.2, 0.1))\nget_train_val_plots(h_2_3, yticks=np.arange(0, 1.2, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:30:11.561634Z","iopub.execute_input":"2023-03-17T07:30:11.562345Z","iopub.status.idle":"2023-03-17T07:30:12.398429Z","shell.execute_reply.started":"2023-03-17T07:30:11.562304Z","shell.execute_reply":"2023-03-17T07:30:12.397408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nsave_best_cb = keras.callbacks.ModelCheckpoint(f'models/experiment-fiters-1', save_best_only=True)\nearly_stopping_cb = keras.callbacks.EarlyStopping(patience=5)\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Conv2D(24, (3, 3), padding='same', activation='relu'))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Conv2D(48, (3, 3), padding='same', activation='relu'))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Conv2D(96, (3, 3), padding='same', activation='relu'))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(keras.layers.Flatten())\nmodel.add(keras.layers.Dense(128, activation='relu'))\nmodel.add(keras.layers.Dense(24, activation='softmax'))\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nhistory = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid), callbacks=[save_best_cb, early_stopping_cb])\nwith open(f'models/experiment-filters-1-history', 'wb') as history_file:\n pickle.dump(history.history, history_file)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:33:20.111907Z","iopub.execute_input":"2023-03-17T07:33:20.112610Z","iopub.status.idle":"2023-03-17T07:34:05.162301Z","shell.execute_reply.started":"2023-03-17T07:33:20.112571Z","shell.execute_reply":"2023-03-17T07:34:05.161294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.models.load_model('models/experiment-fiters-1/')\nmodel.evaluate(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:34:40.909541Z","iopub.execute_input":"2023-03-17T07:34:40.910754Z","iopub.status.idle":"2023-03-17T07:34:42.969016Z","shell.execute_reply.started":"2023-03-17T07:34:40.910708Z","shell.execute_reply":"2023-03-17T07:34:42.967794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h_2_3 = np.load('models/experiment-fmaps-3-history', allow_pickle=True)\nh = np.load('models/experiment-filters-1-history', allow_pickle=True)\nget_train_val_plots(h, yticks=np.arange(0, 1.2, 0.1))\nget_train_val_plots(h_2_3, yticks=np.arange(0, 1.2, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:36:57.723041Z","iopub.execute_input":"2023-03-17T07:36:57.724189Z","iopub.status.idle":"2023-03-17T07:36:58.269884Z","shell.execute_reply.started":"2023-03-17T07:36:57.724142Z","shell.execute_reply":"2023-03-17T07:36:58.268839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stopping_cb = keras.callbacks.EarlyStopping(patience=5)\n\ndropout_rates = [0.3, 0.4, 0.5]\n\nfor index, i in enumerate(dropout_rates):\n model = keras.models.Sequential()\n model.add(keras.layers.Conv2D(24, (5, 5), padding='same', activation='relu'))\n model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n model.add(keras.layers.Dropout(i))\n model.add(keras.layers.Conv2D(48, (5, 5), padding='same', activation='relu'))\n model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n model.add(keras.layers.Dropout(i))\n model.add(keras.layers.Conv2D(96, (5, 5), padding='same', activation='relu'))\n model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))\n model.add(keras.layers.Dropout(i))\n model.add(keras.layers.Flatten())\n model.add(keras.layers.Dense(128, activation='relu'))\n model.add(keras.layers.Dropout(i))\n model.add(keras.layers.Dense(24, activation='softmax'))\n model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n save_best_cb = keras.callbacks.ModelCheckpoint(f'models/experiment-dropout-{index}', save_best_only=True)\n history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid), callbacks=[save_best_cb, early_stopping_cb])\n with open(f'models/experiment-dropout-{index}-history', 'wb') as history_file:\n pickle.dump(history.history, history_file)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:37:06.470258Z","iopub.execute_input":"2023-03-17T07:37:06.470635Z","iopub.status.idle":"2023-03-17T07:40:12.420793Z","shell.execute_reply.started":"2023-03-17T07:37:06.470600Z","shell.execute_reply":"2023-03-17T07:40:12.419389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index in range(3):\n model = keras.models.load_model(f'models/experiment-dropout-{index}')\n model.evaluate(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:40:37.752864Z","iopub.execute_input":"2023-03-17T07:40:37.753167Z","iopub.status.idle":"2023-03-17T07:40:43.307326Z","shell.execute_reply.started":"2023-03-17T07:40:37.753131Z","shell.execute_reply":"2023-03-17T07:40:43.306278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h_0 = np.load('models/experiment-dropout-0-history', allow_pickle=True)\nh_1 = np.load('models/experiment-dropout-1-history', allow_pickle=True)\nh_2 = np.load('models/experiment-dropout-2-history', allow_pickle=True)\n\nget_train_val_plots(h_0)\nget_train_val_plots(h_1)\nget_train_val_plots(h_2)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:40:44.058891Z","iopub.execute_input":"2023-03-17T07:40:44.059235Z","iopub.status.idle":"2023-03-17T07:40:44.945432Z","shell.execute_reply.started":"2023-03-17T07:40:44.059199Z","shell.execute_reply":"2023-03-17T07:40:44.944479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_augmentation = keras.models.Sequential()\ndata_augmentation.add(keras.layers.RandomRotation(0.1, fill_mode='nearest', input_shape=(28, 28, 1)))\ndata_augmentation.add(keras.layers.RandomZoom((0.15, 0.2), fill_mode='nearest'))\ndata_augmentation.add(keras.layers.RandomTranslation(0.1, 0.1, fill_mode='nearest'))\n\nmodel = keras.models.Sequential()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:40:46.640918Z","iopub.execute_input":"2023-03-17T07:40:46.641287Z","iopub.status.idle":"2023-03-17T07:40:48.141296Z","shell.execute_reply.started":"2023-03-17T07:40:46.641248Z","shell.execute_reply":"2023-03-17T07:40:48.140112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = keras.models.load_model('models/experiment-dropout-0/')\nplot_model(best_model, to_file='model.png', show_shapes=True, show_layer_names=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:40:48.984517Z","iopub.execute_input":"2023-03-17T07:40:48.985033Z","iopub.status.idle":"2023-03-17T07:40:50.511974Z","shell.execute_reply.started":"2023-03-17T07:40:48.984990Z","shell.execute_reply":"2023-03-17T07:40:50.510800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/sign-language-mnist/sign_mnist_test/sign_mnist_test.csv')\nX_test, y_test = test_df.drop('label', axis=1), test_df['label']","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:40:51.116542Z","iopub.execute_input":"2023-03-17T07:40:51.116953Z","iopub.status.idle":"2023-03-17T07:40:51.668025Z","shell.execute_reply.started":"2023-03-17T07:40:51.116913Z","shell.execute_reply":"2023-03-17T07:40:51.666944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = keras.models.load_model('models/experiment-dropout-0/')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:40:52.238325Z","iopub.execute_input":"2023-03-17T07:40:52.238642Z","iopub.status.idle":"2023-03-17T07:40:52.767526Z","shell.execute_reply.started":"2023-03-17T07:40:52.238612Z","shell.execute_reply":"2023-03-17T07:40:52.766486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_model(best_model, X_test, y_test, label_binarizer)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:40:53.808238Z","iopub.execute_input":"2023-03-17T07:40:53.808609Z","iopub.status.idle":"2023-03-17T07:40:55.239079Z","shell.execute_reply.started":"2023-03-17T07:40:53.808566Z","shell.execute_reply":"2023-03-17T07:40:55.237855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/sign-language-mnist/sign_mnist_test/sign_mnist_test.csv')\nX_test, y_test = test_df.drop('label', axis=1), test_df['label']","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:40:55.781355Z","iopub.execute_input":"2023-03-17T07:40:55.782271Z","iopub.status.idle":"2023-03-17T07:40:56.315152Z","shell.execute_reply.started":"2023-03-17T07:40:55.782233Z","shell.execute_reply":"2023-03-17T07:40:56.314001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Applying normalisation which is applied for X_train\nX_test /= 255.0","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:40:56.316568Z","iopub.execute_input":"2023-03-17T07:40:56.317750Z","iopub.status.idle":"2023-03-17T07:40:56.344155Z","shell.execute_reply.started":"2023-03-17T07:40:56.317710Z","shell.execute_reply":"2023-03-17T07:40:56.343110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = keras.models.load_model('models/experiment-dropout-0/')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:40:56.345488Z","iopub.execute_input":"2023-03-17T07:40:56.346112Z","iopub.status.idle":"2023-03-17T07:40:56.871028Z","shell.execute_reply.started":"2023-03-17T07:40:56.346080Z","shell.execute_reply":"2023-03-17T07:40:56.869985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_model(best_model, X_test, y_test, label_binarizer)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T07:40:56.876119Z","iopub.execute_input":"2023-03-17T07:40:56.876423Z","iopub.status.idle":"2023-03-17T07:40:58.458081Z","shell.execute_reply.started":"2023-03-17T07:40:56.876394Z","shell.execute_reply":"2023-03-17T07:40:58.456689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TRANSFORMER BASED APPROACH****","metadata":{}},{"cell_type":"code","source":"\nimport torch\nimport torch.nn.functional as F\nimport torch.nn as nn\n\n#num_landmark = 543\nmax_length = 80\nnum_class = 250\nnum_point = 82 # LIP, LHAND, RHAND\n\ndef pack_seq(\n seq,\n):\n length = [len(s) for s in seq]\n batch_size = len(seq)\n num_landmark=seq[0].shape[1]\n\n x = torch.zeros((batch_size, max(length), num_landmark, 3)).to(seq[0].device)\n x_mask = torch.zeros((batch_size, max(length))).to(seq[0].device)\n for b in range(batch_size):\n L = length[b]\n x[b, :L] = seq[b][:L]\n x_mask[b, L:] = 1\n x_mask = (x_mask>0.5)\n x = x.reshape(batch_size,-1,num_landmark*3)\n return x, x_mask\n\n\nclass FeedForward(nn.Module):\n def __init__(self, embed_dim, hidden_dim):\n super().__init__()\n self.mlp = nn.Sequential(\n nn.Linear(embed_dim, hidden_dim),\n nn.ReLU(inplace=True),\n nn.Linear(hidden_dim, embed_dim),\n )\n def forward(self, x):\n return self.mlp(x)\n\n\n#https://pytorch.org/docs/stable/generated/torch.nn.MultiheadAttention.html\nclass MultiHeadAttention(nn.Module):\n def __init__(self,\n embed_dim,\n num_head,\n batch_first,\n ):\n super().__init__()\n self.mha = nn.MultiheadAttention(\n embed_dim,\n num_heads=num_head,\n bias=True,\n add_bias_kv=False,\n kdim=None,\n vdim=None,\n dropout=0.0,\n batch_first=batch_first,\n )\n\n def forward(self, x, x_mask):\n out, _ = self.mha(x,x,x, key_padding_mask=x_mask)\n return out\n\n\ndef positional_encoding(length, embed_dim):\n dim = embed_dim//2\n\n position = np.arange(length)[:, np.newaxis] # (seq, 1)\n dim = np.arange(dim)[np.newaxis, :]/dim # (1, dim)\n\n angle = 1 / (10000**dim) # (1, dim)\n angle = position * angle # (pos, dim)\n\n pos_embed = np.concatenate(\n [np.sin(angle), np.cos(angle)],\n axis=-1\n )\n pos_embed = torch.from_numpy(pos_embed).float()\n return pos_embed\n\nclass TransformerBlock(nn.Module):\n def __init__(self,\n embed_dim,\n num_head,\n out_dim,\n batch_first=True,\n ):\n super().__init__()\n self.attn = MultiHeadAttention(embed_dim, num_head,batch_first)\n self.ffn = FeedForward(embed_dim, out_dim)\n self.norm1 = nn.LayerNorm(embed_dim)\n self.norm2 = nn.LayerNorm(out_dim)\n\n def forward(self, x, x_mask=None):\n x = x + self.attn((self.norm1(x)), x_mask)\n x = x + self.ffn((self.norm2(x)))\n return x\n\nclass Net(nn.Module):\n\n def __init__(self, num_class=num_class):\n super().__init__()\n self.output_type = ['inference', 'loss']\n\n num_block = 1\n embed_dim = 1024\n num_head = 8\n\n pos_embed = positional_encoding(max_length, embed_dim)\n # self.register_buffer('pos_embed', pos_embed)\n self.pos_embed = nn.Parameter(pos_embed)\n\n self.cls_embed = nn.Parameter(torch.zeros((1, embed_dim)))\n self.x_embed = nn.Sequential(\n nn.Linear(num_point * 3, embed_dim, bias=False),\n )\n\n self.encoder = nn.ModuleList([\n TransformerBlock(\n embed_dim,\n num_head,\n embed_dim,\n ) for i in range(num_block)\n ])\n self.logit = nn.Linear(embed_dim, num_class)\n\n def forward(self, batch):\n length = [len(x) for x in batch['xyz']]\n xyz = batch['xyz']\n\n x, x_mask = pack_seq(xyz)\n B,L,_ = x.shape\n x = self.x_embed(x)\n x = x + self.pos_embed[:L].unsqueeze(0)\n\n x = torch.cat([\n self.cls_embed.unsqueeze(0).repeat(B,1,1),\n x\n ],1)\n x_mask = torch.cat([\n torch.zeros(B,1).to(x_mask),\n x_mask\n ],1)\n\n\n #x = F.dropout(x,p=0.25,training=self.training)\n for block in self.encoder:\n x = block(x,x_mask)\n\n cls = x[:,0]\n cls = F.dropout(cls,p=0.4,training=self.training)\n logit = self.logit(cls)\n\n output = {}\n if 'loss' in self.output_type:\n output['label_loss'] = F.cross_entropy(logit, batch['label'])\n\n if 'inference' in self.output_type:\n output['sign'] = torch.softmax(logit,-1)\n\n return output\n\n\ndef pre_process(xyz):\n xyz = xyz - xyz[~torch.isnan(xyz)].mean(0,keepdims=True) #noramlisation to common mean\n xyz = xyz / xyz[~torch.isnan(xyz)].std(0, keepdims=True)\n \n lip = xyz[:, LIP]\n lhand = xyz[:, LHAND]\n rhand = xyz[:, RHAND]\n xyz = torch.cat([ #(none, 82, 3)\n lip,\n lhand,\n rhand,\n ],1)\n xyz[torch.isnan(xyz)] = 0\n xyz = xyz[:max_length]\n return xyz","metadata":{"execution":{"iopub.status.busy":"2023-03-17T08:23:33.148682Z","iopub.execute_input":"2023-03-17T08:23:33.149234Z","iopub.status.idle":"2023-03-17T08:23:34.000254Z","shell.execute_reply.started":"2023-03-17T08:23:33.149197Z","shell.execute_reply":"2023-03-17T08:23:33.999201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"CONVERTING INTO TF LITE****","metadata":{}},{"cell_type":"code","source":"\n#simplfiy for one video input \nmax_length = 96 #reduce this if gets out of memory error\n\nclass InputNet(nn.Module):\n def __init__(self, ):\n super().__init__()\n self.max_length = max_length \n \n def forward(self, xyz):\n xyz = xyz - xyz[~torch.isnan(xyz)].mean(0,keepdim=True) #noramlisation to common maen\n xyz = xyz / xyz[~torch.isnan(xyz)].std(0, keepdim=True)\n\n LIP = [\n 61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n 291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n 78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n 95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n ]\n #LHAND = np.arange(468, 489).tolist()\n #RHAND = np.arange(522, 543).tolist()\n\n lip = xyz[:, LIP]\n lhand = xyz[:, 468:489]\n rhand = xyz[:, 522:543]\n xyz = torch.cat([ # (none, 82, 3)\n lip,\n lhand,\n rhand,\n ], 1)\n xyz[torch.isnan(xyz)] = 0\n x = xyz[:self.max_length]\n return x\n\n\n#overwrite the model used in training ....\n\n# use fix dimension\nclass MultiHeadAttention(nn.Module):\n def __init__(self,\n embed_dim,\n num_head,\n batch_first,\n ):\n super().__init__()\n self.mha = nn.MultiheadAttention(\n embed_dim,\n num_heads=num_head,\n bias=True,\n add_bias_kv=False,\n kdim=None,\n vdim=None,\n dropout=0.0,\n batch_first=batch_first,\n )\n #https://github.com/pytorch/text/blob/60907bf3394a97eb45056a237ca0d647a6e03216/torchtext/modules/multiheadattention.py#L5\n def forward(self, x):\n # out,_ = self.mha(x,x,x,need_weights=False)\n # out,_ = F.multi_head_attention_forward(\n # x, x, x,\n # self.mha.embed_dim,\n # self.mha.num_heads,\n # self.mha.in_proj_weight,\n # self.mha.in_proj_bias,\n # self.mha.bias_k,\n # self.mha.bias_v,\n # self.mha.add_zero_attn,\n # 0,#self.mha.dropout,\n # self.mha.out_proj.weight,\n # self.mha.out_proj.bias,\n # training=False,\n # key_padding_mask=None,\n # need_weights=False,\n # attn_mask=None,\n # average_attn_weights=False\n # )\n \n #qkv = F.linear(x, self.mha.in_proj_weight, self.mha.in_proj_bias)\n #qkv = qkv.reshape(-1,3,1024)\n #q,k,v = qkv[[0],0], qkv[:,1], qkv[:,2]\n\n q = F.linear(x[:1], self.mha.in_proj_weight[:1024], self.mha.in_proj_bias[:1024]) #since we need only cls\n k = F.linear(x, self.mha.in_proj_weight[1024:2048], self.mha.in_proj_bias[1024:2048])\n v = F.linear(x, self.mha.in_proj_weight[2048:], self.mha.in_proj_bias[2048:]) \n q = q.reshape(-1, 8, 128).permute(1, 0, 2)\n k = k.reshape(-1, 8, 128).permute(1, 2, 0)\n v = v.reshape(-1, 8, 128).permute(1, 0, 2)\n dot = torch.matmul(q, k) * (1/128**0.5) # H L L\n attn = F.softmax(dot, -1) # L L\n out = torch.matmul(attn, v) # L H dim\n out = out.permute(1, 0, 2).reshape(-1, 1024)\n out = F.linear(out, self.mha.out_proj.weight, self.mha.out_proj.bias) \n return out\n\n# remove mask\nclass TransformerBlock(nn.Module):\n def __init__(self,\n embed_dim,\n num_head,\n out_dim,\n batch_first=True,\n ):\n super().__init__()\n self.attn = MultiHeadAttention(embed_dim, num_head,batch_first)\n self.ffn = FeedForward(embed_dim, out_dim)\n self.norm1 = nn.LayerNorm(embed_dim)\n self.norm2 = nn.LayerNorm(out_dim)\n\n def forward(self, x): \n x = x[:1] + self.attn((self.norm1(x)))\n x = x + self.ffn((self.norm2(x)))\n return x\n\nclass SingleNet(nn.Module):\n\n def __init__(self, num_class=num_class):\n super().__init__()\n self.num_block = 1\n self.embed_dim = 1024\n self.num_head = 8\n self.max_length = max_length\n self.num_point = num_point\n\n pos_embed = positional_encoding(max_length, self.embed_dim)\n self.pos_embed = nn.Parameter(pos_embed)\n\n self.cls_embed = nn.Parameter(torch.zeros((1, self.embed_dim)))\n self.x_embed = nn.Sequential(\n nn.Linear(num_point * 3, self.embed_dim, bias=False),\n )\n\n self.encoder = nn.ModuleList([\n TransformerBlock(\n self.embed_dim,\n self.num_head,\n self.embed_dim,\n batch_first=False\n ) for i in range(self.num_block)\n ])\n self.logit = nn.Linear(self.embed_dim, num_class)\n\n def forward(self, xyz):\n L = xyz.shape[0]\n x_embed = self.x_embed(xyz.flatten(1)) \n x = x_embed[:L] + self.pos_embed[:L]\n x = torch.cat([\n self.cls_embed,\n x\n ],0)\n #x = x.unsqueeze(1)\n\n #for block in self.encoder: x = block(x) #remove tflite loop\n x = self.encoder[0](x)\n cls = x[[0]]\n logit = self.logit(cls)\n return logit\n ","metadata":{"execution":{"iopub.status.busy":"2023-03-17T08:23:47.782507Z","iopub.execute_input":"2023-03-17T08:23:47.783048Z","iopub.status.idle":"2023-03-17T08:23:47.807347Z","shell.execute_reply.started":"2023-03-17T08:23:47.783011Z","shell.execute_reply":"2023-03-17T08:23:47.806058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pytorch to onnx to tflite\nif 0:\n \n name='transformer-pool-2b' \n input_onnx_file = f'{fold_dir}/{name}.input.onnx'\n single_onnx_file = f'{fold_dir}/{name}.single.onnx' \n input_tf_file = f'{fold_dir}/input_tf'\n single_tf_file = f'{fold_dir}/single_tf'\n tf_file = f'{fold_dir}/tf'\n tflite_file = f'{fold_dir}/{name}-{max_length}.tflite'\n\n def run_convert_onnx(): \n if 1:\n torch.onnx.export(\n input_net,\n #torch.jit.script(input_net),\n #torch.jit.trace(input_net, torch.zeros(100,num_landmark,3)), # model being run \n torch.zeros((100,num_landmark,3)), # model input (or a tuple for multiple inputs)\n input_onnx_file, # where to save the model (can be a file or file-like object)\n export_params = True, # store the trained parameter weights inside the model file\n opset_version = 12, # the ONNX version to export the model to\n do_constant_folding=True, # whether to execute constant folding for optimization \n input_names = ['inputs'], # the model's input names\n output_names = ['outputs'], # the model's output names\n dynamic_axes={\n 'inputs': {0: 'length'},\n #'output': {0: 'length'},\n },\n #verbose = True,\n )\n torch.onnx.export(\n single_net, \n #torch.jit.script(single_net),\n #torch.jit.trace(single_net, torch.zeros(max_length,82,3)), \n\n torch.zeros((max_length,82,3)), \n single_onnx_file, \n export_params = True, \n opset_version = 12, \n do_constant_folding=True, \n input_names = ['inputs'], \n output_names = ['outputs'], \n dynamic_axes={\n 'inputs': {0: 'length'},\n },\n #verbose = True,\n )\n print('torch.onnx.export() passed !!')\n\n if 1:\n for f in [input_onnx_file, single_onnx_file]:\n if f is None: continue\n model = onnx.load(f)\n onnx.checker.check_model(model)\n model_simple, check = onnxsim.simplify(model)\n onnx.save(model_simple, f)\n print('onnx simplify() passed !!')\n\n\n def run_convert_tflite():\n if 1:\n tf_rep = prepare(onnx.load(input_onnx_file))\n tf_rep.export_graph(input_tf_file) \n tf_rep = prepare(onnx.load(single_onnx_file))\n tf_rep.export_graph(single_tf_file) \n print('tf_rep.export_graph() passed !!')\n\n if 1:\n class TFModel(tf.Module):\n def __init__(self):\n super(TFModel, self).__init__()\n self.input = tf.saved_model.load(input_tf_file)\n self.single = tf.saved_model.load(single_tf_file)\n self.input.trainable = False\n self.single.trainable = False\n\n @tf.function(input_signature=[\n tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='inputs')\n ])\n def call(self, input):\n y = {}\n x = self.input(**{'inputs': input})['outputs']\n y['outputs'] = self.single(**{'inputs': x})['outputs'][0]\n return y\n\n tfmodel = TFModel()\n tf.saved_model.save(tfmodel, tf_file, signatures={'serving_default': tfmodel.call})\n print('tf.saved_model() passed !!')\n\n if 1:\n converter = tf.lite.TFLiteConverter.from_saved_model(tf_file)\n # converter.target_spec.supported_ops = [\n # tf.lite.OpsSet.TFLITE_BUILTINS, # enable TensorFlow Lite ops.\n # tf.lite.OpsSet.SELECT_TF_OPS # enable TensorFlow ops.\n # ]\n # converter.optimizations = [tf.lite.Optimize.DEFAULT]\n #converter.allow_custom_ops = True\n #converter.experimental_new_converter = True \n tf_lite_model = converter.convert()\n with open(tflite_file, 'wb') as f:\n f.write(tf_lite_model)\n print('tflite convert() passed !!')\n \n run_convert_onnx()\n run_convert_tflite()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T08:24:02.250193Z","iopub.execute_input":"2023-03-17T08:24:02.250558Z","iopub.status.idle":"2023-03-17T08:24:02.265475Z","shell.execute_reply.started":"2023-03-17T08:24:02.250524Z","shell.execute_reply":"2023-03-17T08:24:02.264438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#submission\n#tflite_file = '/kaggle/input/asl-demo/transformer-pool-2b.tflite' #max_length =180\n#tflite_file = '/kaggle/input/asl-demo/transformer-pool-2b-96.tflite' #max_length =96 \n#tflite_file = '/kaggle/input/asl-demo/transformer-pool-2c-512-80-fixed-int8.tflite'\n\n#tflite_file = '/kaggle/input/asl-demo/transfomer-60-256-lip-hand-my-part-3a-int8.tflite'\n#tflite_file = '/kaggle/input/asl-demo/run10-fold1-swa-transfomer-60-512-lip-hand-crop-center-00a-int8.tflite'\n#tflite_file = '/kaggle/input/asl-demo/run15.tflite'\ntflite_file = '/kaggle/input/asl-demo/run20-aug3-xyz2.tflite'\n\n\n\n\nmode = 'submit' #debug #submit\n\n\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport shutil\nfrom datetime import datetime\nfrom timeit import default_timer as timer\n\n\nif mode in ['debug']: \n try:\n import tflite_runtime\n except:\n !pip install tflite-runtime\n\n import tflite_runtime.interpreter as tflite \n import tflite_runtime\n print(tflite_runtime.__version__)\n #'2.11.0'\n \n #import tensorflow as tf\n #print(tf.__version__)\n # 2.11.0\n\nprint('import ok')\n'''\nYour model must also require less than 40 MB in memory and \nperform inference with less than 100 milliseconds of latency per video. \nExpect to see approximately 40,000 videos in the test set. \nWe allow an additional 10 minute buffer for loading the data and miscellaneous overhead.\n\n'''\ndef time_to_str(t, mode='min'):\n if mode=='min':\n t = int(t)/60\n hr = t//60\n min = t%60\n return '%2d hr %02d min'%(hr,min)\n\n elif mode=='sec':\n t = int(t)\n min = t//60\n sec = t%60\n return '%2d min %02d sec'%(min,sec)\n\n else:\n raise NotImplementedError\n\n \nROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n data_columns = ['x', 'y', 'z']\n data = pd.read_parquet(pq_path, columns=data_columns)\n n_frames = int(len(data) / ROWS_PER_FRAME)\n data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n return data.astype(np.float32)\n\nif mode in ['debug']: \n \n interpreter = tflite.Interpreter(tflite_file)\n prediction_fn = interpreter.get_signature_runner('serving_default')\n\n valid_df = pd.read_csv('/kaggle/input/asl-demo/train_prepared.csv') \n valid_df = valid_df[valid_df.fold==2].reset_index(drop=True)\n valid_df = valid_df[:4_000]\n valid_num = len(valid_df)\n valid = {\n 'sign':[],\n }\n\n start_timer = timer()\n for t, d in valid_df.iterrows():\n\n pq_file = f'/kaggle/input/asl-signs/{d.path}'\n #print(pq_file)\n xyz = load_relevant_data_subset(pq_file)\n\n output = prediction_fn(inputs=xyz)\n p = output['outputs'].reshape(-1)\n\n valid['sign'].append(p)\n\n #---\n if t%100==0:\n time_taken = timer() - start_timer\n print('\\r %8d / %d %s'%(t,valid_num,time_to_str(time_taken,'sec')),end='',flush=True)\n\n print('\\n')\n\n\n truth = valid_df.label.values\n sign = np.stack(valid['sign'])\n predict = np.argsort(-sign, -1)\n correct = predict==truth.reshape(valid_num,1)\n topk = correct.cumsum(-1).mean(0)[:5]\n\n\n print(f'time_taken = {time_to_str(time_taken,\"sec\")}')\n print(f'time_taken for LB = {time_taken*1000/valid_num:05f} msec\\n')\n for i in range(5):\n print(f'topk[{i}] = {topk[i]}') \n print('----- end -----\\n')\n\n\n\n\nshutil.copyfile(tflite_file, 'model.tflite') \n!zip submission.zip 'model.tflite'\n!ls\n\nprint('tflite_file:', tflite_file)\nprint(f'submit ok')\n\n# '''\n\n# 2.11.0\n# import ok\n\n# ######################################################\n# embed_dim = 1024\n# max_length=180\n\n# 7900 / 8000 7 min 49 sec\n# time_taken = 7 min 49 sec\n# time_taken for LB = 58.693773 msec\n\n# topk[0] = 0.588625\n# topk[1] = 0.702\n# topk[2] = 0.755375\n# topk[3] = 0.785375\n# topk[4] = 0.804125\n\n \n# ----- end -----\n\n# updating: model.tflite (deflated 8%)\n# __notebook_source__.ipynb model.tflite submission.zip\n# submit ok\n\n\n\n# ######################################################\n# embed_dim = 1024\n# max_length=96\n\n# import ok\n# 7900 / 8000 6 min 23 sec\n\n# time_taken = 6 min 23 sec\n# time_taken for LB = 47.972998 msec\n\n# topk[0] = 0.58425\n# topk[1] = 0.696375\n# topk[2] = 0.748125\n# topk[3] = 0.77825\n# topk[4] = 0.797125\n\n\n\n# ######################################################\n# embed_dim = 512\n# max_length = 80\n\n# 2.11.0\n# import ok\n# 7900 / 8000 5 min 44 sec\n\n# time_taken = 5 min 44 sec\n# time_taken for LB = 43.067440 msec\n\n# topk[0] = 0.57525\n# topk[1] = 0.690625\n# topk[2] = 0.74\n# topk[3] = 0.77175\n# topk[4] = 0.79375\n# ----- end -----\n\n# transformer-pool-2c-512-80-cut.tflite\n\n# time_taken = 4 min 53 sec\n# time_taken for LB = 36.710013 msec\n\n# topk[0] = 0.574875\n# topk[1] = 0.69025\n# topk[2] = 0.73975\n# topk[3] = 0.7715\n# topk[4] = 0.79375\n\n# ######################################################\n# ldd --version | head -n1\n\n# 2.11.0\n# import ok\n# 17600 / 17670 9 min 39 sec\n\n# time_taken = 9 min 39 sec\n# time_taken for LB = 32.815303 msec\n\n# topk[0] = 0.5780418788907753\n# topk[1] = 0.6922467458970005\n# topk[2] = 0.7432937181663837\n# topk[3] = 0.7735144312393888\n# topk[4] = 0.7942275042444822\n# ----- end -----\n\n# updating: model.tflite (deflated 8%)\n# __notebook_source__.ipynb model.tflite submission.zip\n# submit ok\n\n# ---\n# int8\n\n# 17600 / 17670 11 min 34 sec\n\n# time_taken = 11 min 34 sec\n# time_taken for LB = 39.286630 msec\n\n# topk[0] = 0.5782116581777024\n# topk[1] = 0.6921335597057159\n# topk[2] = 0.7434069043576683\n# topk[3] = 0.7740237691001698\n# topk[4] = 0.7953027730616865\n# '''\n\n# import ok\n# 17600 / 17670 9 min 48 sec\n\n# time_taken = 9 min 48 sec\n# time_taken for LB = 33.307542 msec\n\n# topk[0] = 0.5782116581777024\n# topk[1] = 0.6921335597057159\n# topk[2] = 0.7434634974533108\n# topk[3] = 0.7740237691001698\n# topk[4] = 0.7951895868704019\n# ----- end -----\n\n# adding: model.tflite (deflated 15%)\n# __notebook_source__.ipynb model.tflite submission.zip\n# tflite_file: /kaggle/input/asl-demo/transformer-pool-2c-512-80-fixed-int8.tflite\n# submit ok","metadata":{"execution":{"iopub.status.busy":"2023-03-17T08:24:17.929449Z","iopub.execute_input":"2023-03-17T08:24:17.929894Z","iopub.status.idle":"2023-03-17T08:24:20.314363Z","shell.execute_reply.started":"2023-03-17T08:24:17.929859Z","shell.execute_reply":"2023-03-17T08:24:20.312730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"VISUALIZATION****","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.animation import FuncAnimation\nfrom IPython.display import HTML\n\ndir = '/kaggle/input/asl-signs'\ntrain = pd.read_csv(f'{dir}/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T08:29:28.702000Z","iopub.execute_input":"2023-03-17T08:29:28.702554Z","iopub.status.idle":"2023-03-17T08:29:28.816676Z","shell.execute_reply.started":"2023-03-17T08:29:28.702512Z","shell.execute_reply":"2023-03-17T08:29:28.815595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_to_sign = 'train_landmark_files/16069/1011655866.parquet'\nsign = pd.read_parquet(f'{dir}/{path_to_sign}')\nsign.y = sign.y * -1","metadata":{"execution":{"iopub.status.busy":"2023-03-17T08:29:31.232253Z","iopub.execute_input":"2023-03-17T08:29:31.233215Z","iopub.status.idle":"2023-03-17T08:29:31.259746Z","shell.execute_reply.started":"2023-03-17T08:29:31.233163Z","shell.execute_reply":"2023-03-17T08:29:31.258813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hand_points(hand):\n x = [[hand.iloc[0].x, hand.iloc[1].x, hand.iloc[2].x, hand.iloc[3].x, hand.iloc[4].x], # Thumb\n [hand.iloc[5].x, hand.iloc[6].x, hand.iloc[7].x, hand.iloc[8].x], # Index\n [hand.iloc[9].x, hand.iloc[10].x, hand.iloc[11].x, hand.iloc[12].x], \n [hand.iloc[13].x, hand.iloc[14].x, hand.iloc[15].x, hand.iloc[16].x], \n [hand.iloc[17].x, hand.iloc[18].x, hand.iloc[19].x, hand.iloc[20].x], \n [hand.iloc[0].x, hand.iloc[5].x, hand.iloc[9].x, hand.iloc[13].x, hand.iloc[17].x, hand.iloc[0].x]]\n\n y = [[hand.iloc[0].y, hand.iloc[1].y, hand.iloc[2].y, hand.iloc[3].y, hand.iloc[4].y], #Thumb\n [hand.iloc[5].y, hand.iloc[6].y, hand.iloc[7].y, hand.iloc[8].y], # Index\n [hand.iloc[9].y, hand.iloc[10].y, hand.iloc[11].y, hand.iloc[12].y], \n [hand.iloc[13].y, hand.iloc[14].y, hand.iloc[15].y, hand.iloc[16].y], \n [hand.iloc[17].y, hand.iloc[18].y, hand.iloc[19].y, hand.iloc[20].y], \n [hand.iloc[0].y, hand.iloc[5].y, hand.iloc[9].y, hand.iloc[13].y, hand.iloc[17].y, hand.iloc[0].y]] \n return x, y\n\ndef get_pose_points(pose):\n x = [[pose.iloc[8].x, pose.iloc[6].x, pose.iloc[5].x, pose.iloc[4].x, pose.iloc[0].x, pose.iloc[1].x, pose.iloc[2].x, pose.iloc[3].x, pose.iloc[7].x], \n [pose.iloc[10].x, pose.iloc[9].x], \n [pose.iloc[22].x, pose.iloc[16].x, pose.iloc[20].x, pose.iloc[18].x, pose.iloc[16].x, pose.iloc[14].x, pose.iloc[12].x, \n pose.iloc[11].x, pose.iloc[13].x, pose.iloc[15].x, pose.iloc[17].x, pose.iloc[19].x, pose.iloc[15].x, pose.iloc[21].x], \n [pose.iloc[12].x, pose.iloc[24].x, pose.iloc[26].x, pose.iloc[28].x, pose.iloc[30].x, pose.iloc[32].x, pose.iloc[28].x], \n [pose.iloc[11].x, pose.iloc[23].x, pose.iloc[25].x, pose.iloc[27].x, pose.iloc[29].x, pose.iloc[31].x, pose.iloc[27].x], \n [pose.iloc[24].x, pose.iloc[23].x]\n ]\n\n y = [[pose.iloc[8].y, pose.iloc[6].y, pose.iloc[5].y, pose.iloc[4].y, pose.iloc[0].y, pose.iloc[1].y, pose.iloc[2].y, pose.iloc[3].y, pose.iloc[7].y], \n [pose.iloc[10].y, pose.iloc[9].y], \n [pose.iloc[22].y, pose.iloc[16].y, pose.iloc[20].y, pose.iloc[18].y, pose.iloc[16].y, pose.iloc[14].y, pose.iloc[12].y, \n pose.iloc[11].y, pose.iloc[13].y, pose.iloc[15].y, pose.iloc[17].y, pose.iloc[19].y, pose.iloc[15].y, pose.iloc[21].y], \n [pose.iloc[12].y, pose.iloc[24].y, pose.iloc[26].y, pose.iloc[28].y, pose.iloc[30].y, pose.iloc[32].y, pose.iloc[28].y], \n [pose.iloc[11].y, pose.iloc[23].y, pose.iloc[25].y, pose.iloc[27].y, pose.iloc[29].y, pose.iloc[31].y, pose.iloc[27].y], \n [pose.iloc[24].y, pose.iloc[23].y]\n ]\n return x, y","metadata":{"execution":{"iopub.status.busy":"2023-03-17T08:29:34.796805Z","iopub.execute_input":"2023-03-17T08:29:34.797634Z","iopub.status.idle":"2023-03-17T08:29:34.821269Z","shell.execute_reply.started":"2023-03-17T08:29:34.797593Z","shell.execute_reply":"2023-03-17T08:29:34.820018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def animation_frame(f):\n frame = sign[sign.frame==f]\n left = frame[frame.type=='left_hand']\n right = frame[frame.type=='right_hand']\n pose = frame[frame.type=='pose']\n face = frame[frame.type=='face'][['x', 'y']].values\n lx, ly = get_hand_points(left)\n rx, ry = get_hand_points(right)\n px, py = get_pose_points(pose)\n ax.clear()\n ax.plot(face[:,0], face[:,1], '.')\n for i in range(len(lx)):\n ax.plot(lx[i], ly[i])\n for i in range(len(rx)):\n ax.plot(rx[i], ry[i])\n for i in range(len(px)):\n ax.plot(px[i], py[i])\n plt.xlim(xmin, xmax)\n plt.ylim(ymin, ymax)\n \nprint(f\"The sign being shown here is: {train[train.path==f'{path_to_sign}'].sign.values[0]}\")\n\n## These values set the limits on the graph to stabilize the video\nxmin = sign.x.min() - 0.2\nxmax = sign.x.max() + 0.2\nymin = sign.y.min() - 0.2\nymax = sign.y.max() + 0.2\n\nfig, ax = plt.subplots()\nl, = ax.plot([], [])\nanimation = FuncAnimation(fig, func=animation_frame, frames=sign.frame.unique())\n\nHTML(animation.to_html5_video())","metadata":{"execution":{"iopub.status.busy":"2023-03-17T08:29:39.133130Z","iopub.execute_input":"2023-03-17T08:29:39.133784Z","iopub.status.idle":"2023-03-17T08:29:42.307648Z","shell.execute_reply.started":"2023-03-17T08:29:39.133723Z","shell.execute_reply":"2023-03-17T08:29:42.306427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign = sign[sign.type=='left_hand'].dropna()\ndef animation_frame(f):\n frame = sign[sign.frame==f]\n left = frame[frame.type=='left_hand']\n lx, ly = get_hand_points(left)\n ax.clear()\n for i in range(len(lx)):\n ax.plot(lx[i], ly[i])\n plt.xlim(xmin, xmax)\n plt.ylim(ymin, ymax)\n\n \nprint(f\"The sign being shown here is: {train[train.path==f'{path_to_sign}'].sign.values[0]}\")\n\n## These values set the limits on the graph to stabilize the video\nxmin = sign.x.min() - 0.2\nxmax = sign.x.max() + 0.2\nymin = sign.y.min() - 0.2\nymax = sign.y.max() + 0.2\n\nfig, ax = plt.subplots()\nl, = ax.plot([], [])\nanimation = FuncAnimation(fig, func=animation_frame, frames=sign.frame.unique())\n\nHTML(animation.to_html5_video())","metadata":{"execution":{"iopub.status.busy":"2023-03-17T08:30:00.117391Z","iopub.execute_input":"2023-03-17T08:30:00.117795Z","iopub.status.idle":"2023-03-17T08:30:01.908923Z","shell.execute_reply.started":"2023-03-17T08:30:00.117741Z","shell.execute_reply":"2023-03-17T08:30:01.907840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('The End')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T08:30:36.041481Z","iopub.execute_input":"2023-03-17T08:30:36.042038Z","iopub.status.idle":"2023-03-17T08:30:36.048568Z","shell.execute_reply.started":"2023-03-17T08:30:36.041997Z","shell.execute_reply":"2023-03-17T08:30:36.047389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}