{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-04T02:46:52.140113Z","iopub.execute_input":"2023-05-04T02:46:52.140755Z","iopub.status.idle":"2023-05-04T02:46:52.145334Z","shell.execute_reply.started":"2023-05-04T02:46:52.140723Z","shell.execute_reply":"2023-05-04T02:46:52.144591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n!pip install -q --upgrade tensorflow-io\ntry:\n    import mediapipe as mp\nexcept:\n    !pip install -q mediapipe\n    import mediapipe as mp\n\n# Competition Specific Imports (You'll see why we need these later)\n# mediapipe above\n\n# Machine Learning and Data Science Imports (basics)\nimport tensorflow as tf\nimport tensorflow_io as tfio\nimport pandas as pd\nimport numpy as np\nimport sklearn\n# Built-In Imports (mostly don't worry about these)\nfrom kaggle_datasets import KaggleDatasets\nfrom collections import Counter\nfrom datetime import datetime\nfrom zipfile import ZipFile\nfrom glob import glob\nimport Levenshtein\nimport warnings\nimport requests\nimport hashlib\nimport imageio\nimport IPython\nimport sklearn\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport json\nimport math\nimport time\nimport gzip\nimport ast\nimport sys\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports (overkill)\nfrom matplotlib.animation import FuncAnimation\nfrom matplotlib.colors import ListedColormap\nfrom matplotlib.patches import Rectangle\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nfrom IPython.display import HTML\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm; tqdm.pandas();\nimport plotly.express as px\nimport tifffile as tif\nimport seaborn as sns\nfrom PIL import Image, ImageEnhance; Image.MAX_IMAGE_PIXELS = 5_000_000_000;\nimport matplotlib; print(f\"\\t\\t– MATPLOTLIB VERSION: {matplotlib.__version__}\");\nfrom matplotlib import animation, rc; rc('animation', html='jshtml')\nimport plotly\nimport PIL\nimport cv2\n\nimport plotly.io as pio\nprint(pio.renderers)\n\ndef seed_it_all(seed=7):\n    \"\"\" Attempt to be Reproducible \"\"\"\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\nseed_it_all()\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-04T02:46:52.147999Z","iopub.execute_input":"2023-05-04T02:46:52.148772Z","iopub.status.idle":"2023-05-04T02:47:31.166434Z","shell.execute_reply.started":"2023-05-04T02:46:52.14873Z","shell.execute_reply":"2023-05-04T02:47:31.165194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimport json\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport multiprocessing as mp","metadata":{"execution":{"iopub.status.busy":"2023-05-04T02:47:34.91526Z","iopub.execute_input":"2023-05-04T02:47:34.915688Z","iopub.status.idle":"2023-05-04T02:47:37.783989Z","shell.execute_reply.started":"2023-05-04T02:47:34.915654Z","shell.execute_reply":"2023-05-04T02:47:37.782814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LANDMARK_FILES_DIR = \"/kaggle/input/asl-signs/train_landmark_files\"\nTRAIN_FILE = \"/kaggle/input/asl-signs/train.csv\"\nlabel_map = json.load(open(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\", \"r\"))","metadata":{"execution":{"iopub.status.busy":"2023-05-04T02:48:21.107768Z","iopub.execute_input":"2023-05-04T02:48:21.108147Z","iopub.status.idle":"2023-05-04T02:48:21.118424Z","shell.execute_reply.started":"2023-05-04T02:48:21.10812Z","shell.execute_reply":"2023-05-04T02:48:21.117302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeatureGen(nn.Module):\n    def __init__(self):\n        super(FeatureGen, self).__init__()\n        pass\n    \n    def forward(self, x):\n        face_x = x[:,:468,:].contiguous().view(-1, 468*3)\n        lefth_x = x[:,468:489,:].contiguous().view(-1, 21*3)\n        pose_x = x[:,489:522,:].contiguous().view(-1, 33*3)\n        righth_x = x[:,522:,:].contiguous().view(-1, 21*3)\n        \n        lefth_x = lefth_x[~torch.any(torch.isnan(lefth_x), dim=1),:]\n        righth_x = righth_x[~torch.any(torch.isnan(righth_x), dim=1),:]\n        \n        x1m = torch.mean(face_x, 0)\n        x2m = torch.mean(lefth_x, 0)\n        x3m = torch.mean(pose_x, 0)\n        x4m = torch.mean(righth_x, 0)\n        \n        x1s = torch.std(face_x, 0)\n        x2s = torch.std(lefth_x, 0)\n        x3s = torch.std(pose_x, 0)\n        x4s = torch.std(righth_x, 0)\n        \n        xfeat = torch.cat([x1m,x2m,x3m,x4m, x1s,x2s,x3s,x4s], axis=0)\n        xfeat = torch.where(torch.isnan(xfeat), torch.tensor(0.0, dtype=torch.float32), xfeat)\n        \n        return xfeat\n    \nfeature_converter = FeatureGen()","metadata":{"execution":{"iopub.status.busy":"2023-05-04T02:48:21.467391Z","iopub.execute_input":"2023-05-04T02:48:21.467748Z","iopub.status.idle":"2023-05-04T02:48:21.478875Z","shell.execute_reply.started":"2023-05-04T02:48:21.467722Z","shell.execute_reply":"2023-05-04T02:48:21.47727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T02:48:21.702451Z","iopub.execute_input":"2023-05-04T02:48:21.702821Z","iopub.status.idle":"2023-05-04T02:48:21.708497Z","shell.execute_reply.started":"2023-05-04T02:48:21.702795Z","shell.execute_reply":"2023-05-04T02:48:21.707334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_row(row):\n    x = load_relevant_data_subset(os.path.join(\"/kaggle/input/asl-signs\", row[1].path))\n    x = feature_converter(torch.tensor(x)).cpu().numpy()\n    return x, row[1].label\n\ndef convert_and_save_data():\n    df = pd.read_csv(TRAIN_FILE)\n    df['label'] = df['sign'].map(label_map)\n    npdata = np.zeros((df.shape[0], 3258))\n    nplabels = np.zeros(df.shape[0])\n    with mp.Pool() as pool:\n        results = pool.imap(convert_row, df.iterrows(), chunksize=250)\n        for i, (x,y) in tqdm(enumerate(results), total=df.shape[0]):\n            npdata[i,:] = x\n            nplabels[i] = y\n    \n    np.save(\"feature_data.npy\", npdata)\n    np.save(\"feature_labels.npy\", nplabels)\n        \nconvert_and_save_data()","metadata":{"execution":{"iopub.status.busy":"2023-05-04T02:48:22.090409Z","iopub.execute_input":"2023-05-04T02:48:22.090785Z","iopub.status.idle":"2023-05-04T02:59:15.821872Z","shell.execute_reply.started":"2023-05-04T02:48:22.090756Z","shell.execute_reply":"2023-05-04T02:59:15.82037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x    = np.load(\"feature_data.npy\").astype(np.float32)\ntrain_y    = np.load(\"feature_labels.npy\").astype(np.uint8)\nBATCH_SIZE = 64\n\nN_TOTAL = train_x.shape[0]\nVAL_PCT = 0.1\nN_VAL   = int(N_TOTAL*VAL_PCT)\nN_TRAIN = N_TOTAL-N_VAL\n\nrandom_idxs = random.sample(range(N_TOTAL), N_TOTAL)\ntrain_idxs, val_idxs = np.array(random_idxs[:N_TRAIN]), np.array(random_idxs[N_TRAIN:])\n\nval_x, val_y = train_x[val_idxs], train_y[val_idxs]\ntrain_x, train_y = train_x[train_idxs], train_y[train_idxs]","metadata":{"execution":{"iopub.status.busy":"2023-05-04T03:01:06.343044Z","iopub.execute_input":"2023-05-04T03:01:06.343464Z","iopub.status.idle":"2023-05-04T03:01:09.803105Z","shell.execute_reply.started":"2023-05-04T03:01:06.343434Z","shell.execute_reply":"2023-05-04T03:01:09.801861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fc_block(inputs, output_channels, dropout=0.2):\n    x = tf.keras.layers.Dense(output_channels)(inputs)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation(\"gelu\")(x)\n    x = tf.keras.layers.Dropout(dropout)(x)\n    return x\n\ndef get_model(n_labels=250, init_fc=512, n_blocks=2, _dropout_1=0.2, _dropout_2=0.6, flat_frame_len=3258):\n    _inputs = tf.keras.layers.Input(shape=(flat_frame_len,))\n    x = _inputs\n    \n    # Define layers\n    for i in range(n_blocks):\n        x = fc_block(\n            x, output_channels=init_fc//(2**i), \n            dropout=_dropout_1 if (1+i)!=n_blocks else _dropout_2\n        )\n    \n    # Define output layer\n    _outputs = tf.keras.layers.Dense(n_labels, activation=\"softmax\")(x)\n    \n    # Build the model\n    model = tf.keras.models.Model(inputs=_inputs, outputs=_outputs)\n    return model\n\nmodel = get_model()\nmodel.compile(tf.keras.optimizers.Adam(0.000333), \"sparse_categorical_crossentropy\", metrics=\"acc\")\nmodel.summary()\n\ntf.keras.utils.plot_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T03:01:22.632395Z","iopub.execute_input":"2023-05-04T03:01:22.637575Z","iopub.status.idle":"2023-05-04T03:01:24.355159Z","shell.execute_reply.started":"2023-05-04T03:01:22.637297Z","shell.execute_reply":"2023-05-04T03:01:24.350198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir models\ncb_list = [\n    tf.keras.callbacks.EarlyStopping(patience=5, restore_best_weights=True, verbose=1),\n    tf.keras.callbacks.ReduceLROnPlateau(patience=2, factor=0.8, verbose=1)\n]\nhistory = model.fit(train_x, train_y, validation_data=(val_x, val_y), epochs=100, callbacks=cb_list, batch_size=BATCH_SIZE)\nmodel.save(\"./models/asl_model\")","metadata":{"execution":{"iopub.status.busy":"2023-05-04T03:01:29.888057Z","iopub.execute_input":"2023-05-04T03:01:29.888496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.mean(history.history['acc']))\nprint(np.mean(history.history['val_acc']))\n\nprint(np.mean(history.history['loss']))\nprint(np.mean(history.history['val_loss']))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(val_x, val_y)\nfor x,y in zip(val_x[:10], val_y[:10]):\n    print(f\"PRED: {decoder(np.argmax(model.predict(tf.expand_dims(x, axis=0), verbose=0), axis=-1)[0]):<20} – GT: {decoder(y)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}