This notebook uses the dataset that includes lips created from this notebook created by MED ALI BOUCHHIOUA. Give his work an upvote if you end up using this dataset! \n
\n\n
\n 🛑 WARNING:
THIS IS A WORK IN PROGRESS \n
\n\n\n
\n 👏 IF YOU FORK THIS OR FIND THIS HELPFUL 👏
PLEASE UPVOTE!
This was a lot of work for me and while it may seem silly, it makes me feel appreciated when others like my work. 😅\n
\n\n---\n","metadata":{}},{"cell_type":"code","source":"def flatten_l_o_l(nested_list):\n \"\"\"Flatten a list of lists into a single list.\n\n Args:\n nested_list (list): \n – A list of lists (or iterables) to be flattened.\n\n Returns:\n list: A flattened list containing all items from the input list of lists.\n \"\"\"\n return [item for sublist in nested_list for item in sublist]\n\n\ndef print_ln(symbol=\"-\", line_len=110, newline_before=False, newline_after=False):\n \"\"\"Print a horizontal line of a specified length and symbol.\n\n Args:\n symbol (str, optional): \n – The symbol to use for the horizontal line\n line_len (int, optional): \n – The length of the horizontal line in characters\n newline_before (bool, optional): \n – Whether to print a newline character before the line\n newline_after (bool, optional): \n – Whether to print a newline character after the line\n \"\"\"\n if newline_before: print();\n print(symbol * line_len)\n if newline_after: print();\n \n \ndef read_json_file(file_path):\n \"\"\"Read a JSON file and parse it into a Python object.\n\n Args:\n file_path (str): The path to the JSON file to read.\n\n Returns:\n dict: A dictionary object representing the JSON data.\n \n Raises:\n FileNotFoundError: If the specified file path does not exist.\n ValueError: If the specified file path does not contain valid JSON data.\n \"\"\"\n try:\n # Open the file and load the JSON data into a Python object\n with open(file_path, 'r') as file:\n json_data = json.load(file)\n return json_data\n except FileNotFoundError:\n # Raise an error if the file path does not exist\n raise FileNotFoundError(f\"File not found: {file_path}\")\n except ValueError:\n # Raise an error if the file does not contain valid JSON data\n raise ValueError(f\"Invalid JSON data in file: {file_path}\")\n \ndef get_sign_df(pq_path, invert_y=True):\n sign_df = pd.read_parquet(pq_path)\n \n # y value is inverted (Thanks @danielpeshkov)\n if invert_y: sign_df[\"y\"] *= -1 \n \n return sign_df\n\nROWS_PER_FRAME = 543 # number of landmarks per frame\ndef load_relevant_data_subset(pq_path):\n data_columns = ['x', 'y', 'z']\n data = pd.read_parquet(pq_path, columns=data_columns)\n n_frames = int(len(data) / ROWS_PER_FRAME)\n data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-03-13T20:58:22.721672Z","iopub.execute_input":"2023-03-13T20:58:22.722265Z","iopub.status.idle":"2023-03-13T20:58:22.733672Z","shell.execute_reply.started":"2023-03-13T20:58:22.722224Z","shell.execute_reply":"2023-03-13T20:58:22.732564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" \n\n
2.2 LOAD DATA
\n\n---","metadata":{}},{"cell_type":"code","source":"# Define the path to the root data directory\nDATA_DIR = \"/kaggle/input/asl-signs\"\nEXTEND_TRAIN_DIR = \"/kaggle/input/gislr-extended-train-dataframe\" \nNP_FILE_DIR = \"/kaggle/input/isolated-sign-language-aggregation-preparation\"\n\nprint(\"\\n... BASIC DATA SETUP STARTING ...\\n\")\nprint(\"\\n\\n... LOAD TRAIN DATAFRAME FROM CSV FILE ...\\n\")\n\nLOAD_EXTENDED = True\nif LOAD_EXTENDED and os.path.isfile(os.path.join(EXTEND_TRAIN_DIR, \"extended_train.csv\")):\n train_df = pd.read_csv(os.path.join(EXTEND_TRAIN_DIR, \"extended_train.csv\"))\nelse:\n train_df = pd.read_csv(os.path.join(DATA_DIR, \"train.csv\"))\n train_df[\"path\"] = DATA_DIR+\"/\"+train_df[\"path\"]\ndisplay(train_df)\n\nprint(\"\\n\\n... LOAD SIGN TO PREDICTION INDEX MAP FROM JSON FILE ...\\n\")\ns2p_map = {k.lower():v for k,v in read_json_file(os.path.join(DATA_DIR, \"sign_to_prediction_index_map.json\")).items()}\np2s_map = {v:k for k,v in read_json_file(os.path.join(DATA_DIR, \"sign_to_prediction_index_map.json\")).items()}\nencoder = lambda x: s2p_map.get(x.lower())\ndecoder = lambda x: p2s_map.get(x)\n\nDEMO_ROW = 283\nprint(f\"\\n\\n... DEMO SIGN/EVENT DATAFRAME FOR ROW {DEMO_ROW} - SIGN={train_df.iloc[DEMO_ROW]['sign']} ...\\n\")\ndemo_sign_df = get_sign_df(train_df.iloc[DEMO_ROW][\"path\"])\ndisplay(demo_sign_df)\n\n# Landmark IDs start at 0 for each respective type and count up\nFRAME_TYPE_ORDER_DETAIL = demo_sign_df.groupby(\"frame\")[\"type\"].apply(list).values[0]\nFRAME_TYPE_ORDER = sorted(set(FRAME_TYPE_ORDER_DETAIL))\nprint(FRAME_TYPE_ORDER)\n\n# https://www.kaggle.com/competitions/asl-signs/discussion/391812#2168354\nlipsUpperOuter = [61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291]\nlipsLowerOuter = [146, 91, 181, 84, 17, 314, 405, 321, 375, 291]\nlipsUpperInner = [78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308]\nlipsLowerInner = [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308]\nlips = lipsUpperOuter + lipsLowerOuter + lipsUpperInner + lipsLowerInner\nFRAME_TYPE_IDX_MAP = {\n \"lips\" : np.array(lips),\n \"left_hand\" : np.arange(468, 489),\n \"pose\" : np.arange(489, 522),\n \"right_hand\" : np.arange(522, 543),\n}\n\nfor k,v in FRAME_TYPE_IDX_MAP.items():\n print(k, len(v))","metadata":{"execution":{"iopub.status.busy":"2023-03-13T20:58:22.735338Z","iopub.execute_input":"2023-03-13T20:58:22.736242Z","iopub.status.idle":"2023-03-13T20:58:23.102611Z","shell.execute_reply.started":"2023-03-13T20:58:22.736202Z","shell.execute_reply":"2023-03-13T20:58:23.101428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" \n\n
2.3 PREPARE KFOLD AND NORMALIZE NPY FILES
\n\n---\n\nWe use stratified group k-fold even though it reduces accuracy significantly (0.65 --> 0.45) during training as it will prevent overfitting and leakage between our train and validation dataset.\n* We will use 7 folds across 21 parcipants (3 signers in the val).\n* We will also ensure that we spread out handedness as much as possible.\n\n\nNote I decided to add in hengck's idea of xyz normalization. His code is here:\n* https://www.kaggle.com/competitions/asl-signs/discussion/391265\n\n```python\ndef pre_process(xyz):\n lip = xyz[:, LIP]\n lhand = xyz[:, LHAND]\n rhand = xyz[:, RHAND]\n xyz = torch.cat([ #(none, 82, 3)\n lip,\n lhand,\n rhand,\n ],1)\n xyz = xyz - xyz[~torch.isnan(xyz)].mean(0,keepdims=True) #noramlisation to common maen\n xyz = xyz / xyz[~torch.isnan(xyz)].std(0, keepdims=True)\n xyz[torch.isnan(xyz)] = 0\n xyz = xyz[:max_length]\n return xyz\n```\n","metadata":{}},{"cell_type":"code","source":"all_x = np.load(os.path.join(NP_FILE_DIR, \"feature_data.npy\")).astype(np.float32)\nall_y = np.load(os.path.join(NP_FILE_DIR, \"feature_labels.npy\")).astype(np.uint8)\n\n# add nan back in not to screw up means/std\nall_x = np.where(all_x==0.0, np.nan, all_x)\n\n# Get mean and std ignoring nans\nall_mean = np.nanmean(all_x, keepdims=True, axis=0)\nall_std = np.nanstd(all_x, keepdims=True, axis=0)\n\n# Standardize around 0\nall_x = (all_x-all_mean)/all_std\n# Technically I don't need to do the were because np.nan \n# subtracting or dividing anything still results in np.nan\n# - all_x = np.where(np.isnan(all_x), all_x, all_x-all_mean)\n# - all_x = np.where(np.isnan(all_x), all_x, all_x/all_std)\n\n# Back to 0s\nall_x = np.nan_to_num(all_x)\n\n# There are 21 participants so we use 7 folds\n# --> i.e. 3 participants in val every time\nN_PARTICIPANTS = train_df.participant_id.nunique()\n\n# Thanks Robert Hatch! \n# --> https://www.kaggle.com/competitions/asl-signs/discussion/392335\n#\n# We are including 29302 in RH signer\nRH_SIGNERS = [26734, 28656, 25571, 62590, 29302, \n 49445, 53618, 18796, 4718, 2044, \n 37779, 30680]\n\n# We are including 37055 in LH Signer\nLH_SIGNERS = [16069, 32319, 36257, 22343, 27610, 61333, 34503, 55372, 37055]\n\n# Since the sign label itself if relatively \n# There are 21 participants so we use 7 folds\n# --> i.e. 3 participants in val every time\n\nK_FOLDS = N_PARTICIPANTS\ndef get_folds(df, k_folds, force_lh=True, lh_signers=LH_SIGNERS[:-1]):\n while True:\n sgkf = StratifiedGroupKFold(n_splits=K_FOLDS, shuffle=True)\n _fold_ds_idx_map = {\n i:{\"train\":t_idxs, \"val\":v_idxs} \\\n for i, (t_idxs, v_idxs) in enumerate(sgkf.split(df.index, df.sign, df.participant_id))\n }\n \n # Ensure only one left hander in every val group\n if force_lh:\n if all([len(set(df.iloc[_idxs['val']][\"participant_id\"].unique()).intersection(set(lh_signers)))>=1 for _idxs in _fold_ds_idx_map.values()]):\n return _fold_ds_idx_map\n else:\n print(\".\", end=\"\")\n else:\n return _fold_ds_idx_map\n \nfold_ds_idx_map = get_folds(train_df, K_FOLDS, force_lh=False)\nif K_FOLDS==N_PARTICIPANTS:\n fold_2_val_pid_map = {k:train_df.iloc[v[\"val\"]].participant_id.values[0] for k,v in fold_ds_idx_map.items()}\n print(fold_2_val_pid_map)\nprint(\" APPROPRIATE KFOLD SPLIT FOUND!\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-03-13T20:58:23.105820Z","iopub.execute_input":"2023-03-13T20:58:23.106265Z","iopub.status.idle":"2023-03-13T20:58:24.805648Z","shell.execute_reply.started":"2023-03-13T20:58:23.106227Z","shell.execute_reply":"2023-03-13T20:58:24.804468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" \n\n\n\n
\n\n---\n","metadata":{}},{"cell_type":"code","source":"def plot_training_data(histories, fold_2_pid_map=None, _cmap=\"tab20\"):\n \"\"\"\n Plots the accuracy and loss for ten folds of training data on the same figure using Matplotlib.\n\n Args:\n - histories: A list of ten history objects returned by the `fit` method of a Keras model.\n \"\"\"\n cmap = plt.get_cmap(_cmap)\n clrs = [cmap(x) for x in np.linspace(0, 1, len(histories[:20]))]\n if len(clrs)<21: clrs=[(0.0, 0.0, 0.502, 1.0),]+clrs\n n_plots = len(histories[0].history)\n fig, axs = plt.subplots(n_plots, 1, figsize=(20, 20*(n_plots//2)))\n min_vals, max_vals = [1e5,]*len(histories[0].history), [0.0,]*len(histories[0].history)\n # plot accuracy and loss for each fold\n for i in range(len(histories)):\n for j, (_mkey, _mval) in enumerate(histories[i].history.items()):\n if fold_2_pid_map is None:\n axs[j].plot(_mval, label=f'Fold {i+1}', c=clrs[i])\n else:\n axs[j].plot(_mval, label=f'Fold {i+1} - Participant {fold_2_pid_map.get(i)}', c=clrs[i])\n if max(_mval)>max_vals[j]: max_vals[j]=max(_mval)\n if min(_mval)\n\n
4.2 CONFUSION MATRIX -- CLASSWISE METRICS
\n\n---\n","metadata":{}},{"cell_type":"code","source":"def evaluate_model(model, data_x, data_y, decoder):\n \"\"\"\n Evaluates the given model on the given data and prints predictions and ground truth for the last few samples.\n \n Args:\n - model: The trained model to evaluate.\n - data_x: The input data to evaluate the model on.\n - data_y: The target data to evaluate the model on.\n - decoder: A function to decode the model's output into readable text.\n \"\"\"\n # Evaluate the model and print predictions and ground truth for the last few samples\n model_loss, model_accuracy = model.evaluate(data_x, data_y)\n print(f\"Model Loss: {model_loss:.4f}, Model Accuracy: {model_accuracy:.4f}\\n\")\n \n last_samples_x, last_samples_y = data_x[-10:], data_y[-10:]\n print(\"Predictions and Ground Truth for Last Few Samples in Training Data:\\n\")\n for x, y in zip(last_samples_x, last_samples_y):\n pred = np.argmax(model.predict(tf.expand_dims(x, axis=0), verbose=0), axis=-1)[0]\n print(f\"PRED: {decoder(pred):<20} – GT: {decoder(y)}\")\n\n first_samples_x, first_samples_y = data_x[:10], data_y[:10]\n print(\"\\nPredictions and Ground Truth for First Few Samples in Validation Data:\\n\")\n for x, y in zip(first_samples_x, first_samples_y):\n pred = np.argmax(model.predict(tf.expand_dims(x, axis=0), verbose=0), axis=-1)[0]\n print(f\"PRED: {decoder(pred):<20} – GT: {decoder(y)}\")\n \n\ndef compute_evaluation_metrics(model, data_x, data_y, decoder, plt_cm=False, verbose=True):\n \"\"\"\n Computes the evaluation metrics for the given model on the given data and prints classwise confusion matrix.\n \n Args:\n - model: The trained model to evaluate.\n - data_x: The input data to evaluate the model on.\n - data_y: The target data to evaluate the model on.\n - decoder: A function to decode the model's output into readable text.\n \"\"\"\n # Compute the predicted classes and confusion matrix\n batch_size = 1024\n y_pred = model.predict(data_x, batch_size=1024, verbose=verbose)\n y_pred_classes = tf.cast(np.argmax(y_pred, axis=1), tf.uint8)\n confusion_mtx = tf.math.confusion_matrix(data_y, y_pred_classes)\n \n #if plt_cm:\n # print(\"\\n... PLOTTING CONFUSION MATRIX ...\")\n # sns.heatmap(confusion_mtx.numpy(), annot=True, cmap='Blues', xticklabels=list(s2p_map.keys()), yticklabels=list(s2p_map.keys()))\n # plt.show()\n \n # Compute the evaluation metrics by class\n num_classes = confusion_mtx.shape[0]\n classwise_performance = {}\n for i in range(num_classes):\n tp = confusion_mtx[i,i]\n fp = tf.reduce_sum(confusion_mtx[:,i]) - tp\n fn = tf.reduce_sum(confusion_mtx[i,:]) - tp\n tn = tf.reduce_sum(confusion_mtx[i]) - (tp - fp - fn)\n\n classwise_performance[i] = dict(\n accuracy=(tp + tn) / (tp + fp + tn + fn),\n precision = tp / (tp + fp),\n recall = tp / (tp + fn),\n )\n classwise_performance[i]['f1_score'] = 2 * (classwise_performance[i]['precision'] * classwise_performance[i]['recall']) / (classwise_performance[i]['precision'] + classwise_performance[i]['recall'])\n classwise_performance[i] = {k:v.numpy() for k,v in classwise_performance[i].items()}\n\n # Sort the classwise performance by f1_score and print the results\n if verbose:\n classwise_performance = dict(sorted(classwise_performance.items(), key=lambda x: x[1][\"f1_score\"], reverse=True))\n print(\"\\n\\n... OOF CLASSWISE CONFUSION MATRIX... \\n\")\n for i, perf in classwise_performance.items():\n print(f\"Class {i:<3} ({decoder(i):^13}) --> Accuracy: {perf['accuracy']:.2f}, Precision: {perf['precision']:.2f}, Recall: {perf['recall']:.2f}, F1 Score: {perf['f1_score']:.2f}\")\n return classwise_performance\n\nMODEL_PATHS = sorted(glob(os.path.join(MODEL_DIR, \"*\")), key=lambda x: int(x.rsplit(\"_\")[-3]))\nmodel_perf_dfs = []\nfor i, mpath in enumerate(MODEL_PATHS):\n c_perf = compute_evaluation_metrics(\n tf.keras.models.load_model(MODEL_PATHS[i], compile=False), \n all_x[fold_ds_idx_map[int(MODEL_PATHS[i].rsplit(\"_\")[-3])-1][\"val\"]], \n all_y[fold_ds_idx_map[int(MODEL_PATHS[i].rsplit(\"_\")[-3])-1][\"val\"]], \n decoder=decoder, verbose=False\n )\n model_perf_dfs.append(pd.DataFrame(dict(sorted(c_perf.items(), key=lambda x:x[0]))).T)\n\nMODEL_PATHS = sorted(MODEL_PATHS, reverse=True, key=lambda x: int(x.rsplit(\"__\", 1)[-1]))\nfor i, mdf in enumerate(model_perf_dfs): mdf.columns = [f\"model_{i}_oof_\"+_c for _c in mdf.columns]\noof_perf_df = pd.concat(model_perf_dfs, axis=1).reset_index().rename(columns={\"index\":\"class_idx\"})\noof_perf_df.insert(0, \"class_str\", oof_perf_df['class_idx'].apply(decoder))\ndisplay(oof_perf_df)","metadata":{"execution":{"iopub.status.busy":"2023-03-13T22:12:37.134266Z","iopub.execute_input":"2023-03-13T22:12:37.134798Z","iopub.status.idle":"2023-03-13T22:13:34.040779Z","shell.execute_reply.started":"2023-03-13T22:12:37.134758Z","shell.execute_reply":"2023-03-13T22:13:34.039669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" \n\n
4.3 PLOT OOF PERFORMANCE (BOTTOM AND TOP 5)
\n\n---\n","metadata":{}},{"cell_type":"code","source":"### PLOT ALL CLASSES:\n# fig = px.bar(oof_perf_df, [_c for _c in oof_perf_df if \"f1_score\" in _c], \"class_str\", barmode=\"group\", orientation=\"h\", height=10000)\n# fig.show()\n\n## PLOT WORST 5 CLASSES:\nworst_5 = [150, 12, 41, 224, 97]\nfig = px.bar(oof_perf_df.iloc[worst_5], \"class_str\", [_c for _c in oof_perf_df if \"f1_score\" in _c], barmode=\"group\", title=\"OOF PERFORMANCE ON WORST 5 CLASSES\", labels={\"value\":\"F1 Score\", \"variable\":\"Fold\"})\nfig.update_layout(showlegend=False)\nfig.show()\n\n## PLOT BEST 5 CLASSES:\nbest_5 = [2, 162, 89, 59, 182]\nfig = px.bar(oof_perf_df.iloc[best_5], \"class_str\", [_c for _c in oof_perf_df if \"f1_score\" in _c], barmode=\"group\", title=\"OOF PERFORMANCE ON BEST 5 CLASSES\", labels={\"value\":\"F1 Score\", \"variable\":\"Fold\"})\nfig.update_layout(showlegend=False)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-13T22:14:17.229617Z","iopub.execute_input":"2023-03-13T22:14:17.230348Z","iopub.status.idle":"2023-03-13T22:14:17.544116Z","shell.execute_reply.started":"2023-03-13T22:14:17.230312Z","shell.execute_reply":"2023-03-13T22:14:17.542953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" \n\nNOTE: It doesn't appear to be the lefties oof that is doing poorly... SEE BELOW","metadata":{}},{"cell_type":"code","source":"for signer, oof_acc in {fold_2_val_pid_map[int(x.rsplit(\"_\")[-3])-1]:float(x.rsplit(\"__\" , 1)[-1])/100000 for x in MODEL_PATHS}.items():\n print(f\"SINGER: {signer:>5} ({'LH' if signer in LH_SIGNERS else 'RH'}) ––> {oof_acc}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-13T22:14:18.234396Z","iopub.execute_input":"2023-03-13T22:14:18.235137Z","iopub.status.idle":"2023-03-13T22:14:18.242655Z","shell.execute_reply.started":"2023-03-13T22:14:18.235099Z","shell.execute_reply":"2023-03-13T22:14:18.241479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" \n\n\n\n
\n\n---\n","metadata":{}},{"cell_type":"code","source":"def dumb_tf_mean(x, axis=None):\n return tf.math.reduce_mean(x, axis=axis)\n\ndef dumb_tf_std(x, axis=None):\n x = tf.experimental.numpy.var(x, axis=axis, dtype=tf.float32, ddof=1)\n return tf.experimental.numpy.sqrt(x)\n\nclass PrepInputs(tf.keras.layers.Layer):\n def __init__(self, lh_idx_range=(468, 489), pose_idx_range=(489, 522), rh_idx_range=(522, 543), distribution_mean=all_mean, distribution_std=all_std):\n super(PrepInputs, self).__init__()\n self.lips = tf.constant([61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291, 78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308])\n self.idx_ranges = [lh_idx_range, pose_idx_range, rh_idx_range]\n self.flat_feat_lens = [2*self.lips.shape[0],]+[2*(_range[1]-_range[0]) for _range in self.idx_ranges]\n self.distribution_mean = tf.constant(distribution_mean, dtype=tf.float32)\n self.distribution_std = tf.constant(distribution_std, dtype=tf.float32)\n \n def call(self, x_in):\n \n # Split the single vector into 4\n xs = [tf.gather(x_in[..., :2], self.lips, axis=1),]+[x_in[:, _range[0]:_range[1], :2] for _range in self.idx_ranges]\n \n # Reshape based on specific number of keypoints\n xs = [tf.reshape(_x, (-1, flat_feat_len)) for _x, flat_feat_len in zip(xs, self.flat_feat_lens)]\n \n # Drop empty rows - Empty rows are present in \n # --> face, lh, rh\n # --> so we don't have to for face\n xs = [tf.boolean_mask(_x, tf.reduce_all(tf.logical_not(tf.math.is_nan(_x)), axis=1), axis=0) for _x in xs]\n \n # Get means and stds\n x_means = [dumb_tf_mean(_x, axis=0) for _x in xs]\n x_stds = [dumb_tf_std(_x, axis=0) for _x in xs]\n \n x_out = tf.concat([*x_means, *x_stds], axis=0)\n x_out = tf.expand_dims(tf.where(tf.math.is_nan(x_out), tf.zeros_like(x_out), x_out), axis=0)\n x_out = self.standardize_tensor(x_out)\n return x_out\n \n def standardize_tensor(self, tensor):\n return tf.where(tensor!=0, (tensor-self.distribution_mean)/self.distribution_std, tf.zeros_like(tensor))\n \np_demo = PrepInputs()(load_relevant_data_subset(train_df.path[0]))\nprint(p_demo.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-13T22:14:25.817311Z","iopub.execute_input":"2023-03-13T22:14:25.818308Z","iopub.status.idle":"2023-03-13T22:14:25.932080Z","shell.execute_reply.started":"2023-03-13T22:14:25.818267Z","shell.execute_reply":"2023-03-13T22:14:25.930865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" \n\n
5.2 ENSEMBLE OUR FOLDS TOGETHER
\n\n---\n","metadata":{}},{"cell_type":"code","source":"class ISLRModel(tf.keras.Model):\n \"\"\"\n TensorFlow Lite model that takes input tensors and applies:\n – a preprocessing model\n – the ISLR model \n \"\"\"\n\n def __init__(self, islr_fold_models):\n \"\"\"\n Initializes the TFLiteModel with the specified preprocessing model and ISLR model.\n \"\"\"\n super(ISLRModel, self).__init__()\n\n # Load the feature generation and main models\n self.islr_fold_models = list(islr_fold_models.values())\n self.model_weights = tf.repeat(tf.expand_dims(tf.constant([float(k)/100_000. for k in islr_fold_models.keys()], dtype=tf.float32), axis=-1), 250, axis=-1)\n \n def __call__(self, inputs, training=None):\n \"\"\"\n Applies the feature generation model and main model to the input tensors.\n\n Args:\n inputs: Input tensor with shape [batch_size, 543, 3].\n\n Returns:\n A dictionary with a single key 'outputs' and corresponding output tensor.\n \"\"\"\n batch_size = tf.shape(inputs)[0]\n outputs = tf.concat([_model(inputs, training=training) for _model in self.islr_fold_models], axis=0)\n outputs = tf.reduce_mean(outputs, axis=0, keepdims=True)\n\n ### Compute the weighted sum and the sum of the weights and compute the weighted mean\n #outputs = tf.stack([_model(inputs, training=training) for _model in self.islr_fold_models], axis=1)\n #outputs = tf.reduce_sum(tf.multiply(outputs, tf.repeat(tf.expand_dims(self.model_weights, axis=0), batch_size, axis=0)), axis=1)\n #outputs = tf.divide(outputs, tf.reduce_sum(tf.repeat(tf.expand_dims(self.model_weights, axis=0), batch_size, axis=0), axis=1))\n \n # Return a dictionary with the output tensor\n return outputs\n\nclass TFLiteModel(tf.Module):\n \"\"\"\n TensorFlow Lite model that takes input tensors and applies:\n – a preprocessing model\n – the ISLR model \n \"\"\"\n\n def __init__(self, islr_fold_models, islr_fold_pp_fn):\n \"\"\"\n Initializes the TFLiteModel with the specified preprocessing model and ISLR model.\n \"\"\"\n super(TFLiteModel, self).__init__()\n\n # Load the feature generation and main models\n self.prep_inputs = islr_fold_pp_fn()\n self.islr_fold_models = list(islr_fold_models.values())\n self.model_weights = tf.repeat(tf.expand_dims(tf.constant([float(k)/100_000. for k in islr_fold_models.keys()], dtype=tf.float32), axis=-1), 250, axis=-1)\n \n @tf.function(input_signature=[tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='inputs')])\n def __call__(self, inputs):\n \"\"\"\n Applies the feature generation model and main model to the input tensors.\n\n Args:\n inputs: Input tensor with shape [batch_size, 543, 3].\n\n Returns:\n A dictionary with a single key 'outputs' and corresponding output tensor.\n \"\"\"\n x = self.prep_inputs(tf.cast(inputs, dtype=tf.float32))\n outputs = tf.concat([_model(x) for _model in self.islr_fold_models], axis=0)\n \n # Compute the weighted sum and the sum of the weights and compute the weighted mean\n #outputs = tf.reduce_sum(tf.multiply(outputs, self.model_weights), axis=0)\n #outputs = tf.divide(outputs, tf.reduce_sum(self.model_weights, axis=0))\n outputs = tf.reduce_mean(outputs, axis=0, keepdims=True)\n \n # Return a dictionary with the output tensor\n return {'outputs': outputs}\n \nONLY_KFOLD=False\n\nif ONLY_KFOLD:\n ISLR_FOLD_MODELS = {_path.rsplit(\"__\", 1)[-1]:tf.keras.models.load_model(_path, compile=False) for _path in MODEL_PATHS}\n tflite_keras_model = TFLiteModel(ISLR_FOLD_MODELS, PrepInputs)\n out = tflite_keras_model(load_relevant_data_subset(train_df.path[0]))[\"outputs\"]\n np.argmax(out)","metadata":{"execution":{"iopub.status.busy":"2023-03-13T22:16:01.652243Z","iopub.execute_input":"2023-03-13T22:16:01.652812Z","iopub.status.idle":"2023-03-13T22:16:01.669423Z","shell.execute_reply.started":"2023-03-13T22:16:01.652769Z","shell.execute_reply":"2023-03-13T22:16:01.668053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" \n\n
6.1 CREATE THE REQUIRED PREPROCESSING IF DIFFERENT THAN OURS
\n\n---\n","metadata":{}},{"cell_type":"code","source":"def tf_nan_mean(x, axis=0):\n return tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis) / tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis)\n\ndef tf_nan_std(x, axis=0):\n d = x - tf_nan_mean(x, axis=axis)\n return tf.math.sqrt(tf_nan_mean(d * d, axis=axis))\n\ndef flatten_means_and_stds(x, axis=0):\n # Get means and stds\n x_mean = tf_nan_mean(x, axis=0)\n x_std = tf_nan_std(x, axis=0)\n\n x_out = tf.concat([x_mean, x_std], axis=0)\n x_out = tf.reshape(x_out, (1, INPUT_SHAPE[1]*2))\n x_out = tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n return x_out\n\nclass FeatureGen_1(tf.keras.layers.Layer):\n def __init__(self):\n super(FeatureGen_1, self).__init__()\n \n def call(self, x_in):\n x = tf.gather(x_in, point_landmarks, axis=1)\n\n x_padded = x\n for i in range(SEGMENTS):\n p0 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) != 0) , 1, 0)\n p1 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) == 0) , 1, 0)\n paddings = [[p0, p1], [0, 0], [0, 0]]\n x_padded = tf.pad(x_padded, paddings, mode=\"SYMMETRIC\")\n x_list = tf.split(x_padded, SEGMENTS)\n x_list = [flatten_means_and_stds(_x, axis=0) for _x in x_list]\n\n x_list.append(flatten_means_and_stds(x, axis=0))\n \n ## Resize only dimension 0. Resize can't handle nan, so replace nan with that dimension's avg value to reduce impact.\n x = tf.image.resize(tf.where(tf.math.is_finite(x), x, tf_nan_mean(x, axis=0)), [NUM_FRAMES, LANDMARKS])\n x = tf.reshape(x, (1, INPUT_SHAPE[0]*INPUT_SHAPE[1]))\n x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n x_list.append(x)\n x = tf.concat(x_list, axis=1)\n return x\n \nR__DROP_Z = False\n\nR__NUM_FRAMES = 15\nR__SEGMENTS = 3\n\nR__LEFT_HAND_OFFSET = 468\nR__POSE_OFFSET = R__LEFT_HAND_OFFSET+21\nR__RIGHT_HAND_OFFSET = R__POSE_OFFSET+33\n\n## average over the entire face, and the entire 'pose'\nR__averaging_sets = [[0, 468], [R__POSE_OFFSET, 33]]\n\nR__lip_landmarks = [61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n 291,146, 91,181, 84, 17, 314, 405, 321, 375, \n 78, 191, 80, 81, 82, 13, 312, 311, 310, 415, \n 95, 88, 178, 87, 14,317, 402, 318, 324, 308]\nR__left_hand_landmarks = list(range(R__LEFT_HAND_OFFSET, R__LEFT_HAND_OFFSET+21))\nR__right_hand_landmarks = list(range(R__RIGHT_HAND_OFFSET, R__RIGHT_HAND_OFFSET+21))\n\nR__point_landmarks = [item for sublist in [R__lip_landmarks, R__left_hand_landmarks, R__right_hand_landmarks] for item in sublist]\n\nR__LANDMARKS = len(R__point_landmarks) + len(R__averaging_sets)\n\nif R__DROP_Z:\n R__INPUT_SHAPE = (R__NUM_FRAMES,R__LANDMARKS*2)\nelse:\n R__INPUT_SHAPE = (R__NUM_FRAMES,R__LANDMARKS*3)\n\nR__FLAT_INPUT_SHAPE = (R__INPUT_SHAPE[0] + 2 * (R__SEGMENTS + 1)) * R__INPUT_SHAPE[1]\n\ndef R__flatten_means_and_stds(x, axis=0):\n # Get means and stds\n x_mean = tf_nan_mean(x, axis=0)\n x_std = tf_nan_std(x, axis=0)\n\n x_out = tf.concat([x_mean, x_std], axis=0)\n x_out = tf.reshape(x_out, (1, R__INPUT_SHAPE[1]*2))\n x_out = tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n return x_out\n \nclass RobertFeatureGen(tf.keras.layers.Layer):\n def __init__(self):\n super(RobertFeatureGen, self).__init__()\n \n def call(self, x_in):\n if R__DROP_Z:\n x_in = x_in[:, :, 0:2]\n x_list = [tf.expand_dims(tf_nan_mean(x_in[:, av_set[0]:av_set[0]+av_set[1], :], axis=1), axis=1) for av_set in R__averaging_sets]\n x_list.append(tf.gather(x_in, R__point_landmarks, axis=1))\n x = tf.concat(x_list, 1)\n\n x_padded = x\n for i in range(R__SEGMENTS):\n p0 = tf.where( ((tf.shape(x_padded)[0] % R__SEGMENTS) > 0) & ((i % 2) != 0) , 1, 0)\n p1 = tf.where( ((tf.shape(x_padded)[0] % R__SEGMENTS) > 0) & ((i % 2) == 0) , 1, 0)\n paddings = [[p0, p1], [0, 0], [0, 0]]\n x_padded = tf.pad(x_padded, paddings, mode=\"SYMMETRIC\")\n x_list = tf.split(x_padded, R__SEGMENTS)\n x_list = [R__flatten_means_and_stds(_x, axis=0) for _x in x_list]\n\n x_list.append(R__flatten_means_and_stds(x, axis=0))\n \n ## Resize only dimension 0. Resize can't handle nan, so replace nan with that dimension's avg value to reduce impact.\n x = tf.image.resize(tf.where(tf.math.is_finite(x), x, tf_nan_mean(x, axis=0)), [R__NUM_FRAMES, R__LANDMARKS])\n x = tf.reshape(x, (1, R__INPUT_SHAPE[0]*R__INPUT_SHAPE[1]))\n x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n x_list.append(x)\n x = tf.concat(x_list, axis=1)\n return x","metadata":{"execution":{"iopub.status.busy":"2023-03-13T22:44:23.544132Z","iopub.execute_input":"2023-03-13T22:44:23.544514Z","iopub.status.idle":"2023-03-13T22:44:23.573371Z","shell.execute_reply.started":"2023-03-13T22:44:23.544477Z","shell.execute_reply":"2023-03-13T22:44:23.572135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" \n\n
6.2 ENSEMBLE OUR FOLDS WITH OTHER MODEL
\n\n---\n","metadata":{}},{"cell_type":"code","source":"class TFLiteModel(tf.Module):\n \"\"\"\n TensorFlow Lite model that takes input tensors and applies:\n – a preprocessing model\n – the ISLR model \n \"\"\"\n\n def __init__(self, islr_fold_models, islr_fold_pp_fn, gwg_model, gwg_pp_fn, robert_model, robert_pp_fn):\n \"\"\"\n Initializes the TFLiteModel with the specified preprocessing model and ISLR model.\n \"\"\"\n super(TFLiteModel, self).__init__()\n\n # Load the feature generation and main models\n self.prep_inputs_1 = islr_fold_pp_fn()\n self.prep_inputs_2 = gwg_pp_fn()\n self.prep_inputs_3 = robert_pp_fn()\n self.models_1 = list(islr_fold_models.values())\n self.model_2 = gwg_model\n self.model_3 = robert_model\n \n @tf.function(input_signature=[tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='inputs')])\n def __call__(self, inputs):\n \"\"\"\n Applies the feature generation model and main model to the input tensors.\n\n Args:\n inputs: Input tensor with shape [batch_size, 543, 3].\n\n Returns:\n A dictionary with a single key 'outputs' and corresponding output tensor.\n \"\"\"\n x1 = self.prep_inputs_1(tf.cast(inputs, dtype=tf.float32))\n x2 = self.prep_inputs_2(tf.cast(inputs, dtype=tf.float32))\n x3 = self.prep_inputs_3(tf.cast(inputs, dtype=tf.float32))\n \n outputs_1 = tf.concat([_model(x1) for _model in self.models_1], axis=0)\n outputs_2 = self.model_2(x2) \n outputs_3 = self.model_3(x3) \n \n # 2x weighting higher score via repeat 14\n outputs = tf.reduce_mean(tf.concat([\n outputs_1, \n tf.repeat(outputs_2, 11, axis=0),\n tf.repeat(outputs_3, 9, axis=0),\n ], axis=0), axis=0, keepdims=True)\n \n # Return a dictionary with the output tensor\n return {'outputs': outputs}\n\nR__MODEL_PATH = \"/kaggle/input/ensemble-dataset/ensemble_basic/robert/models/asl_model\"\nN_TOP_MODELS=7\nISLR_FOLD_MODELS = {_path.rsplit(\"__\", 1)[-1]:tf.keras.models.load_model(_path, compile=False) for _path in MODEL_PATHS[:N_TOP_MODELS]}\ntflite_keras_model = TFLiteModel(\n ISLR_FOLD_MODELS, PrepInputs, \n gwg_model, FeatureGen_1,\n tf.keras.models.load_model(R__MODEL_PATH, compile=False), RobertFeatureGen\n)\nout = tflite_keras_model(load_relevant_data_subset(train_df.path[0]))[\"outputs\"]\nnp.argmax(out)","metadata":{"execution":{"iopub.status.busy":"2023-03-13T22:51:43.357663Z","iopub.execute_input":"2023-03-13T22:51:43.358597Z","iopub.status.idle":"2023-03-13T22:51:56.828912Z","shell.execute_reply.started":"2023-03-13T22:51:43.358555Z","shell.execute_reply":"2023-03-13T22:51:56.827725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" \n\n
6.3 GENERATE THE TFLITE SUBMISSION
\n\n---\n\nSee these threads for details on int8 storage optimization in TFLite:\n* https://www.kaggle.com/competitions/asl-signs/discussion/394371\n * Dynamic range quantization (https://www.tensorflow.org/lite/performance/post_training_quantization) is the best solution I found. You add one line of code: `converter.optimizations = [tf.lite.Optimize.DEFAULT]` and all your model will be reduced several times (up to 5). But remember that the execution time remains, this method does not help here, I saw a comment that it even increases the execution time, but it all depends on the model. Regarding model accuracy, I see that this method slightly degrades LB(<0.01). You can see an example in my public notebook (https://www.kaggle.com/code/aikhmelnytskyy/gislr-tf-on-the-shoulders-ensamble-v2-0-67). Just remove `converter.optimizations = [tf.lite.Optimize.DEFAULT]`\nand see the difference in the size of the `submission.zip`\n* https://www.kaggle.com/competitions/asl-signs/discussion/391265#2180165\n * \"one (or two) layer transformer is sufficient. for one fold, memory is only 2mb ( **`tf.lite.Optimize.DEFAULT`, i.e. int8 in storage**) and 25min for all 40_000 test videos (or 30 msec per video). Future because of missing frame, long term attention has its advantage\"","metadata":{}},{"cell_type":"code","source":"# Helps reduce overall size but can decrease performance\nDO_OPTIMIZATION = True\n\n# Let's remove our previous TFLITE_PATH model and submission\nif ONLY_KFOLD:\n !rm -rf {TFLITE_PATH}\n !rm -rf ./submission.zip\n\nkeras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\nkeras_model_converter.optimizations = [tf.lite.Optimize.DEFAULT]\ntflite_model = keras_model_converter.convert()\n\nTFLITE_PATH = '/kaggle/working/models/model.tflite'\nwith open(TFLITE_PATH, 'wb') as f:\n f.write(tflite_model)\n!zip submission.zip {TFLITE_PATH}\n\ninterpreter = tflite.Interpreter(TFLITE_PATH)\nfound_signatures = list(interpreter.get_signature_list().keys())\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=load_relevant_data_subset(train_df.path[0]))\nsign = np.argmax(output[\"outputs\"])\n\nprint(\"PRED : \", decoder[sign])\nprint(\"GT : \", train_df.sign[0])","metadata":{"execution":{"iopub.status.busy":"2023-03-13T22:51:56.830902Z","iopub.execute_input":"2023-03-13T22:51:56.831197Z","iopub.status.idle":"2023-03-13T22:53:00.136642Z","shell.execute_reply.started":"2023-03-13T22:51:56.831169Z","shell.execute_reply":"2023-03-13T22:53:00.135399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" \n\n\n\n
\n\nTBD","metadata":{}},{"cell_type":"code","source":"def compute_evaluation_metrics_v2(model, x_paths, data_y):\n \"\"\"\n Computes the evaluation metrics for the given model on the given data and prints classwise confusion matrix.\n \n Args:\n - model: The trained model to evaluate.\n - x_paths: The input data to evaluate the model on.\n - data_y: The target data to evaluate the model on.\n \"\"\"\n # Compute the predicted classes and confusion matrix\n batch_size = 1024\n y_pred = [model(load_relevant_data_subset(x_path))[\"outputs\"] for x_path in tqdm(x_paths)]\n y_pred_classes = tf.cast(np.argmax(y_pred, axis=0), tf.uint8)\n confusion_mtx = tf.math.confusion_matrix(data_y, y_pred_classes)\n \n # Compute the evaluation metrics by class\n num_classes = confusion_mtx.shape[0]\n classwise_performance = {}\n for i in range(num_classes):\n tp = confusion_mtx[i,i]\n fp = tf.reduce_sum(confusion_mtx[:,i]) - tp\n fn = tf.reduce_sum(confusion_mtx[i,:]) - tp\n tn = tf.reduce_sum(confusion_mtx[i]) - (tp - fp - fn)\n\n classwise_performance[i] = dict(\n accuracy=(tp + tn) / (tp + fp + tn + fn),\n precision = tp / (tp + fp),\n recall = tp / (tp + fn),\n )\n classwise_performance[i]['f1_score'] = 2 * (classwise_performance[i]['precision'] * classwise_performance[i]['recall']) / (classwise_performance[i]['precision'] + classwise_performance[i]['recall'])\n classwise_performance[i] = {k:v.numpy() for k,v in classwise_performance[i].items()}\n\n return classwise_performance","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# small_subset_indices = train_df.iloc[fold_ds_idx_map[0][\"val\"]].sample(1000)\n# x_paths = small_subset_indices.path.to_list()\n# labels = [encoder(x) for x in small_subset_indices.sign.to_list()]","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_perf_dfs.append(pd.DataFrame(dict(sorted(c_perf.items(), key=lambda x:x[0]))).T)\n \n# for i, mdf in enumerate(model_perf_dfs): mdf.columns = [f\"model_{i}_oof_\"+_c for _c in mdf.columns]\n# oof_perf_df = pd.concat(model_perf_dfs, axis=1).reset_index().rename(columns={\"index\":\"class_idx\"})\n# oof_perf_df.insert(0, \"class_str\", oof_perf_df['class_idx'].apply(decoder))\n# display(oof_perf_df)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]}]}
![\"Google](\"https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png\")
![\"asl](\"https://cdn-cbkob.nitrocdn.com/TiGMibPGMREAJjbFYNLfxxdkUjUGroSw/assets/images/optimized/rev-22fc791/wp-content/uploads/2022/12/sign-language.jpg\")