{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:04:32.602240Z","iopub.execute_input":"2023-04-13T02:04:32.602613Z","iopub.status.idle":"2023-04-13T02:04:33.616441Z","shell.execute_reply.started":"2023-04-13T02:04:32.602577Z","shell.execute_reply":"2023-04-13T02:04:33.615250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

0    REFERENCES

","metadata":{}},{"cell_type":"markdown","source":"Processed Data: [GISLR Feature Data: On the Shoulders](https://www.kaggle.com/code/roberthatch/gislr-feature-data-on-the-shoulders)
\nFeatureGEN class: [GISLR Feature Data: On the Shoulders](https://www.kaggle.com/code/roberthatch/gislr-feature-data-on-the-shoulders)
\nChapter html format: [🤟 GISLR 🤟 - 📚Learn – 🔭EDA – 🤖Baseline\n](https://www.kaggle.com/code/dschettler8845/gislr-learn-eda-baseline)
\n\n\nIt's my first time participating in Kaggle competition. Please let me know if there is something I need to know","metadata":{}},{"cell_type":"markdown","source":"

1    INSTALL LIBRARY

","metadata":{}},{"cell_type":"code","source":"!pip install tflite_runtime","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:04:36.343492Z","iopub.execute_input":"2023-04-13T02:04:36.344227Z","iopub.status.idle":"2023-04-13T02:04:47.277647Z","shell.execute_reply.started":"2023-04-13T02:04:36.344186Z","shell.execute_reply":"2023-04-13T02:04:47.276447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

2    IMPORT LIBRARY

","metadata":{}},{"cell_type":"code","source":"!pip install onnxsim\n!pip install onnx_tf\nimport os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom glob import glob\n\nimport onnx\nimport onnxsim\nfrom onnx_tf.backend import prepare\nimport tensorflow as tf\n\nimport torch\nimport torch.nn as nn\nimport torch.optim.lr_scheduler as scheduler\nfrom torch.utils.data import Dataset, DataLoader, random_split","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:04:48.797123Z","iopub.execute_input":"2023-04-13T02:04:48.798243Z","iopub.status.idle":"2023-04-13T02:05:29.349056Z","shell.execute_reply.started":"2023-04-13T02:04:48.798195Z","shell.execute_reply":"2023-04-13T02:05:29.347977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

3    DATASET

","metadata":{}},{"cell_type":"code","source":"path = '/kaggle/input/gislr-feature-data-on-the-shoulders'","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:05:32.507251Z","iopub.execute_input":"2023-04-13T02:05:32.508064Z","iopub.status.idle":"2023-04-13T02:05:32.513289Z","shell.execute_reply.started":"2023-04-13T02:05:32.508024Z","shell.execute_reply":"2023-04-13T02:05:32.511810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n data_columns = ['x', 'y', 'z']\n data = pd.read_parquet(pq_path, columns=data_columns)\n n_frames = int(len(data) / ROWS_PER_FRAME)\n data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:05:34.015853Z","iopub.execute_input":"2023-04-13T02:05:34.016842Z","iopub.status.idle":"2023-04-13T02:05:34.022224Z","shell.execute_reply.started":"2023-04-13T02:05:34.016804Z","shell.execute_reply":"2023-04-13T02:05:34.020673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PreprocessedGISLDataset(Dataset):\n def __init__(self, path) -> None:\n super().__init__()\n \n self.dataset = np.load(os.path.join(path, \"feature_data.npy\"))\n self.labels = np.load(os.path.join(path, \"feature_labels.npy\"))\n \n def __len__(self):\n return len(self.dataset)\n \n def __getitem__(self, idx):\n return self.dataset[idx, :], self.labels[idx]\n","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:05:35.886702Z","iopub.execute_input":"2023-04-13T02:05:35.887296Z","iopub.status.idle":"2023-04-13T02:05:35.895160Z","shell.execute_reply.started":"2023-04-13T02:05:35.887258Z","shell.execute_reply":"2023-04-13T02:05:35.893884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = PreprocessedGISLDataset(path)\n\ndataset_size = len(dataset)\ntrain_size = int(dataset_size * 0.9)\nvalidation_size = dataset_size - train_size\n\ntrain_dataset, validation_dataset = random_split(dataset, [train_size, validation_size])\n\ntrain_dataloader = DataLoader(train_dataset, batch_size=256, shuffle=True, drop_last=True)\nvalidation_dataset = DataLoader(validation_dataset, batch_size=64, shuffle=True, drop_last=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:05:37.873839Z","iopub.execute_input":"2023-04-13T02:05:37.874216Z","iopub.status.idle":"2023-04-13T02:06:12.643667Z","shell.execute_reply.started":"2023-04-13T02:05:37.874183Z","shell.execute_reply":"2023-04-13T02:06:12.642583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4    MODEL

","metadata":{}},{"cell_type":"code","source":"class SimpleNet(nn.Module):\n\n def __init__(self):\n super(SimpleNet, self).__init__()\n self.in_feature = 5796\n self._1_feature = 1024\n self._2_feature = 512\n self.out_feature = 250\n\n self.l1 = nn.Sequential(nn.Linear(self.in_feature, self._1_feature),\n nn.BatchNorm1d(self._1_feature),\n nn.ReLU(),\n nn.Dropout(p=0.4),\n )\n \n self.l2 = nn.Sequential(nn.Linear(self._1_feature, self._2_feature),\n nn.BatchNorm1d(self._2_feature),\n nn.ReLU(),\n nn.Dropout(p=0.4),\n )\n \n self.l3 = nn.Sequential(nn.Linear(self._2_feature, self.out_feature),\n nn.BatchNorm1d(self.out_feature),\n nn.ReLU(),\n nn.Dropout(p=0.4),\n )\n self.softmax = nn.Softmax(dim=1)\n \n def forward(self, input):\n out = self.l1(input)\n out = self.l2(out)\n out = self.l3(out)\n out = self.softmax(out)\n return out","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:06:22.636135Z","iopub.execute_input":"2023-04-13T02:06:22.637054Z","iopub.status.idle":"2023-04-13T02:06:22.647917Z","shell.execute_reply.started":"2023-04-13T02:06:22.637003Z","shell.execute_reply":"2023-04-13T02:06:22.646695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = SimpleNet().to(\"cuda\")","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:06:26.276021Z","iopub.execute_input":"2023-04-13T02:06:26.276455Z","iopub.status.idle":"2023-04-13T02:06:31.942491Z","shell.execute_reply.started":"2023-04-13T02:06:26.276414Z","shell.execute_reply":"2023-04-13T02:06:31.941448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5    TRAINING

","metadata":{}},{"cell_type":"code","source":"epochs = 250\nloss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.AdamW(model.parameters(), lr=3e-4, weight_decay=1e-6)\ncosine_scheduler = scheduler.CosineAnnealingLR(optimizer=optimizer,\n T_max=100,\n last_epoch=-1,\n verbose=False)\n\nfor e in range(epochs):\n train_loss = 0\n train_acc = 0\n num_iter = 0\n model.train()\n for x, y in tqdm(train_dataloader):\n num_iter+=1\n pred = model(x.to(torch.float32).to(\"cuda\"))\n loss = loss_fn(pred.to(torch.float32), y.type(torch.LongTensor).to(\"cuda\"))\n \n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n \n pred = pred.max(1).indices\n train_acc+=(torch.sum(pred == y.to(\"cuda\"))/x.shape[0])\n cosine_scheduler.step()\n\n print(f\"Training Acc @ epoch {e}: {train_acc/num_iter*100}%\")\n\n val_loss = 0\n val_acc = 0\n num_iter = 0\n model.eval()\n for x, y in validation_dataset:\n num_iter+=1\n pred = model(x.to(\"cuda\").to(torch.float32))\n loss = loss_fn(pred.to(torch.float32), y.type(torch.LongTensor).to(\"cuda\"))\n \n pred = pred.max(1).indices\n val_acc+=(torch.sum(pred == y.to(\"cuda\"))/x.shape[0])\n print(f\"Validation Acc @ epoch {e}: {val_acc/num_iter*100}%\")\n\ntest_loss = 0\ntest_acc = 0\nnum_iter = 0\nmodel.eval()\n# for x, y in validation_dataset:\n# num_iter+=1\n# pred = model(x.to(\"cuda\").to(torch.float32))\n# loss = loss_fn(pred.to(torch.float32), y.type(torch.LongTensor).to(\"cuda\"))\n\n# pred = pred.max(1).indices\n# test_acc+=(torch.sum(pred == y.to(\"cuda\"))/x.shape[0])\n# print(f\"Test Acc: {test_acc/num_iter}%\")\n","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:06:45.546704Z","iopub.execute_input":"2023-04-13T02:06:45.547736Z","iopub.status.idle":"2023-04-13T02:21:24.961622Z","shell.execute_reply.started":"2023-04-13T02:06:45.547695Z","shell.execute_reply":"2023-04-13T02:21:24.960582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

6    FEATURE PROCESSING MODULE

","metadata":{}},{"cell_type":"code","source":"DROP_Z = False\n\nNUM_FRAMES = 15\nSEGMENTS = 3\n\nLEFT_HAND_OFFSET = 468\nPOSE_OFFSET = LEFT_HAND_OFFSET + 21\nRIGHT_HAND_OFFSET = POSE_OFFSET + 33\n\n## average over the entire face, and the entire 'pose'\naveraging_sets = [[0, 468], [POSE_OFFSET, 33]]\n\nlip_landmarks = [\n 61,\n 185,\n 40,\n 39,\n 37,\n 0,\n 267,\n 269,\n 270,\n 409,\n 291,\n 146,\n 91,\n 181,\n 84,\n 17,\n 314,\n 405,\n 321,\n 375,\n 78,\n 191,\n 80,\n 81,\n 82,\n 13,\n 312,\n 311,\n 310,\n 415,\n 95,\n 88,\n 178,\n 87,\n 14,\n 317,\n 402,\n 318,\n 324,\n 308,\n]\nleft_hand_landmarks = list(range(LEFT_HAND_OFFSET, LEFT_HAND_OFFSET + 21))\nright_hand_landmarks = list(range(RIGHT_HAND_OFFSET, RIGHT_HAND_OFFSET + 21))\n\npoint_landmarks = [\n item\n for sublist in [lip_landmarks, left_hand_landmarks, right_hand_landmarks]\n for item in sublist\n]\n\nLANDMARKS = len(point_landmarks) + len(averaging_sets)\nprint(LANDMARKS)\nif DROP_Z:\n INPUT_SHAPE = (NUM_FRAMES, LANDMARKS * 2)\nelse:\n INPUT_SHAPE = (NUM_FRAMES, LANDMARKS * 3)\n\nFLAT_INPUT_SHAPE = (INPUT_SHAPE[0] + 2 * (SEGMENTS + 1)) * INPUT_SHAPE[1]","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:25:18.546397Z","iopub.execute_input":"2023-04-13T02:25:18.547079Z","iopub.status.idle":"2023-04-13T02:25:18.559728Z","shell.execute_reply.started":"2023-04-13T02:25:18.547042Z","shell.execute_reply":"2023-04-13T02:25:18.558603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tf_nan_mean(x, axis=0):\n return tf.reduce_sum(\n tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis\n ) / tf.reduce_sum(\n tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis\n )\n\n\ndef tf_nan_std(x, axis=0):\n d = x - tf_nan_mean(x, axis=axis)\n return tf.math.sqrt(tf_nan_mean(d * d, axis=axis))\n\n\ndef flatten_means_and_stds(x, axis=0):\n # Get means and stds\n x_mean = tf_nan_mean(x, axis=0)\n x_std = tf_nan_std(x, axis=0)\n\n x_out = tf.concat([x_mean, x_std], axis=0)\n x_out = tf.reshape(x_out, (1, INPUT_SHAPE[1] * 2))\n x_out = tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n return x_out","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:25:21.928454Z","iopub.execute_input":"2023-04-13T02:25:21.928842Z","iopub.status.idle":"2023-04-13T02:25:21.937364Z","shell.execute_reply.started":"2023-04-13T02:25:21.928809Z","shell.execute_reply":"2023-04-13T02:25:21.936241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeatureGen(tf.keras.layers.Layer):\n def __init__(self):\n super(FeatureGen, self).__init__()\n \n def call(self, x_in):\n# print(right_hand_percentage(x))\n x_list = [tf.expand_dims(tf_nan_mean(x_in[:, av_set[0]:av_set[0]+av_set[1], :], axis=1), axis=1) for av_set in averaging_sets]\n x_list.append(tf.gather(x_in, point_landmarks, axis=1))\n x = tf.concat(x_list, 1)\n\n x_padded = x\n for i in range(SEGMENTS):\n p0 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) != 0) , 1, 0)\n p1 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) == 0) , 1, 0)\n paddings = [[p0, p1], [0, 0], [0, 0]]\n x_padded = tf.pad(x_padded, paddings, mode=\"SYMMETRIC\")\n x_list = tf.split(x_padded, SEGMENTS)\n x_list = [flatten_means_and_stds(_x, axis=0) for _x in x_list]\n\n x_list.append(flatten_means_and_stds(x, axis=0))\n \n ## Resize only dimension 0. Resize can't handle nan, so replace nan with that dimension's avg value to reduce impact.\n x = tf.image.resize(tf.where(tf.math.is_finite(x), x, tf_nan_mean(x, axis=0)), [NUM_FRAMES, LANDMARKS])\n x = tf.reshape(x, (1, INPUT_SHAPE[0]*INPUT_SHAPE[1]))\n x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n x_list.append(x)\n x = tf.concat(x_list, axis=1)\n return x\n","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:25:24.950155Z","iopub.execute_input":"2023-04-13T02:25:24.950524Z","iopub.status.idle":"2023-04-13T02:25:24.961704Z","shell.execute_reply.started":"2023-04-13T02:25:24.950492Z","shell.execute_reply":"2023-04-13T02:25:24.960609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

7    TORCH TO ONNX

","metadata":{}},{"cell_type":"code","source":"x = torch.randn(2, 5796, requires_grad=True).to(\"cuda\")\n# torch_out = torch_model(x)\n\nmodel = model.to(\"cuda\")\n\ntorch.onnx.export(model, \n x, \n \"MLP.onnx\", \n export_params=True, \n opset_version=10, \n do_constant_folding=True, \n input_names = ['input'], \n output_names = ['output'],\n dynamic_axes={'input' : {0 : 'batch_size'}, \n 'output' : {0 : 'batch_size'}})","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:25:27.917805Z","iopub.execute_input":"2023-04-13T02:25:27.918361Z","iopub.status.idle":"2023-04-13T02:25:28.370513Z","shell.execute_reply.started":"2023-04-13T02:25:27.918316Z","shell.execute_reply":"2023-04-13T02:25:28.368884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

8    ONNX to TF

","metadata":{}},{"cell_type":"code","source":"tf_model_path = \"tf_MLP\"\nonnx_asl_module = onnx.load(\"MLP.onnx\")\ntf_rep = prepare(onnx_asl_module)\ntf_rep.export_graph(tf_model_path)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:25:32.064878Z","iopub.execute_input":"2023-04-13T02:25:32.065346Z","iopub.status.idle":"2023-04-13T02:25:36.171397Z","shell.execute_reply.started":"2023-04-13T02:25:32.065310Z","shell.execute_reply":"2023-04-13T02:25:36.170334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TFModel(tf.Module):\n def __init__(self, tf_model_path):\n super().__init__()\n\n self.feature_gen = FeatureGen()\n self.model = tf.saved_model.load(tf_model_path)\n self.feature_gen.trainable = False\n self.model.trainable = False\n\n @tf.function(\n input_signature=[\n tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name=\"inputs\")\n ]\n )\n def call(self, input):\n output_tensors = {}\n features = self.feature_gen(tf.cast(input, dtype=tf.float32))\n\n output_tensors[\"outputs\"] = self.model(**{\"input\": features})[\"output\"][0, :]\n\n return output_tensors\n\n\nmytfmodel = TFModel(\"./tf_MLP\")\ntf.saved_model.save(\n mytfmodel, \"tf_infer_model\", signatures={\"serving_default\": mytfmodel.call}\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:25:43.938808Z","iopub.execute_input":"2023-04-13T02:25:43.939400Z","iopub.status.idle":"2023-04-13T02:25:45.418520Z","shell.execute_reply.started":"2023-04-13T02:25:43.939362Z","shell.execute_reply":"2023-04-13T02:25:45.417450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

9    TF to TFlite

","metadata":{}},{"cell_type":"code","source":"tf_infer_model_path = \"./tf_infer_model\"\nconverter = tf.lite.TFLiteConverter.from_saved_model(tf_infer_model_path)\ntflite_model = converter.convert()\n\ntflite_model_path = \"model.tflite\"\n\n# Save the model\nwith open(tflite_model_path, \"wb\") as f:\n f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:25:48.417995Z","iopub.execute_input":"2023-04-13T02:25:48.418362Z","iopub.status.idle":"2023-04-13T02:25:50.320317Z","shell.execute_reply.started":"2023-04-13T02:25:48.418330Z","shell.execute_reply":"2023-04-13T02:25:50.318350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pq_path = \"/kaggle/input/asl-signs/train_landmark_files/53618/1001379621.parquet\"\n\nimport tflite_runtime.interpreter as tflite\n\ninterpreter = tflite.Interpreter(tflite_model_path)\ninterpreter.allocate_tensors()\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\n# if REQUIRED_SIGNATURE not in found_signatures:\n# raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=load_relevant_data_subset(pq_path))\nsign = np.argmax(output[\"outputs\"])\n\nprint(sign, output[\"outputs\"].shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:25:55.547947Z","iopub.execute_input":"2023-04-13T02:25:55.548727Z","iopub.status.idle":"2023-04-13T02:25:55.750961Z","shell.execute_reply.started":"2023-04-13T02:25:55.548689Z","shell.execute_reply":"2023-04-13T02:25:55.749863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip $tflite_model_path","metadata":{"execution":{"iopub.status.busy":"2023-04-13T02:26:00.540630Z","iopub.execute_input":"2023-04-13T02:26:00.541766Z","iopub.status.idle":"2023-04-13T02:26:02.953084Z","shell.execute_reply.started":"2023-04-13T02:26:00.541713Z","shell.execute_reply":"2023-04-13T02:26:02.951832Z"},"trusted":true},"execution_count":null,"outputs":[]}]}