{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# End to End Pytorch:\n\nThis notebook trains a pytorch model or rather 2 pytorch models to do our work. The pre-processing part of the work which converts our data to the final form is done by a set of torch operations inside a dummy model without any trainable parameters. This helps ONNX convert those operations easily so that we don't have to write extra tensorflow code during inference to format our data. I have explained it in more detail over [here](https://www.kaggle.com/competitions/asl-signs/discussion/391301).\n\nIf you are looking to understand the data and task, you can check out my [EDA notebook](https://www.kaggle.com/code/mayukh18/sign-language-eda-visualization/).","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\n\nimport json\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nimport warnings\nwarnings.filterwarnings(action='ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-04T06:22:41.221235Z","iopub.execute_input":"2023-03-04T06:22:41.222088Z","iopub.status.idle":"2023-03-04T06:22:42.887401Z","shell.execute_reply.started":"2023-03-04T06:22:41.222014Z","shell.execute_reply":"2023-03-04T06:22:42.886282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LANDMARK_FILES_DIR = \"/kaggle/input/asl-signs/train_landmark_files\"\nTRAIN_FILE = \"/kaggle/input/asl-signs/train.csv\"\nlabel_map = json.load(open(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\", \"r\"))","metadata":{"execution":{"iopub.status.busy":"2023-03-04T06:22:42.889432Z","iopub.execute_input":"2023-03-04T06:22:42.890537Z","iopub.status.idle":"2023-03-04T06:22:42.900352Z","shell.execute_reply.started":"2023-03-04T06:22:42.890497Z","shell.execute_reply":"2023-03-04T06:22:42.899323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Gen / Pre-Process Model\n\nConverts the (n_frames, 543, 3) data to (n_features,) form.","metadata":{}},{"cell_type":"code","source":"class FeatureGen(nn.Module):\n    def __init__(self):\n        super(FeatureGen, self).__init__()\n        pass\n    \n    def forward(self, x):\n        face_x = x[:,:468,:].contiguous().view(-1, 468*3)\n        lefth_x = x[:,468:489,:].contiguous().view(-1, 21*3)\n        pose_x = x[:,489:522,:].contiguous().view(-1, 33*3)\n        righth_x = x[:,522:,:].contiguous().view(-1, 21*3)\n        \n        lefth_x = lefth_x[~torch.any(torch.isnan(lefth_x), dim=1),:]\n        righth_x = righth_x[~torch.any(torch.isnan(righth_x), dim=1),:]\n        \n        x1m = torch.mean(face_x, 0)\n        x2m = torch.mean(lefth_x, 0)\n        x3m = torch.mean(pose_x, 0)\n        x4m = torch.mean(righth_x, 0)\n        \n        x1s = torch.std(face_x, 0)\n        x2s = torch.std(lefth_x, 0)\n        x3s = torch.std(pose_x, 0)\n        x4s = torch.std(righth_x, 0)\n        \n        xfeat = torch.cat([x1m,x2m,x3m,x4m, x1s,x2s,x3s,x4s], axis=0)\n        xfeat = torch.where(torch.isnan(xfeat), torch.tensor(0.0, dtype=torch.float32), xfeat)\n        \n        return xfeat\n    \nfeature_converter = FeatureGen()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T06:22:42.902238Z","iopub.execute_input":"2023-03-04T06:22:42.902595Z","iopub.status.idle":"2023-03-04T06:22:42.914252Z","shell.execute_reply.started":"2023-03-04T06:22:42.902560Z","shell.execute_reply":"2023-03-04T06:22:42.913291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Our actual model","metadata":{}},{"cell_type":"code","source":"class ASLModel(nn.Module):\n    def __init__(self, p):\n        super(ASLModel, self).__init__()\n        self.dropout = nn.Dropout(p)\n        self.layer0 = nn.Linear(3258, 1024)\n        self.layer1 = nn.Linear(1024, 512)\n        self.layer2 = nn.Linear(512, 250)\n        \n    def forward(self, x):\n        x = self.layer0(x)\n        x = self.dropout(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-03-04T06:22:42.916698Z","iopub.execute_input":"2023-03-04T06:22:42.917075Z","iopub.status.idle":"2023-03-04T06:22:42.926119Z","shell.execute_reply.started":"2023-03-04T06:22:42.917039Z","shell.execute_reply":"2023-03-04T06:22:42.925118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\nclass ASLData(Dataset):\n    def __init__(self, datax, datay):\n        self.datax = datax\n        self.datay = datay\n        \n    def __getitem__(self, index):\n        return self.datax[index,:], self.datay[index]\n        \n    def __len__(self):\n        return len(self.datay)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T06:22:43.068990Z","iopub.execute_input":"2023-03-04T06:22:43.069684Z","iopub.status.idle":"2023-03-04T06:22:43.076790Z","shell.execute_reply.started":"2023-03-04T06:22:43.069639Z","shell.execute_reply":"2023-03-04T06:22:43.075625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Code for Feature Gen /Pre Process\nIt takes about 11 mins with multiprocessing, the current data is saved in a dataset. Run this code when you do your own processing.","metadata":{}},{"cell_type":"code","source":"# import multiprocessing as mp\n\n# def convert_row(row):\n#     x = load_relevant_data_subset(os.path.join(\"/kaggle/input/asl-signs\", row[1].path))\n#     x = feature_converter(torch.tensor(x)).cpu().numpy()\n#     return x, row[1].label\n\n# def convert_and_save_data():\n#     df = pd.read_csv(TRAIN_FILE)\n#     df['label'] = df['sign'].map(label_map)\n#     npdata = np.zeros((df.shape[0], 3258))\n#     nplabels = np.zeros(df.shape[0])\n#     with mp.Pool() as pool:\n#         results = pool.imap(convert_row, df.iterrows(), chunksize=250)\n#         for i, (x,y) in tqdm(enumerate(results), total=df.shape[0]):\n#             npdata[i,:] = x\n#             nplabels[i] = y\n    \n#     np.save(\"feature_data.npy\", npdata)\n#     np.save(\"feature_labels.npy\", nplabels)\n        \n# convert_and_save_data()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T06:22:43.950366Z","iopub.execute_input":"2023-03-04T06:22:43.950744Z","iopub.status.idle":"2023-03-04T06:22:43.956121Z","shell.execute_reply.started":"2023-03-04T06:22:43.950711Z","shell.execute_reply":"2023-03-04T06:22:43.954991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"datax = np.load(\"/kaggle/input/gislr-feature-data/feature_data.npy\")\ndatay = np.load(\"/kaggle/input/gislr-feature-data/feature_labels.npy\")","metadata":{"execution":{"iopub.status.busy":"2023-03-04T06:22:44.594938Z","iopub.execute_input":"2023-03-04T06:22:44.595660Z","iopub.status.idle":"2023-03-04T06:23:06.291699Z","shell.execute_reply.started":"2023-03-04T06:22:44.595615Z","shell.execute_reply":"2023-03-04T06:23:06.290669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 40\nBATCH_SIZE = 64\n\ntrainx, testx, trainy, testy = train_test_split(datax, datay, test_size=0.15, random_state=42)\n\ntrain_data = ASLData(trainx, trainy)\nvalid_data = ASLData(testx, testy)\n\ntrain_loader = DataLoader(train_data, batch_size=BATCH_SIZE, num_workers=4, shuffle=True)\nval_loader = DataLoader(valid_data, batch_size=BATCH_SIZE, num_workers=4, shuffle=False)\n\nmodel = ASLModel(0.2).cuda()\nopt = torch.optim.Adam(model.parameters(), lr=0.005)\ncriterion = nn.CrossEntropyLoss()\nsched = torch.optim.lr_scheduler.StepLR(opt, step_size=300, gamma=0.95)\n\nfor i in range(EPOCHS):\n    model.train()\n    \n    train_loss_sum = 0.\n    train_correct = 0\n    train_total = 0\n    train_bar = train_loader\n    for x,y in train_bar:\n        x = torch.Tensor(x).float().cuda()\n        y = torch.Tensor(y).long().cuda()  \n        y_pred = model(x)\n        \n        loss = criterion(y_pred, y)\n        loss.backward()\n        opt.step()\n        opt.zero_grad()\n        \n        train_loss_sum += loss.item()\n        train_correct += np.sum((np.argmax(y_pred.detach().cpu().numpy(), axis=1) == y.cpu().numpy()))\n        train_total += 1\n        sched.step()\n        \n    val_loss_sum = 0.\n    val_correct = 0\n    val_total = 0\n    model.eval()\n    for x,y in val_loader:\n        x = torch.Tensor(x).float().cuda()\n        y = torch.Tensor(y).long().cuda()\n        \n        with torch.no_grad():\n            y_pred = model(x)\n            loss = criterion(y_pred, y)\n            val_loss_sum += loss.item()\n            val_correct += np.sum((np.argmax(y_pred.cpu().numpy(), axis=1) == y.cpu().numpy()))\n            val_total += 1\n                              \n    print(f\"Epoch:{i} > Train Loss: {(train_loss_sum/train_total):.04f}, Train Acc: {train_correct/len(train_data):0.04f}\")\n    print(f\"Epoch:{i} > Val Loss: {(val_loss_sum/val_total):.04f}, Val Acc: {val_correct/len(valid_data):0.04f}\")\n    print(\"=\"*50)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T06:23:06.293771Z","iopub.execute_input":"2023-03-04T06:23:06.294136Z","iopub.status.idle":"2023-03-04T06:28:28.448255Z","shell.execute_reply.started":"2023-03-04T06:23:06.294100Z","shell.execute_reply":"2023-03-04T06:28:28.446935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-28T19:48:49.112741Z","iopub.execute_input":"2023-02-28T19:48:49.113036Z","iopub.status.idle":"2023-02-28T19:48:49.285718Z","shell.execute_reply.started":"2023-02-28T19:48:49.113002Z","shell.execute_reply":"2023-02-28T19:48:49.281793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorflow Conversion","metadata":{}},{"cell_type":"code","source":"!pip install onnx-tf\n!pip install tflite-runtime","metadata":{"execution":{"iopub.status.busy":"2023-02-28T19:48:49.288785Z","iopub.execute_input":"2023-02-28T19:48:49.289301Z","iopub.status.idle":"2023-02-28T19:49:11.832122Z","shell.execute_reply.started":"2023-02-28T19:48:49.289261Z","shell.execute_reply":"2023-02-28T19:49:11.830895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_input = torch.rand((50, 543, 3))\nonnx_feat_gen_path = 'feature_gen.onnx'\n\nfeature_converter.eval()\n\ntorch.onnx.export(\n    feature_converter,                  # PyTorch Model\n    sample_input,                    # Input tensor\n    onnx_feat_gen_path,        # Output file (eg. 'output_model.onnx')\n    opset_version=12,       # Operator support version\n    input_names=['input'],   # Input tensor name (arbitary)\n    output_names=['output'], # Output tensor name (arbitary)\n    dynamic_axes={\n        'input' : {0: 'input'}\n    }\n)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T20:00:19.204075Z","iopub.execute_input":"2023-02-28T20:00:19.205015Z","iopub.status.idle":"2023-02-28T20:00:19.292397Z","shell.execute_reply.started":"2023-02-28T20:00:19.204961Z","shell.execute_reply":"2023-02-28T20:00:19.291387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_input = torch.rand((1, 3258)).cuda()\nonnx_model_path = 'asl_model.onnx'\n\nmodel.eval()\n\ntorch.onnx.export(\n    model,                  # PyTorch Model\n    sample_input,                    # Input tensor\n    onnx_model_path,        # Output file (eg. 'output_model.onnx')\n    opset_version=12,       # Operator support version\n    input_names=['input'],   # Input tensor name (arbitary)\n    output_names=['output'], # Output tensor name (arbitary)\n    dynamic_axes={\n        'input' : {0: 'input'}\n    }\n)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T20:00:19.924783Z","iopub.execute_input":"2023-02-28T20:00:19.925490Z","iopub.status.idle":"2023-02-28T20:00:21.887040Z","shell.execute_reply.started":"2023-02-28T20:00:19.925450Z","shell.execute_reply":"2023-02-28T20:00:21.885892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import onnx\nfrom onnx_tf.backend import prepare\n\n\ntf_feat_gen_path = '/kaggle/working/tf_feat_gen'\nonnx_feat_gen = onnx.load(onnx_feat_gen_path)\ntf_rep = prepare(onnx_feat_gen)\ntf_rep.export_graph(tf_feat_gen_path)\n\n\ntf_model_path = '/kaggle/working/tf_model'\nonnx_model = onnx.load(onnx_model_path)\ntf_rep = prepare(onnx_model)\ntf_rep.export_graph(tf_model_path)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T20:00:39.908137Z","iopub.execute_input":"2023-02-28T20:00:39.909129Z","iopub.status.idle":"2023-02-28T20:00:48.393161Z","shell.execute_reply.started":"2023-02-28T20:00:39.909091Z","shell.execute_reply":"2023-02-28T20:00:48.390238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Inference Model in Tensorflow\nBoth of the converted models will be used here one after another.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\nclass ASLInferModel(tf.Module):\n    def __init__(self):\n        super(ASLInferModel, self).__init__()\n        self.feature_gen = tf.saved_model.load(tf_feat_gen_path)\n        self.model = tf.saved_model.load(tf_model_path)\n        self.feature_gen.trainable = False\n        self.model.trainable = False\n    \n    @tf.function(input_signature=[\n      tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='inputs')\n    ])\n    def call(self, input):\n        output_tensors = {}\n        features = self.feature_gen(**{'input': input})['output']\n        output_tensors['outputs'] = self.model(**{'input': tf.expand_dims(features, 0)})['output'][0,:]\n        return output_tensors\n    \n    \nmytfmodel = ASLInferModel()\ntf.saved_model.save(mytfmodel, '/kaggle/working/tf_infer_model', signatures={'serving_default': mytfmodel.call})","metadata":{"execution":{"iopub.status.busy":"2023-02-28T20:05:34.208008Z","iopub.execute_input":"2023-02-28T20:05:34.208999Z","iopub.status.idle":"2023-02-28T20:05:35.968351Z","shell.execute_reply.started":"2023-02-28T20:05:34.208957Z","shell.execute_reply":"2023-02-28T20:05:35.967105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# Convert the model\n\ntf_infer_model_path = '/kaggle/working/tf_infer_model'\nconverter = tf.lite.TFLiteConverter.from_saved_model(tf_infer_model_path)\ntflite_model = converter.convert()\n\ntflite_model_path = 'model.tflite'\n\n# Save the model\nwith open(tflite_model_path, 'wb') as f:\n    f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T20:08:31.339838Z","iopub.execute_input":"2023-02-28T20:08:31.340536Z","iopub.status.idle":"2023-02-28T20:08:32.954620Z","shell.execute_reply.started":"2023-02-28T20:08:31.340496Z","shell.execute_reply":"2023-02-28T20:08:32.953296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\npq_path = \"/kaggle/input/asl-signs/train_landmark_files/53618/1001379621.parquet\"\n\nimport tflite_runtime.interpreter as tflite\ninterpreter = tflite.Interpreter(tflite_model_path)\ninterpreter.allocate_tensors()\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\n# if REQUIRED_SIGNATURE not in found_signatures:\n#     raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=load_relevant_data_subset(pq_path))\nsign = np.argmax(output[\"outputs\"])\n\nprint(sign, output[\"outputs\"].shape)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T20:09:47.663833Z","iopub.execute_input":"2023-02-28T20:09:47.665013Z","iopub.status.idle":"2023-02-28T20:09:47.688552Z","shell.execute_reply.started":"2023-02-28T20:09:47.664969Z","shell.execute_reply":"2023-02-28T20:09:47.687510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip $tflite_model_path","metadata":{},"execution_count":null,"outputs":[]}]}