{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## GISLR: PyTorch->TFLite Baseline\n\nCode modified from LONNIE's [Isolated Sign Language Recognition with DNN](https://www.kaggle.com/code/lonnieqin/isolated-sign-language-recognition-with-dnn).\nIn addition, I converted the Keras code to PyTorch.","metadata":{}},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"class CFG:\n    data_path = \"../input/asl-signs/\"\n    quick_experiment = False\n    is_training = True\n    use_aggregation_dataset = True\n    num_classes = 250\n    rows_per_frame = 543 \n    batch_size=128\n    epochs = 150","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:21:07.537375Z","iopub.execute_input":"2023-02-27T12:21:07.538711Z","iopub.status.idle":"2023-02-27T12:21:07.548003Z","shell.execute_reply.started":"2023-02-27T12:21:07.538659Z","shell.execute_reply":"2023-02-27T12:21:07.544526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Library","metadata":{}},{"cell_type":"code","source":"!pip install onnx_tf\n!pip install tflite-runtime","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:21:32.083801Z","iopub.execute_input":"2023-02-27T12:21:32.084207Z","iopub.status.idle":"2023-02-27T12:21:51.290875Z","shell.execute_reply.started":"2023-02-27T12:21:32.084164Z","shell.execute_reply":"2023-02-27T12:21:51.289588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tqdm import tqdm\nimport json\nimport os\nimport gc\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nfrom torch import nn\nfrom torchinfo import summary\nimport onnx\nimport onnx_tf\nfrom onnx_tf.backend import prepare","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:21:51.293188Z","iopub.execute_input":"2023-02-27T12:21:51.293527Z","iopub.status.idle":"2023-02-27T12:22:02.738139Z","shell.execute_reply.started":"2023-02-27T12:21:51.293495Z","shell.execute_reply":"2023-02-27T12:22:02.737013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities","metadata":{}},{"cell_type":"code","source":"def load_relevant_data_subset_with_imputation(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    data.replace(np.nan, 0, inplace=True)\n    n_frames = int(len(data) / CFG.rows_per_frame)\n    data = data.values.reshape(n_frames, CFG.rows_per_frame, len(data_columns))\n    return data.astype(np.float32)\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / CFG.rows_per_frame)\n    data = data.values.reshape(n_frames, CFG.rows_per_frame, len(data_columns))\n    return data.astype(np.float32)\n\ndef read_dict(file_path):\n    path = os.path.expanduser(file_path)\n    with open(path, \"r\") as f:\n        dic = json.load(f)\n    return dic","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:06.465638Z","iopub.execute_input":"2023-02-27T12:22:06.466407Z","iopub.status.idle":"2023-02-27T12:22:06.475864Z","shell.execute_reply.started":"2023-02-27T12:22:06.466370Z","shell.execute_reply":"2023-02-27T12:22:06.474540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(f\"{CFG.data_path}train.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:07.943585Z","iopub.execute_input":"2023-02-27T12:22:07.944521Z","iopub.status.idle":"2023-02-27T12:22:08.134133Z","shell.execute_reply.started":"2023-02-27T12:22:07.944481Z","shell.execute_reply":"2023-02-27T12:22:08.133024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.participant_id.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:08.890264Z","iopub.execute_input":"2023-02-27T12:22:08.891461Z","iopub.status.idle":"2023-02-27T12:22:08.904429Z","shell.execute_reply.started":"2023-02-27T12:22:08.891415Z","shell.execute_reply":"2023-02-27T12:22:08.903043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.participant_id.value_counts().plot(kind=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:09.409619Z","iopub.execute_input":"2023-02-27T12:22:09.410309Z","iopub.status.idle":"2023-02-27T12:22:09.732860Z","shell.execute_reply.started":"2023-02-27T12:22:09.410268Z","shell.execute_reply":"2023-02-27T12:22:09.731856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:09.975260Z","iopub.execute_input":"2023-02-27T12:22:09.975857Z","iopub.status.idle":"2023-02-27T12:22:09.984878Z","shell.execute_reply.started":"2023-02-27T12:22:09.975816Z","shell.execute_reply":"2023-02-27T12:22:09.983856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_index = read_dict(f\"{CFG.data_path}sign_to_prediction_index_map.json\")\nindex_label = dict([(label_index[key], key) for key in label_index])\nprint(label_index)\ntrain[\"label\"] = train[\"sign\"].map(lambda sign: label_index[sign])\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:10.465933Z","iopub.execute_input":"2023-02-27T12:22:10.466664Z","iopub.status.idle":"2023-02-27T12:22:10.552083Z","shell.execute_reply.started":"2023-02-27T12:22:10.466625Z","shell.execute_reply":"2023-02-27T12:22:10.551103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"sign\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:11.204216Z","iopub.execute_input":"2023-02-27T12:22:11.205325Z","iopub.status.idle":"2023-02-27T12:22:11.219839Z","shell.execute_reply.started":"2023-02-27T12:22:11.205286Z","shell.execute_reply":"2023-02-27T12:22:11.218768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling","metadata":{}},{"cell_type":"code","source":"if CFG.is_training:\n    if CFG.use_aggregation_dataset == False:\n        xs = []\n        ys = []\n        num_frames = np.zeros(len(train))\n        for i in tqdm(range(len(train))):\n            path = f\"{CFG.data_path}{train.iloc[i].path}\"\n            data = load_relevant_data_subset_with_imputation(path)\n            ## Mean Aggregation\n            xs.append(np.mean(data, axis=0))\n            ys.append(train.iloc[i].label)\n            num_frames[i] = data.shape[0]\n            if CFG.quick_experiment and i == 4999:\n                break\n        ## Save number of frames of each training sample for data analysis\n        train[\"num_frames\"] = num_frames\n        X = np.array(xs)\n        y = np.array(ys)\n        print(train[\"num_frames\"].describe())\n        train.to_csv(\"train.csv\", index=False)\n    else:\n        X = np.load(\"/kaggle/input/isolated-sign-language-aggregation-dataset/X.npy\")\n        y = np.load(\"/kaggle/input/isolated-sign-language-aggregation-dataset/y.npy\")\n    print(X.shape, y.shape)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:12.881652Z","iopub.execute_input":"2023-02-27T12:22:12.882348Z","iopub.status.idle":"2023-02-27T12:22:15.681781Z","shell.execute_reply.started":"2023-02-27T12:22:12.882310Z","shell.execute_reply":"2023-02-27T12:22:15.680590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PyTorch Code","metadata":{}},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"mps\" if torch.backends.mps.is_available() else \"cpu\"\nprint(f\"Using {device} device\")","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:15.736477Z","iopub.execute_input":"2023-02-27T12:22:15.736788Z","iopub.status.idle":"2023-02-27T12:22:15.805495Z","shell.execute_reply.started":"2023-02-27T12:22:15.736759Z","shell.execute_reply":"2023-02-27T12:22:15.803550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super(Model, self).__init__()\n        self.layer1 = nn.Linear(3, 128)\n        self.layer2 = nn.Linear(128, 64)\n        self.layer3 = nn.Linear(64, 32)\n        self.layer4 = nn.Linear(32, 16)\n        self.layer5 = nn.Linear(16 * 543, 250)\n        self.relu = nn.ReLU()\n        self.flatten = nn.Flatten()\n\n    def forward(self, x):\n        x = self.relu(self.layer1(x))\n        x = self.relu(self.layer2(x))\n        x = self.relu(self.layer3(x))\n        x = self.relu(self.layer4(x))\n        x = self.flatten(x)\n        x = self.layer5(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:16.593786Z","iopub.execute_input":"2023-02-27T12:22:16.594216Z","iopub.status.idle":"2023-02-27T12:22:16.603119Z","shell.execute_reply.started":"2023-02-27T12:22:16.594180Z","shell.execute_reply":"2023-02-27T12:22:16.601651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    def __init__(self, X, y):\n        self.X = X\n        self.y = y\n\n    def __len__(self):\n        return len(self.y)\n\n    def __getitem__(self, i):\n        return self.X[i].astype(np.float32), self.y[i]","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:17.444830Z","iopub.execute_input":"2023-02-27T12:22:17.447391Z","iopub.status.idle":"2023-02-27T12:22:17.453578Z","shell.execute_reply.started":"2023-02-27T12:22:17.447349Z","shell.execute_reply":"2023-02-27T12:22:17.452275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_fit(train_dataloader, test_dataloader, model, criterion, optimizer, epochs):\n    for iepoch in range(epochs):\n        train_loss, train_correct, train_n, val_loss, val_correct, val_n = 0,0,0,0,0,0\n        \n        model.train()\n        for ibatch, (X, y) in enumerate(train_dataloader):\n            X, y = X.to(device), y.to(device)\n            \n            optimizer.zero_grad()\n            y_pred = model(X)\n            loss = criterion(y_pred, y)\n            \n            train_n += y.size(0)\n            train_loss += loss.item()\n            train_correct += (y_pred.argmax(1) == y).type(torch.float).sum().item()\n            \n            loss.backward()\n            optimizer.step()\n            \n        train_loss /= ibatch   \n        train_correct /= train_n\n        \n        model.eval()\n        \n        for ibatch, (X, y) in enumerate(test_dataloader):\n            X, y = X.to(device), y.to(device)\n            with torch.no_grad():\n                y_pred = model(X)\n                loss = criterion(y_pred, y)\n                val_n += y.size(0)\n                val_loss += loss.item()\n                val_correct += (y_pred.argmax(1) == y).type(torch.float).sum().item()\n        \n        val_loss /= ibatch   \n        val_correct /= val_n\n        \n        print('Epoch %d/%d loss:%.4f accuracy:%.4f val_loss:%.4f val_accuracy:%.4f' %(iepoch + 1, epochs, train_loss, train_correct, val_loss, val_correct))","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:18.273775Z","iopub.execute_input":"2023-02-27T12:22:18.274189Z","iopub.status.idle":"2023-02-27T12:22:18.285954Z","shell.execute_reply.started":"2023-02-27T12:22:18.274146Z","shell.execute_reply":"2023-02-27T12:22:18.284818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.is_training:\n    num_workers = 2\n    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\n    del X, y\n    gc.collect()\n    train_dataset = Dataset(X_train, y_train)\n    val_dataset = Dataset(X_val, y_val)\n    train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True,\n                     num_workers=num_workers, pin_memory=True, drop_last=True)\n    test_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=CFG.batch_size, shuffle=False,num_workers=num_workers, pin_memory=True)\n    \n    model = Model().to(device)\n    criterion = torch.nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)\n    model_fit(train_dataloader, test_dataloader, model, criterion, optimizer, epochs=CFG.epochs)\n\ntorch.save(model.state_dict(), 'model.pytorch')\nsummary(model=model, input_size=(CFG.batch_size, 543, 3))","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:19.231915Z","iopub.execute_input":"2023-02-27T12:22:19.232662Z","iopub.status.idle":"2023-02-27T12:22:37.673632Z","shell.execute_reply.started":"2023-02-27T12:22:19.232621Z","shell.execute_reply":"2023-02-27T12:22:37.672111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Model for inference","metadata":{}},{"cell_type":"code","source":"class Model_infe(Model):\n    \n    def __init__(self):\n        super().__init__()\n        self.softmax = nn.Softmax()\n        \n    def forward(self, x):\n        x = torch.where(torch.isnan(x), torch.tensor(0.0, dtype=torch.float32).to(device), x)\n        x = torch.mean(x, dim=0, keepdim=False)\n        x = self.relu(self.layer1(x))\n\n        x = self.relu(self.layer2(x))\n        x = self.relu(self.layer3(x))\n        x = self.relu(self.layer4(x))\n        x = self.flatten(x)\n        x = self.layer5(x)\n        return self.softmax(x)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:48.319846Z","iopub.execute_input":"2023-02-27T12:22:48.320284Z","iopub.status.idle":"2023-02-27T12:22:48.329821Z","shell.execute_reply.started":"2023-02-27T12:22:48.320244Z","shell.execute_reply":"2023-02-27T12:22:48.328168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_infe = Model_infe()\n\nmodel_infe.load_state_dict(torch.load('model.pytorch'), strict=False)\nmodel_infe = model_infe.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:49.431407Z","iopub.execute_input":"2023-02-27T12:22:49.432345Z","iopub.status.idle":"2023-02-27T12:22:49.472033Z","shell.execute_reply.started":"2023-02-27T12:22:49.432290Z","shell.execute_reply":"2023-02-27T12:22:49.470993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert from Pytorch to Tensorflow Lite model\n#### Reference\n[1] [YACINE BOUAOUNI - Tensorflow/Pytorch To TFLite](https://www.kaggle.com/code/jarvisai7/tensorflow-pytorch-to-tflite)\n\n[2] [RAJAT MHETRE - pytorch -> tflite](https://www.kaggle.com/code/mhetrerajat/pytorch-tflite)","metadata":{}},{"cell_type":"code","source":"img_size = (1,543, 3)\nbatch_size = 1\nonnx_model_path = 'model.onnx'\n\nmodel_infe.eval()\n\nsample_input = torch.rand((batch_size, *img_size)).to(device)\n\ny = model_infe(sample_input)\n\ntorch.onnx.export(\n    model_infe,\n    sample_input, \n    onnx_model_path,\n    verbose=False,\n    input_names=['inputs'],\n    output_names=['outputs'],\n    opset_version=12\n)\n\n\nonnx_model_path = 'model.onnx'\ntf_model_path = 'model_tf'\n\nonnx_model = onnx.load(onnx_model_path)\ntf_rep = prepare(onnx_model)\ntf_rep.export_graph(tf_model_path)\n\nprint(tf_rep)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:51.241745Z","iopub.execute_input":"2023-02-27T12:22:51.242426Z","iopub.status.idle":"2023-02-27T12:22:55.640620Z","shell.execute_reply.started":"2023-02-27T12:22:51.242389Z","shell.execute_reply":"2023-02-27T12:22:55.639428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create submission file","metadata":{}},{"cell_type":"code","source":"saved_model_dir = 'model_tf'\ntflite_model_path = 'model.tflite'\n\nconverter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir)\ntflite_model = converter.convert()\n\nwith open(tflite_model_path, 'wb') as f:\n    f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:22:58.380253Z","iopub.execute_input":"2023-02-27T12:22:58.381526Z","iopub.status.idle":"2023-02-27T12:22:59.067338Z","shell.execute_reply.started":"2023-02-27T12:22:58.381471Z","shell.execute_reply":"2023-02-27T12:22:59.066037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip $tflite_model_path","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:23:00.426439Z","iopub.execute_input":"2023-02-27T12:23:00.426830Z","iopub.status.idle":"2023-02-27T12:23:01.889684Z","shell.execute_reply.started":"2023-02-27T12:23:00.426794Z","shell.execute_reply":"2023-02-27T12:23:01.888459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making Prediction","metadata":{}},{"cell_type":"code","source":"import tflite_runtime.interpreter as tflite\ninterpreter = tflite.Interpreter(tflite_model_path)\nfound_signatures = list(interpreter.get_signature_list().keys())\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\nfor i in range(100):\n    frames = load_relevant_data_subset(f'/kaggle/input/asl-signs/{train.iloc[i].path}')\n    output = prediction_fn(inputs=frames)\n    sign = np.argmax(output[\"outputs\"])\n    print(f\"Predicted label: {index_label[sign]}, Actual Label: {train.iloc[i].sign}\")","metadata":{"execution":{"iopub.status.busy":"2023-02-27T12:23:04.611310Z","iopub.execute_input":"2023-02-27T12:23:04.612518Z","iopub.status.idle":"2023-02-27T12:23:06.378777Z","shell.execute_reply.started":"2023-02-27T12:23:04.612475Z","shell.execute_reply":"2023-02-27T12:23:06.377568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}