{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport math\n\nimport json\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\n\nimport tensorflow as tf\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nimport warnings\nwarnings.filterwarnings(action='ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-15T09:52:36.052806Z","iopub.execute_input":"2023-03-15T09:52:36.053274Z","iopub.status.idle":"2023-03-15T09:52:44.928680Z","shell.execute_reply.started":"2023-03-15T09:52:36.053233Z","shell.execute_reply":"2023-03-15T09:52:44.927415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LANDMARK_FILES_DIR = \"/kaggle/input/asl-signs/train_landmark_files\"\nTRAIN_FILE = \"/kaggle/input/asl-signs/train.csv\"\nlabel_map = json.load(open(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\", \"r\"))","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:44.931381Z","iopub.execute_input":"2023-03-15T09:52:44.932719Z","iopub.status.idle":"2023-03-15T09:52:44.942326Z","shell.execute_reply.started":"2023-03-15T09:52:44.932673Z","shell.execute_reply":"2023-03-15T09:52:44.941160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Gen / Pre-Process Model\n\nConverts the (n_frames, 543, 3) data to (n_features,) form.","metadata":{}},{"cell_type":"code","source":"class FeatureGen(nn.Module):\n    def __init__(self):\n        super(FeatureGen, self).__init__()\n        pass\n    \n    def forward(self, x):\n        face_x = x[:,:468,:].contiguous().view(-1, 468*3)\n        lefth_x = x[:,468:489,:].contiguous().view(-1, 21*3)\n        pose_x = x[:,489:522,:].contiguous().view(-1, 33*3)\n        righth_x = x[:,522:,:].contiguous().view(-1, 21*3)\n        \n        lefth_x = lefth_x[~torch.any(torch.isnan(lefth_x), dim=1),:]\n        righth_x = righth_x[~torch.any(torch.isnan(righth_x), dim=1),:]\n        \n        x1m = torch.mean(face_x, 0)\n        x2m = torch.mean(lefth_x, 0)\n        x3m = torch.mean(pose_x, 0)\n        x4m = torch.mean(righth_x, 0)\n        \n        x1s = torch.std(face_x, 0)\n        x2s = torch.std(lefth_x, 0)\n        x3s = torch.std(pose_x, 0)\n        x4s = torch.std(righth_x, 0)\n        \n        xfeat = torch.cat([x1m,x2m,x3m,x4m, x1s,x2s,x3s,x4s], axis=0)\n        xfeat = torch.where(torch.isnan(xfeat), torch.tensor(0.0, dtype=torch.float32), xfeat)\n        \n        return xfeat\n    \nfeature_converter = FeatureGen()","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:44.945339Z","iopub.execute_input":"2023-03-15T09:52:44.946330Z","iopub.status.idle":"2023-03-15T09:52:44.959497Z","shell.execute_reply.started":"2023-03-15T09:52:44.946289Z","shell.execute_reply":"2023-03-15T09:52:44.958241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DROP_Z = False\n\nNUM_FRAMES = 15\nSEGMENTS = 3\n\nLEFT_HAND_OFFSET = 468\nPOSE_OFFSET = LEFT_HAND_OFFSET+21\nRIGHT_HAND_OFFSET = POSE_OFFSET+33\n\n## average over the entire face, and the entire 'pose'\naveraging_sets = [[0, 468], [POSE_OFFSET, 33]]\n\nlip_landmarks = [61, 185, 40, 39, 37,  0, 267, 269, 270, 409,\n                 291,146, 91,181, 84, 17, 314, 405, 321, 375, \n                 78, 191, 80, 81, 82, 13, 312, 311, 310, 415, \n                 95, 88, 178, 87, 14,317, 402, 318, 324, 308]\nleft_hand_landmarks = list(range(LEFT_HAND_OFFSET, LEFT_HAND_OFFSET+21))\nright_hand_landmarks = list(range(RIGHT_HAND_OFFSET, RIGHT_HAND_OFFSET+21))\n\npoint_landmarks = [item for sublist in [lip_landmarks, left_hand_landmarks, right_hand_landmarks] for item in sublist]\n\nLANDMARKS = len(point_landmarks) + len(averaging_sets)\nprint(LANDMARKS)\nif DROP_Z:\n    INPUT_SHAPE = (NUM_FRAMES,LANDMARKS*2)\nelse:\n    INPUT_SHAPE = (NUM_FRAMES,LANDMARKS*3)\n\nFLAT_INPUT_SHAPE = (INPUT_SHAPE[0] + 2 * (SEGMENTS + 1)) * INPUT_SHAPE[1]\n# Configuration\n\nNUM_FRAMES = 15\nSEGMENTS = 3\n\nLEFT_HAND_OFFSET = 468\nPOSE_OFFSET = LEFT_HAND_OFFSET+21\nRIGHT_HAND_OFFSET = POSE_OFFSET+33\n\n## average over the entire face, and the entire 'pose'\naveraging_sets = [[0, 468], [POSE_OFFSET, 33]]\n\nlip_landmarks = [61, 185, 40, 39, 37,  0, 267, 269, 270, 409,\n                 291,146, 91,181, 84, 17, 314, 405, 321, 375, \n                 78, 191, 80, 81, 82, 13, 312, 311, 310, 415, \n                 95, 88, 178, 87, 14,317, 402, 318, 324, 308]\nleft_hand_landmarks = list(range(LEFT_HAND_OFFSET, LEFT_HAND_OFFSET+21))\nright_hand_landmarks = list(range(RIGHT_HAND_OFFSET, RIGHT_HAND_OFFSET+21))\n\npoint_landmarks = [item for sublist in [lip_landmarks, left_hand_landmarks, right_hand_landmarks] for item in sublist]\n\nLANDMARKS = len(point_landmarks) + len(averaging_sets)\nprint(LANDMARKS)\nINPUT_SHAPE = (NUM_FRAMES,LANDMARKS*3)\n\n# Helper Functions\ndef tf_nan_mean(x, axis=0):\n    return tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis) / tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis)\n\ndef tf_nan_std(x, axis=0):\n    d = x - tf_nan_mean(x, axis=axis)\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis))\n\ndef flatten_means_and_stds(x, axis=0):\n    # Get means and stds\n    x_mean = tf_nan_mean(x, axis=0)\n    x_std  = tf_nan_std(x,  axis=0)\n\n    x_out = tf.concat([x_mean, x_std], axis=0)\n    x_out = tf.reshape(x_out, (1, INPUT_SHAPE[1]*2))\n    x_out = tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n    return x_out\n\nclass FeatureGenTF(tf.keras.layers.Layer):\n    def __init__(self):\n        super(FeatureGenTF, self).__init__()\n    \n    def call(self, x_in):\n#         print(right_hand_percentage(x))\n        x_list = [tf.expand_dims(tf_nan_mean(x_in[:, av_set[0]:av_set[0]+av_set[1], :], axis=1), axis=1) for av_set in averaging_sets]\n        x_list.append(tf.gather(x_in, point_landmarks, axis=1))\n        x = tf.concat(x_list, 1)\n\n        x_padded = x\n        for i in range(SEGMENTS):\n            p0 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) != 0) , 1, 0)\n            p1 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) == 0) , 1, 0)\n            paddings = [[p0, p1], [0, 0], [0, 0]]\n            x_padded = tf.pad(x_padded, paddings, mode=\"SYMMETRIC\")\n        x_list = tf.split(x_padded, SEGMENTS)\n        x_list = [flatten_means_and_stds(_x, axis=0) for _x in x_list]\n\n        x_list.append(flatten_means_and_stds(x, axis=0))\n        \n        ## Resize only dimension 0. Resize can't handle nan, so replace nan with that dimension's avg value to reduce impact.\n        x = tf.image.resize(tf.where(tf.math.is_finite(x), x, tf_nan_mean(x, axis=0)), [NUM_FRAMES, LANDMARKS])\n        x = tf.reshape(x, (1, INPUT_SHAPE[0]*INPUT_SHAPE[1]))\n        x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n        x_list.append(x)\n        x = tf.concat(x_list, axis=1)\n        return x\ndef tf_nan_mean(x, axis=0):\n    return tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis) / tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis)\n\ndef tf_nan_std(x, axis=0):\n    d = x - tf_nan_mean(x, axis=axis)\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis))\n\ndef flatten_means_and_stds(x, axis=0):\n    # Get means and stds\n    x_mean = tf_nan_mean(x, axis=0)\n    x_std  = tf_nan_std(x,  axis=0)\n\n    x_out = tf.concat([x_mean, x_std], axis=0)\n    x_out = tf.reshape(x_out, (1, INPUT_SHAPE[1]*2))\n    x_out = tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n    return x_out\n\nclass FeatureGen_1(tf.keras.layers.Layer):\n    def __init__(self):\n        super(FeatureGen_1, self).__init__()\n    \n    def call(self, x_in):\n        x = tf.gather(x_in, point_landmarks, axis=1)\n\n        x_padded = x\n        for i in range(SEGMENTS):\n            p0 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) != 0) , 1, 0)\n            p1 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) == 0) , 1, 0)\n            paddings = [[p0, p1], [0, 0], [0, 0]]\n            x_padded = tf.pad(x_padded, paddings, mode=\"SYMMETRIC\")\n        x_list = tf.split(x_padded, SEGMENTS)\n        x_list = [flatten_means_and_stds(_x, axis=0) for _x in x_list]\n\n        x_list.append(flatten_means_and_stds(x, axis=0))\n        \n        ## Resize only dimension 0. Resize can't handle nan, so replace nan with that dimension's avg value to reduce impact.\n        x = tf.image.resize(tf.where(tf.math.is_finite(x), x, tf_nan_mean(x, axis=0)), [NUM_FRAMES, LANDMARKS])\n        x = tf.reshape(x, (1, INPUT_SHAPE[0]*INPUT_SHAPE[1]))\n        x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n        x_list.append(x)\n        x = tf.concat(x_list, axis=1)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:44.963703Z","iopub.execute_input":"2023-03-15T09:52:44.964135Z","iopub.status.idle":"2023-03-15T09:52:45.003479Z","shell.execute_reply.started":"2023-03-15T09:52:44.964103Z","shell.execute_reply":"2023-03-15T09:52:45.001962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Our actual model","metadata":{}},{"cell_type":"code","source":"class ASLModel(nn.Module):\n    def __init__(self, p, in_features, n_class):\n        super(ASLModel, self).__init__()\n        self.dropout = nn.Dropout(p)\n        self.layer0 = nn.Linear(in_features, 1024)\n        self.layer1 = nn.Linear(1024, 512)\n        self.layer2 = nn.Linear(512, n_class)\n        \n    def forward(self, x):\n        x = self.layer0(x)\n        x = self.dropout(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        return x\n    \nclass ASLLinearModel(nn.Module):\n    def __init__(\n        self,\n        in_features: int,\n        first_out_features: int,\n        num_classes: int,\n        num_blocks: int,\n        drop_rate: float,\n        loss_fn,\n        arcface\n    ):\n        super().__init__()\n\n        blocks = []\n        out_features = first_out_features\n        for idx in range(num_blocks):\n            # if idx == num_blocks - 1:\n            #     out_features = num_classes\n\n            blocks.append(self._make_block(in_features, out_features, drop_rate))\n\n            in_features = out_features\n            out_features = out_features // 2\n        \n        self.model = nn.Sequential(*blocks)\n        self.loss_fn = loss_fn\n        self.fc_probs = nn.Linear(256, num_classes)\n        self.arcface = arcface\n\n    def _make_block(self, in_features, out_features, drop_rate):\n        return nn.Sequential(\n            nn.Linear(in_features, out_features),\n            nn.BatchNorm1d(out_features),\n            nn.ReLU(),\n            nn.Dropout(drop_rate),\n        )\n\n    def forward(self, x):\n        x = self.model(x)\n        if self.arcface:\n            arcface = self.loss_fn(x, y)\n            return self.fc_probs(x), arcface\n        else:\n            return self.fc_probs(x)\n        \nclass SoftmaxLoss(nn.Module):\n    def __init__(self, in_features, out_features):\n        \"\"\"\n        Regular softmax loss (1 fc layer without bias + CrossEntropyLoss)\n        Args:\n            out_features: The number of classes in your training dataset\n            in_features: The size of the embeddings that you pass into\n        \"\"\"\n        super().__init__()\n        self.out_features = out_features\n        self.in_features = in_features\n        \n        self.W = torch.nn.Parameter(torch.Tensor(out_features, in_features))\n        nn.init.xavier_normal_(self.W)\n        \n    def forward(self, embeddings, labels):\n        \"\"\"\n        Args:\n            embeddings: (None, in_features)\n            labels: (None,)\n        Returns:\n            loss: scalar\n        \"\"\"\n        logits = F.linear(embeddings, self.W)\n        return logits\n    \n    \nclass ArcMarginProduct(nn.Module):\n    def __init__(\n        self,\n        in_features,\n        out_features,\n        scale=30.0,\n        margin=0.50,\n        easy_margin=False,\n        ls_eps=0.0,\n    ):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.scale = scale\n        self.margin = margin\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(margin)\n        self.sin_m = math.sin(margin)\n        self.th = math.cos(math.pi - margin)\n        self.mm = math.sin(math.pi - margin) * margin\n\n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device=label.device)\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.scale\n\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:45.005882Z","iopub.execute_input":"2023-03-15T09:52:45.006555Z","iopub.status.idle":"2023-03-15T09:52:45.027897Z","shell.execute_reply.started":"2023-03-15T09:52:45.006515Z","shell.execute_reply":"2023-03-15T09:52:45.026624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\nclass ASLData(Dataset):\n    def __init__(self, datax, datay):\n        self.datax = datax\n        self.datay = datay\n        \n    def __getitem__(self, index):\n        return self.datax[index,:], self.datay[index]\n        \n    def __len__(self):\n        return len(self.datay)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:45.031427Z","iopub.execute_input":"2023-03-15T09:52:45.032381Z","iopub.status.idle":"2023-03-15T09:52:45.042374Z","shell.execute_reply.started":"2023-03-15T09:52:45.032344Z","shell.execute_reply":"2023-03-15T09:52:45.041303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Code for Feature Gen /Pre Process\nIt takes about 11 mins with multiprocessing, the current data is saved in a dataset. Run this code when you do your own processing.","metadata":{}},{"cell_type":"code","source":"# import multiprocessing as mp\n\n# def convert_row(row):\n#     x = load_relevant_data_subset(os.path.join(\"/kaggle/input/asl-signs\", row[1].path))\n#     x = feature_converter(torch.tensor(x)).cpu().numpy()\n#     return x, row[1].label\n\n# def convert_and_save_data():\n#     df = pd.read_csv(TRAIN_FILE)\n#     df['label'] = df['sign'].map(label_map)\n#     npdata = np.zeros((df.shape[0], 3258))\n#     nplabels = np.zeros(df.shape[0])\n#     with mp.Pool() as pool:\n#         results = pool.imap(convert_row, df.iterrows(), chunksize=250)\n#         for i, (x,y) in tqdm(enumerate(results), total=df.shape[0]):\n#             npdata[i,:] = x\n#             nplabels[i] = y\n    \n#     np.save(\"feature_data.npy\", npdata)\n#     np.save(\"feature_labels.npy\", nplabels)\n        \n# convert_and_save_data()","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:45.043959Z","iopub.execute_input":"2023-03-15T09:52:45.044650Z","iopub.status.idle":"2023-03-15T09:52:45.052671Z","shell.execute_reply.started":"2023-03-15T09:52:45.044549Z","shell.execute_reply":"2023-03-15T09:52:45.051629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# datax = np.load(\"/kaggle/input/gislr-feature-data/feature_data.npy\")\n# datay = np.load(\"/kaggle/input/gislr-feature-data/feature_labels.npy\")","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:45.054024Z","iopub.execute_input":"2023-03-15T09:52:45.055153Z","iopub.status.idle":"2023-03-15T09:52:45.063943Z","shell.execute_reply.started":"2023-03-15T09:52:45.055113Z","shell.execute_reply":"2023-03-15T09:52:45.062893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# EPOCHS = 40\n# BATCH_SIZE = 64\n\n# trainx, testx, trainy, testy = train_test_split(datax, datay, test_size=0.15, random_state=42)\n\n# train_data = ASLData(trainx, trainy)\n# valid_data = ASLData(testx, testy)\n\n# train_loader = DataLoader(train_data, batch_size=BATCH_SIZE, num_workers=4, shuffle=True)\n# val_loader = DataLoader(valid_data, batch_size=BATCH_SIZE, num_workers=4, shuffle=False)\n\n# model = ASLModel(0.2).cuda()\n# opt = torch.optim.Adam(model.parameters(), lr=0.005)\n# criterion = nn.CrossEntropyLoss()\n# sched = torch.optim.lr_scheduler.StepLR(opt, step_size=300, gamma=0.95)\n\n# for i in range(EPOCHS):\n#     model.train()\n    \n#     train_loss_sum = 0.\n#     train_correct = 0\n#     train_total = 0\n#     train_bar = train_loader\n#     for x,y in train_bar:\n#         x = torch.Tensor(x).float().cuda()\n#         y = torch.Tensor(y).long().cuda()  \n#         y_pred = model(x)\n        \n#         loss = criterion(y_pred, y)\n#         loss.backward()\n#         opt.step()\n#         opt.zero_grad()\n        \n#         train_loss_sum += loss.item()\n#         train_correct += np.sum((np.argmax(y_pred.detach().cpu().numpy(), axis=1) == y.cpu().numpy()))\n#         train_total += 1\n#         sched.step()\n        \n#     val_loss_sum = 0.\n#     val_correct = 0\n#     val_total = 0\n#     model.eval()\n#     for x,y in val_loader:\n#         x = torch.Tensor(x).float().cuda()\n#         y = torch.Tensor(y).long().cuda()\n        \n#         with torch.no_grad():\n#             y_pred = model(x)\n#             loss = criterion(y_pred, y)\n#             val_loss_sum += loss.item()\n#             val_correct += np.sum((np.argmax(y_pred.cpu().numpy(), axis=1) == y.cpu().numpy()))\n#             val_total += 1\n                              \n#     print(f\"Epoch:{i} > Train Loss: {(train_loss_sum/train_total):.04f}, Train Acc: {train_correct/len(train_data):0.04f}\")\n#     print(f\"Epoch:{i} > Val Loss: {(val_loss_sum/val_total):.04f}, Val Acc: {val_correct/len(valid_data):0.04f}\")\n#     print(\"=\"*50)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:45.065573Z","iopub.execute_input":"2023-03-15T09:52:45.065997Z","iopub.status.idle":"2023-03-15T09:52:45.075746Z","shell.execute_reply.started":"2023-03-15T09:52:45.065956Z","shell.execute_reply":"2023-03-15T09:52:45.074710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PATH = '/kaggle/input/baseline-asl/best.pth'\n# PATH = '/kaggle/input/best-base-nn-bn/best_base_nn_bn.pth'\n# PATH = '/kaggle/input/best-base-nn-bn-2/best_base_nn_bn_2.pth'\n# PATH = '/kaggle/input/best-base-nn-bn-3/best_base_nn_bn_3.pth'\n# PATH = '/kaggle/input/best-base-nn-bn-4/best_base_nn_bn_4.pth'\n# PATH = '/kaggle/input/best-base-nn-participant-id/best_base_nn_bn_participant_id.pth'\n# PATH = '/kaggle/input/best-base-nn-softmax/best_base_nn_softmax.pth'\n# PATH = '/kaggle/input/best-base-nn-softmax-2/best_base_nn_softmax_2.pth'\n# PATH = '/kaggle/input/best-base-nn-bn-arcface-2/best_base_nn_bn_arcface_2.pth'\n# PATH = '/kaggle/input/nn-bn-arc/nn_bn_arc.pth'\n# PATH = '/kaggle/input/nn-bn-arc-2/nn_bn_arc_2.pth'\nPATH = '/kaggle/input/nn-bn-arc-3/nn_bn_arc_3.pth'","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:45.080421Z","iopub.execute_input":"2023-03-15T09:52:45.081259Z","iopub.status.idle":"2023-03-15T09:52:45.088765Z","shell.execute_reply.started":"2023-03-15T09:52:45.081229Z","shell.execute_reply":"2023-03-15T09:52:45.087780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn = ArcMarginProduct(\n    in_features=256,\n    out_features=250,\n    scale=30.0,\n    margin=0.5,\n    easy_margin=False,\n    ls_eps=0.0\n)\nmodel = ASLLinearModel(in_features=5796,\n                       first_out_features=1024,\n                       num_classes=250,\n                       num_blocks=3,\n                       drop_rate=0.4,\n                       arcface=False,\n                      loss_fn=loss_fn).cuda()\ncheckpoint = torch.load(PATH)\nmodel.load_state_dict(checkpoint['model'])","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:45.090143Z","iopub.execute_input":"2023-03-15T09:52:45.091197Z","iopub.status.idle":"2023-03-15T09:52:50.785018Z","shell.execute_reply.started":"2023-03-15T09:52:45.091157Z","shell.execute_reply":"2023-03-15T09:52:50.783808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:50.786671Z","iopub.execute_input":"2023-03-15T09:52:50.787338Z","iopub.status.idle":"2023-03-15T09:52:50.997529Z","shell.execute_reply.started":"2023-03-15T09:52:50.787297Z","shell.execute_reply":"2023-03-15T09:52:50.996175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorflow Conversion","metadata":{}},{"cell_type":"code","source":"!pip install onnx-tf\n!pip install tflite-runtime","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:52:50.999800Z","iopub.execute_input":"2023-03-15T09:52:51.000320Z","iopub.status.idle":"2023-03-15T09:53:11.984363Z","shell.execute_reply.started":"2023-03-15T09:52:51.000214Z","shell.execute_reply":"2023-03-15T09:53:11.983109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_input = torch.rand((50, 543, 3))\nonnx_feat_gen_path = 'feature_gen.onnx'\n\nfeature_converter.eval()\n\ntorch.onnx.export(\n    feature_converter,                  # PyTorch Model\n    sample_input,                    # Input tensor\n    onnx_feat_gen_path,        # Output file (eg. 'output_model.onnx')\n    opset_version=12,       # Operator support version\n    input_names=['input'],   # Input tensor name (arbitary)\n    output_names=['output'], # Output tensor name (arbitary)\n    dynamic_axes={\n        'input' : {0: 'input'}\n    }\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:53:11.986820Z","iopub.execute_input":"2023-03-15T09:53:11.987479Z","iopub.status.idle":"2023-03-15T09:53:12.162897Z","shell.execute_reply.started":"2023-03-15T09:53:11.987435Z","shell.execute_reply":"2023-03-15T09:53:12.161670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_input = torch.rand((32, 5796)).cuda()\nonnx_model_path = 'asl_model.onnx'\n\nmodel.eval()\n\ntorch.onnx.export(\n    model,                  # PyTorch Model\n    sample_input,                    # Input tensor\n    onnx_model_path,        # Output file (eg. 'output_model.onnx')\n    opset_version=12,       # Operator support version\n    input_names=['input'],   # Input tensor name (arbitary)\n    output_names=['output'], # Output tensor name (arbitary)\n    dynamic_axes={\n        'input' : {0: 'input'}\n    }\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:53:12.164484Z","iopub.execute_input":"2023-03-15T09:53:12.165435Z","iopub.status.idle":"2023-03-15T09:53:14.084098Z","shell.execute_reply.started":"2023-03-15T09:53:12.165394Z","shell.execute_reply":"2023-03-15T09:53:14.083032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import onnx\nfrom onnx_tf.backend import prepare\n\n\ntf_feat_gen_path = '/kaggle/working/tf_feat_gen'\nonnx_feat_gen = onnx.load(onnx_feat_gen_path)\ntf_rep = prepare(onnx_feat_gen)\ntf_rep.export_graph(tf_feat_gen_path)\n\n\ntf_model_path = '/kaggle/working/tf_model'\nonnx_model = onnx.load(onnx_model_path)\ntf_rep = prepare(onnx_model)\ntf_rep.export_graph(tf_model_path)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:53:14.085796Z","iopub.execute_input":"2023-03-15T09:53:14.086177Z","iopub.status.idle":"2023-03-15T09:53:25.453394Z","shell.execute_reply.started":"2023-03-15T09:53:14.086139Z","shell.execute_reply":"2023-03-15T09:53:25.452312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Inference Model in Tensorflow\nBoth of the converted models will be used here one after another.","metadata":{}},{"cell_type":"code","source":"# import tensorflow as tf\n\n# class ASLInferModel(tf.Module):\n#     def __init__(self):\n#         super(ASLInferModel, self).__init__()\n#         self.feature_gen = tf.saved_model.load(tf_feat_gen_path)\n#         self.model = tf.saved_model.load(tf_model_path)\n#         self.feature_gen.trainable = False\n#         self.model.trainable = False\n    \n#     @tf.function(input_signature=[\n#       tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='inputs')\n#     ])\n#     def call(self, input):\n#         output_tensors = {}\n#         features = self.feature_gen(**{'input': input})['output']\n#         output_tensors['outputs'] = self.model(**{'input': tf.expand_dims(features, 0)})['output'][0,:]\n#         return output_tensors\n    \n    \n# mytfmodel = ASLInferModel()\n# tf.saved_model.save(mytfmodel, '/kaggle/working/tf_infer_model', signatures={'serving_default': mytfmodel.call})\n\nimport tensorflow as tf\n\nclass ASLInferModel(tf.Module):\n    def __init__(self):\n        super(ASLInferModel, self).__init__()\n        self.feature_gen = FeatureGenTF()\n        self.model = tf.saved_model.load(tf_model_path)\n        self.feature_gen.trainable = False\n        self.model.trainable = False\n    \n    @tf.function(input_signature=[\n      tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='inputs')\n    ])\n    def call(self, input):\n        output_tensors = {}\n        features = self.feature_gen(tf.cast(input, dtype=tf.float32))\n        output_tensors['outputs'] = self.model(**{'input': features})['output'][0,:]\n        return output_tensors\n    \n    \nmytfmodel = ASLInferModel()\ntf.saved_model.save(mytfmodel, '/kaggle/working/tf_infer_model', signatures={'serving_default': mytfmodel.call})","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:53:25.455281Z","iopub.execute_input":"2023-03-15T09:53:25.455689Z","iopub.status.idle":"2023-03-15T09:53:26.773101Z","shell.execute_reply.started":"2023-03-15T09:53:25.455652Z","shell.execute_reply":"2023-03-15T09:53:26.772040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# Convert the model\n\ntf_infer_model_path = '/kaggle/working/tf_infer_model'\nconverter = tf.lite.TFLiteConverter.from_saved_model(tf_infer_model_path)\ntflite_model = converter.convert()\n\ntflite_model_path = 'model.tflite'\n\n# Save the model\nwith open(tflite_model_path, 'wb') as f:\n    f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:53:26.775356Z","iopub.execute_input":"2023-03-15T09:53:26.779600Z","iopub.status.idle":"2023-03-15T09:53:28.793430Z","shell.execute_reply.started":"2023-03-15T09:53:26.779534Z","shell.execute_reply":"2023-03-15T09:53:28.792347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\npq_path = \"/kaggle/input/asl-signs/train_landmark_files/53618/1001379621.parquet\"\n\nimport tflite_runtime.interpreter as tflite\ninterpreter = tflite.Interpreter(tflite_model_path)\ninterpreter.allocate_tensors()\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\n# if REQUIRED_SIGNATURE not in found_signatures:\n#     raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=load_relevant_data_subset(pq_path))\nsign = np.argmax(output[\"outputs\"])\n\nprint(sign, output[\"outputs\"].shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:53:28.798397Z","iopub.execute_input":"2023-03-15T09:53:28.800780Z","iopub.status.idle":"2023-03-15T09:53:28.910022Z","shell.execute_reply.started":"2023-03-15T09:53:28.800739Z","shell.execute_reply":"2023-03-15T09:53:28.908945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip $tflite_model_path","metadata":{"execution":{"iopub.status.busy":"2023-03-15T09:53:28.911762Z","iopub.execute_input":"2023-03-15T09:53:28.912494Z","iopub.status.idle":"2023-03-15T09:53:31.416357Z","shell.execute_reply.started":"2023-03-15T09:53:28.912453Z","shell.execute_reply":"2023-03-15T09:53:31.415096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}