{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### This code is a Google - Isolated Sign Language Recognition 4th place inferrence code.   \n\n+ Reads pre-trained models and creates tflite.\n+ Note: The model in the final submission is an ensemble of six. \n+ However, due to many cases of timeouts, this code restricted to 5 models.\n\n","metadata":{}},{"cell_type":"code","source":"!pip install tflite-runtime\n!pip install onnx_tf","metadata":{"execution":{"iopub.status.busy":"2023-05-05T14:58:29.653684Z","iopub.execute_input":"2023-05-05T14:58:29.654684Z","iopub.status.idle":"2023-05-05T14:58:55.432571Z","shell.execute_reply.started":"2023-05-05T14:58:29.654628Z","shell.execute_reply":"2023-05-05T14:58:55.431355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport tensorflow as tf\nimport onnx_tf\nimport onnx\nimport tflite_runtime\nimport tflite_runtime.interpreter\n\nfrom torchvision.ops import StochasticDepth\nimport math\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\nimport sys\nimport torch\nimport torch.nn as nn\n\nimport warnings\nwarnings.filterwarnings(action='ignore')\n","metadata":{"execution":{"iopub.status.busy":"2023-05-05T14:58:55.434999Z","iopub.execute_input":"2023-05-05T14:58:55.435447Z","iopub.status.idle":"2023-05-05T14:59:08.488940Z","shell.execute_reply.started":"2023-05-05T14:58:55.435399Z","shell.execute_reply":"2023-05-05T14:59:08.487397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass ArcMarginProduct(nn.Module):\n    def __init__(self, in_features, out_features, s=30.0, m=0.5, easy_margin=False):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.weight = torch.nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, x, label=None):\n        cosine = torch.nn.functional.linear(torch.nn.functional.normalize(\n            x), torch.nn.functional.normalize(self.weight)).float()\n        sine = torch.sqrt((1.0 - torch.pow(cosine, 2)).clamp(0, 1))\n        phi = cosine * self.cos_m - sine * self.sin_m\n\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n\n        if label is None:\n            output = cosine\n        else:\n            one_hot = torch.zeros(cosine.size(), device='cuda')\n            one_hot.scatter_(1, label.cuda().view(-1, 1).long(), 1)\n            # you can use torch.where if your torch.__version__ is 0.4\n            output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n\n        return output\n\n\nclass LRUnit1D(nn.Module):\n    def __init__(self, in_dim, stochastic_depth_prob=0.0, actf=torch.nn.ReLU()):\n        super(LRUnit1D, self).__init__()\n        self.layer0 = nn.Linear(in_dim, in_dim, bias=False)\n        self.bn0 = nn.BatchNorm1d(in_dim)\n        self.layer1 = nn.Linear(in_dim, in_dim, bias=False)\n        self.bn1 = nn.BatchNorm1d(in_dim)\n        self.actf = actf\n        self.stochastic_depth = StochasticDepth(stochastic_depth_prob, \"row\")\n\n    def forward(self, x):\n        h = self.actf(self.bn0(self.layer0(x)))\n        h = self.bn1(self.layer1(h))\n        h = self.stochastic_depth(h)\n        return x + h\n\n\nclass RSUnit1D(nn.Module):\n    def __init__(self, in_dim, kernel_size=3, padding=1,\n                 padding_mode='zeros', actf=torch.nn.ReLU()):\n        super(RSUnit1D, self).__init__()\n        self.layer0 = nn.Conv1d(in_dim, in_dim, kernel_size=kernel_size, padding=padding, padding_mode=padding_mode, bias=False)\n        self.bn0 = nn.BatchNorm1d(in_dim)\n        self.layer1 = nn.Conv1d(in_dim, in_dim, kernel_size=kernel_size, padding=padding, padding_mode=padding_mode, bias=False)\n        self.bn1 = nn.BatchNorm1d(in_dim)\n        self.actf = actf\n\n    def forward(self, x):\n        h = self.actf(self.bn0(self.layer0(x)))\n        h = self.bn1(self.layer1(h))\n        return x + h\n\n\n\nclass BackboneVariableLen(nn.Module):\n    def __init__(self,\n                 in_dim=42,\n                 dim1=128,\n                 dim2=512,\n                 kernel_size=3, padding=1,\n                 negative_slope=0.1,\n                 ):\n        super(BackboneVariableLen, self).__init__()\n        self.actf = torch.nn.LeakyReLU(negative_slope=negative_slope)\n        self.bn0 = nn.BatchNorm1d(in_dim)\n        self.conv1 = nn.Conv1d(in_dim, dim1, kernel_size=kernel_size, padding=padding, bias=True)\n        self.conv2 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv3 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv4 = RSUnit1D(dim1, kernel_size=1, padding=0, actf=self.actf)\n        self.conv5 = RSUnit1D(dim1, kernel_size=1, padding=0, actf=self.actf)\n        self.conv6 = RSUnit1D(dim1, kernel_size=1, padding=0, actf=self.actf)\n        self.conv6a = nn.Conv1d(dim1, dim2, kernel_size=1, padding=0, bias=True)\n\n        self.flatten = torch.nn.Flatten(1)\n        self.g_pool = torch.nn.AdaptiveMaxPool1d(1, return_indices=False)\n        self.pool = torch.nn.MaxPool1d(2)\n\n    def forward(self, h):\n        h = self.bn0(h)\n        h = self.conv1(h)\n        h = self.conv2(h)\n        h = self.conv3(h)\n        h = self.conv4(h)\n        h = self.conv5(h)\n        h = self.conv6(h)\n        h = self.conv6a(h)\n        print(h.shape)\n\n        # Remove padding to match during trainning\n        h = h[:, :, :-5]\n\n        print(h.shape)\n        h = self.g_pool(h)\n        h = self.flatten(h)\n        return h\n\n\nclass ModelVariableLen(nn.Module):\n    def __init__(self,\n                 in_dim_H=42,\n                 in_dim_L=80,\n                 dim1_H=128,\n                 dim1_L=64,\n                 dim2=512,\n                 n_class=250,\n                 kernel_size=3, padding=1,\n                 arc_m=0.5,\n                 arc_s=15,\n                 easy_margin=False,\n                 stochastic_depth_prob=0.0,\n                 negative_slope=0.1,\n                 n_recyile=1\n                 ):\n        super(ModelVariableLen, self).__init__()\n        self.actf = torch.nn.LeakyReLU(negative_slope=negative_slope)\n        self.hand_module = BackboneVariableLen(in_dim=in_dim_H, dim1=dim1_H, dim2=dim2, negative_slope=negative_slope)\n        self.other_module = BackboneVariableLen(in_dim=in_dim_L, dim1=dim1_L, dim2=dim2, negative_slope=negative_slope)\n\n        self.line1 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.line2 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.line3 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.end = ArcMarginProduct(dim2, n_class, easy_margin=easy_margin, s=arc_s, m=arc_m)\n        self.flatten = torch.nn.Flatten(1)\n        self.g_pool = torch.nn.AdaptiveMaxPool1d(1, return_indices=False)\n        self.pool = torch.nn.MaxPool1d(2)\n\n        self.n_recyile = n_recyile\n\n    def forward(self, hand, other):\n        hand = self.hand_module(hand)\n        other = self.other_module(other)\n        h = hand + other\n        for n in range(self.n_recyile):\n            h = self.line1(h)\n        for n in range(self.n_recyile):\n            h = self.line2(h)\n        for n in range(self.n_recyile):\n            h = self.line3(h)\n        h = self.end(h)\n        return h\n\n# %%\n\n\nclass BackboneFixedLen(nn.Module):\n    def __init__(self,\n                 in_dim=42,\n                 dim1=128,\n                 dim2=512,\n                 kernel_size=3, padding=1,\n                 negative_slope=0.1,\n                 ):\n        super(BackboneFixedLen, self).__init__()\n        self.actf = torch.nn.LeakyReLU(negative_slope=negative_slope)\n        self.bn0 = nn.BatchNorm1d(in_dim)\n        self.conv1 = nn.Conv1d(in_dim, dim1, kernel_size=3, padding=1, bias=True)\n        self.conv2 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv3 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv4 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv5 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv6 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv6a = nn.Conv1d(dim1, dim2, kernel_size=1, padding=0, bias=True)\n\n        self.flatten = torch.nn.Flatten(1)\n        self.g_pool = torch.nn.AdaptiveMaxPool1d(1, return_indices=False)\n        self.pool = torch.nn.MaxPool1d(2)\n\n    def forward(self, x):\n        h = self.bn0(x)\n        h = self.conv1(h)\n        h = self.conv2(h)\n        h = self.pool(h)\n        h = self.conv3(h)\n        h = self.pool(h)\n        h = self.conv4(h)\n        h = self.pool(h)\n        h = self.conv5(h)\n        h = self.conv6(h)\n        h = self.conv6a(h)\n        h = self.g_pool(h)\n        h = self.flatten(h)\n        return h\n\n\nclass ModelFixedLen(nn.Module):\n    def __init__(self,\n                 in_dim_H=42,\n                 in_dim_L=80,\n                 dim1_H=128,\n                 dim1_L=64,\n                 dim2=512,\n                 n_class=250,\n                 kernel_size=3, padding=1,\n                 arc_m=0.5,\n                 arc_s=15,\n                 easy_margin=False,\n                 stochastic_depth_prob=0.0,\n                 negative_slope=0.1,\n                 n_recyile=1\n                 ):\n        super(ModelFixedLen, self).__init__()\n        self.actf = torch.nn.LeakyReLU(negative_slope=negative_slope)\n        self.hand_module = BackboneFixedLen(in_dim=in_dim_H, dim1=dim1_H, dim2=dim2, negative_slope=negative_slope)\n        self.other_module = BackboneFixedLen(in_dim=in_dim_L, dim1=dim1_L, dim2=dim2, negative_slope=negative_slope)\n\n        self.line1 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.line2 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.line3 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.end = ArcMarginProduct(dim2, n_class, easy_margin=easy_margin, s=arc_s, m=arc_m)\n        self.flatten = torch.nn.Flatten(1)\n        self.g_pool = torch.nn.AdaptiveMaxPool1d(1, return_indices=False)\n        self.pool = torch.nn.MaxPool1d(2)\n\n        self.n_recyile = n_recyile\n\n    def forward(self, hand, other):\n        hand = self.hand_module(hand)\n        other = self.other_module(other)\n        h = hand + other\n        for n in range(self.n_recyile):\n            h = self.line1(h)\n        for n in range(self.n_recyile):\n            h = self.line2(h)\n        for n in range(self.n_recyile):\n            h = self.line3(h)\n        h = self.end(h)\n        return h\n","metadata":{"execution":{"iopub.status.busy":"2023-05-05T14:59:08.491346Z","iopub.execute_input":"2023-05-05T14:59:08.492138Z","iopub.status.idle":"2023-05-05T14:59:08.810600Z","shell.execute_reply.started":"2023-05-05T14:59:08.492093Z","shell.execute_reply":"2023-05-05T14:59:08.809266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n@torch.jit.script\ndef normalize_by_midwayBetweenEyes(x):\n\n    midwayBetweenEyes = x[:, 168]\n    mask = ~torch.isnan(midwayBetweenEyes[:, 0])\n    masked = midwayBetweenEyes[mask]  # .mean(keepdims=True)\n    m = torch.mean(masked, dim=0)\n    if torch.any(torch.isnan(m)): \n        return x\n    else:\n        m = torch.unsqueeze(m, 0)\n        m = torch.unsqueeze(m, 0)\n    return x - m\n\n\n@torch.jit.script\ndef apply_hand_mask(x):\n\n    hand_mask = ~torch.isnan(x[:, 0, 0])\n    m = x[~torch.isnan(x)].mean(0)  # noramlisation to common maen\n    m = torch.unsqueeze(m, 0)\n    m = torch.unsqueeze(m, 0)\n    x = x - m\n    s = x[~torch.isnan(x)].std(0)\n    s = torch.unsqueeze(s, 0)\n    s = torch.unsqueeze(s, 0)\n    x = x / s\n    x = torch.where(torch.isnan(x), torch.tensor(0.0, dtype=torch.float32), x)\n\n    if hand_mask.sum() > 0:\n        hand = x[hand_mask, :21]\n    else:\n        hand = x[:, :21]\n    other = x[:, 21:]\n    return hand, other\n\n\n@torch.jit.script\ndef apply_autoflip(xy):\n\n    left_start_index = 468\n    right_start_index = 522\n    length = xy[:, 0, 0].shape[0]\n    l_num = length - torch.isnan(xy[:, left_start_index, 0]).sum()\n    r_num = length - torch.isnan(xy[:, right_start_index, 0]).sum()\n\n    # 左手主体の場合\n    if r_num < l_num:\n        xy = torch.stack((-xy[:, :, 0], xy[:, :, 1]), dim=2)\n\n        hand_indexes = torch.tensor((468, 469, 470, 471, 472, 473, 474, 475, 476, 477,\n                                     478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488))\n\n        lipsUpperOuter_indexes = torch.tensor((291, 409, 270, 269, 267, 0, 37, 39, 40, 185, 61))\n        lipsLowerOuter_indexes = torch.tensor((375, 321, 405, 314, 17, 84, 181, 91, 46))\n        lipsUpperInner_indexes = torch.tensor((308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78))\n        lipsLowerInner_indexes = torch.tensor((324, 318, 402, 317, 14, 87, 178, 88, 95))\n\n    else:\n\n        hand_indexes = torch.tensor((522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534,\n                                     535, 536, 537, 538, 539, 540, 541, 542))\n\n        lipsUpperOuter_indexes = torch.tensor((61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291))\n        lipsLowerOuter_indexes = torch.tensor((46, 91, 181, 84, 17, 314, 405, 321, 375))\n        lipsUpperInner_indexes = torch.tensor((78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308))\n        lipsLowerInner_indexes = torch.tensor((95, 88, 178, 87, 14, 317, 402, 318, 324))\n\n    # l_hand = xy[:, l_hand_indexes]\n    hand = xy[:, hand_indexes]\n\n    lipsUpperOuter = xy[:, lipsUpperOuter_indexes]\n    lipsLowerOuter = xy[:, lipsLowerOuter_indexes]\n    lipsUpperInner = xy[:, lipsUpperInner_indexes]\n    lipsLowerInner = xy[:, lipsLowerInner_indexes]\n\n    x = torch.concat((hand,\n                      lipsUpperOuter,\n                      lipsLowerOuter,\n                      lipsUpperInner,\n                      lipsLowerInner), dim=1)\n    print(\"feature\", x.shape)\n    return x\n\n\nclass AslData_tf(nn.Module):\n\n    def __init__(self, seq_length):\n        super(AslData_tf, self).__init__()\n        self.seq_length = seq_length\n\n    def forward(self, data):\n        # TxDx3\n        # print(x.shape)\n\n        xy = data[:, :, :2]  # TxDx2\n        xy = normalize_by_midwayBetweenEyes(xy)\n\n        # x = x[:, self.indexes]  # Txdx2\n        xy = apply_autoflip(xy)\n\n        hand, other = apply_hand_mask(xy)\n\n        hand = torch.reshape(hand, (hand.shape[0], -1))  # T , 2*d\n        hand = torch.permute(hand, (1, 0))  # 2*d, T\n        print(hand.shape)\n        other = torch.reshape(other, (other.shape[0], -1))  # T , 2*d\n        other = torch.permute(other, (1, 0))  # 2*d, T\n        print(other.shape)\n\n        hand = torch.unsqueeze(hand, 0)\n        other = torch.unsqueeze(other, 0)\n\n        hand_fixed_len = torch.unsqueeze(hand, 0)\n        hand_fixed_len = torch.nn.functional.interpolate(hand_fixed_len, size=(hand_fixed_len.shape[2], self.seq_length))\n        hand_fixed_len = hand_fixed_len[0]\n\n        other_fixed_len = torch.unsqueeze(other, 0)\n        other_fixed_len = torch.nn.functional.interpolate(other_fixed_len, size=(other_fixed_len.shape[2], self.seq_length))\n        other_fixed_len = other_fixed_len[0]\n\n        # Since the batch size is 1 during inference,\n        # there is no need for padding, but only 5 padding is used to match during learning.\n        hand_variable_len = torch.nn.ConstantPad1d((0, 5), 0)(hand)\n        other_variable_len = torch.nn.ConstantPad1d((0, 5), 0)(other)\n\n        print(\"hand_fixed_len\", hand_fixed_len.shape)\n        print(\"other_fixed_len\", other_fixed_len.shape)\n        print(\"hand_variable_len\", hand_variable_len.shape)\n        print(\"other_variable_len\", other_variable_len.shape)\n        return hand_fixed_len, other_fixed_len, hand_variable_len, other_variable_len\n","metadata":{"execution":{"iopub.status.busy":"2023-05-05T14:59:08.812589Z","iopub.execute_input":"2023-05-05T14:59:08.813353Z","iopub.status.idle":"2023-05-05T14:59:08.891832Z","shell.execute_reply.started":"2023-05-05T14:59:08.813299Z","shell.execute_reply":"2023-05-05T14:59:08.890284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass SubmitModel(torch.nn.Module):\n    def __init__(self,\n                 ):\n        super(SubmitModel, self).__init__()\n        self.prepare = AslData_tf(SEQ_LEN)\n        self.fix_len_models = nn.ModuleList()\n        self.var_len_models = nn.ModuleList()\n\n    def __call__(self, x):\n        hand_fixed_len, other_fixed_len, hand_variable_len, other_variable_len = self.prepare(x)\n        print(len(self.fix_len_models), len(self.var_len_models))\n\n        ys = []\n        for i in range(len(self.fix_len_models)):\n            ys.append((self.fix_len_models[i](hand_fixed_len, other_fixed_len)))\n        for i in range(len(self.var_len_models)):\n            ys.append((self.var_len_models[i](hand_variable_len, other_variable_len)))\n        y = torch.cat(ys, dim=0).mean(dim=0)\n        return y\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-05T14:59:08.895406Z","iopub.execute_input":"2023-05-05T14:59:08.895830Z","iopub.status.idle":"2023-05-05T14:59:08.911739Z","shell.execute_reply.started":"2023-05-05T14:59:08.895775Z","shell.execute_reply":"2023-05-05T14:59:08.910395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\nDATA_PATH = \"../data/\"\nINPUT_PATH = \"../input/asl-signs/\"\nLANDMARK_FILES_DIR = os.path.join(INPUT_PATH, \"train_landmark_files\")\nTRAIN_FILE = os.path.join(INPUT_PATH, \"train.csv\")\nJSON_FILE = os.path.join(INPUT_PATH, \"sign_to_prediction_index_map.json\")\n\nn_class = 250\nROWS_PER_FRAME = 543\n\nSEQ_LEN = 96\narc_s = 47\narc_m = 0.0\neasy_margin = True\nnegative_slope = 0.1\n\nin_dim_H = 42\nin_dim_L = 80\ndim1_H = 128\ndim1_L = 64\ndim2 = 512\nopset_version=12\n\nresult_path = \"/kaggle/input/asl-4th-place-model/ASL_4th_place_model/\"\n","metadata":{"execution":{"iopub.status.busy":"2023-05-05T14:59:08.913736Z","iopub.execute_input":"2023-05-05T14:59:08.914740Z","iopub.status.idle":"2023-05-05T14:59:08.926491Z","shell.execute_reply.started":"2023-05-05T14:59:08.914685Z","shell.execute_reply":"2023-05-05T14:59:08.925360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = SubmitModel()\n\nmodel_path = os.path.join(result_path, \"ModelFixedLen_deep_seed0_cleanlab_False\", \"swa_300.pt\")\ntmp_model = ModelFixedLen(in_dim_H=in_dim_H,\n                          in_dim_L=in_dim_L,\n                          dim1_H=dim1_H,\n                          dim1_L=dim1_L,\n                          dim2=dim2,\n                          easy_margin=easy_margin,\n                          arc_s=arc_s, arc_m=arc_m,\n                          stochastic_depth_prob=0.5,\n                          negative_slope=negative_slope,\n                          n_recyile=3\n                          )\nswa_model = torch.optim.swa_utils.AveragedModel(tmp_model)\nswa_model.module.eval()\nswa_model.load_state_dict(torch.load(model_path, map_location=\"cpu\"))\nmodel.fix_len_models.append(swa_model.module)\n\"\"\"\nmodel_path = os.path.join(result_path, \"ModelFixedLen_deep_seed5_cleanlab_True\", \"swa_300.pt\")\ntmp_model = ModelFixedLen(in_dim_H=in_dim_H,\n                          in_dim_L=in_dim_L,\n                          dim1_H=dim1_H,\n                          dim1_L=dim1_L,\n                          dim2=dim2,\n                          easy_margin=easy_margin,\n                          arc_s=arc_s, arc_m=arc_m,\n                          stochastic_depth_prob=0.5,\n                          negative_slope=negative_slope,\n                          n_recyile=3\n                          )\nswa_model = torch.optim.swa_utils.AveragedModel(tmp_model)\nswa_model.module.eval()\nswa_model.load_state_dict(torch.load(model_path, map_location=\"cpu\"))\nmodel.fix_len_models.append(swa_model.module)\n\"\"\"\n# %%\nmodel_path = os.path.join(result_path, \"ModelFixedLen_deep_seed6_cleanlab_True\", \"swa_300.pt\")\ntmp_model = ModelFixedLen(in_dim_H=in_dim_H,\n                          in_dim_L=in_dim_L,\n                          dim1_H=dim1_H,\n                          dim1_L=dim1_L,\n                          dim2=dim2,\n                          easy_margin=easy_margin,\n                          arc_s=arc_s, arc_m=arc_m,\n                          stochastic_depth_prob=0.5,\n                          negative_slope=negative_slope,\n                          n_recyile=3\n                          )\nswa_model = torch.optim.swa_utils.AveragedModel(tmp_model)\nswa_model.module.eval()\nswa_model.load_state_dict(torch.load(model_path, map_location=\"cpu\"))\nmodel.fix_len_models.append(swa_model.module)\n# %%\nmodel_path = os.path.join(result_path, \"ModelFixedLen_shallow_seed5_cleanlab_False\", \"swa_300.pt\")\ntmp_model = ModelFixedLen(in_dim_H=in_dim_H,\n                          in_dim_L=in_dim_L,\n                          dim1_H=dim1_H,\n                          dim1_L=dim1_L,\n                          dim2=dim2,\n                          easy_margin=easy_margin,\n                          arc_s=arc_s, arc_m=arc_m,\n                          stochastic_depth_prob=0.1,\n                          negative_slope=negative_slope,\n                          n_recyile=1\n                          )\nswa_model = torch.optim.swa_utils.AveragedModel(tmp_model)\nswa_model.module.eval()\nswa_model.load_state_dict(torch.load(model_path, map_location=\"cpu\"))\nmodel.fix_len_models.append(swa_model.module)\n\n# %%\nmodel_path = os.path.join(result_path, \"ModelVariableLen_deep_seed35_cleanlab_True\", \"swa_300.pt\")\ntmp_model = ModelVariableLen(in_dim_H=in_dim_H, in_dim_L=in_dim_L,\n                             dim1_H=dim1_H,\n                             dim1_L=dim1_L,\n                             dim2=dim2,\n                             easy_margin=easy_margin,\n                             arc_s=arc_s, arc_m=arc_m,\n                             stochastic_depth_prob=0.5,\n                             negative_slope=negative_slope,\n                             n_recyile=3\n                             )\nswa_model = torch.optim.swa_utils.AveragedModel(tmp_model)\nswa_model.module.eval()\nswa_model.load_state_dict(torch.load(model_path, map_location=\"cpu\"))\nmodel.var_len_models.append(swa_model.module)\n\n# %%\nmodel_path = os.path.join(result_path, \"ModelVariableLen_shallow_seed150_cleanlab_False\", \"swa_300.pt\")\ntmp_model = ModelVariableLen(in_dim_H=in_dim_H, in_dim_L=in_dim_L,\n                             dim1_H=dim1_H,\n                             dim1_L=dim1_L,\n                             dim2=dim2,\n                             easy_margin=easy_margin,\n                             arc_s=arc_s, arc_m=arc_m,\n                             stochastic_depth_prob=0.1,\n                             negative_slope=negative_slope,\n                             n_recyile=1\n                             )\nswa_model = torch.optim.swa_utils.AveragedModel(tmp_model)\nswa_model.module.eval()\nswa_model.load_state_dict(torch.load(model_path, map_location=\"cpu\"))\nmodel.var_len_models.append(swa_model.module)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-05T14:59:08.928469Z","iopub.execute_input":"2023-05-05T14:59:08.929318Z","iopub.status.idle":"2023-05-05T14:59:10.316392Z","shell.execute_reply.started":"2023-05-05T14:59:08.929261Z","shell.execute_reply":"2023-05-05T14:59:10.315065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(TRAIN_FILE)\nlabel_map = json.load(open(JSON_FILE, \"r\"))\ndf['label'] = df['sign'].map(label_map)\nrow = df.loc[0]\n\nsample_input = load_relevant_data_subset(os.path.join(INPUT_PATH, row.path))\nsample_input = torch.tensor(sample_input)\na = model(sample_input)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-05T14:59:10.318465Z","iopub.execute_input":"2023-05-05T14:59:10.318937Z","iopub.status.idle":"2023-05-05T14:59:10.958133Z","shell.execute_reply.started":"2023-05-05T14:59:10.318886Z","shell.execute_reply":"2023-05-05T14:59:10.956922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_path = \"/kaggle/working/\"\n\nos.makedirs(models_path, exist_ok=True)\nonnx_feat_gen_path = os.path.join(models_path, 'model.onnx')\ntf_feat_gen_path = os.path.join(models_path, 'tf_model')\ntflite_model_path = os.path.join(models_path, 'model.tflite')\n\n\nprint(sample_input.shape)\n\ntorch.onnx.export(\n    model,                  # PyTorch Model\n    sample_input,                    # Input tensor\n    onnx_feat_gen_path,        # Output file (eg. 'output_model.onnx')\n    opset_version=opset_version,       # Operator support version\n    input_names=['inputs'],   # Input tensor name (arbitary)\n    output_names=['outputs'],  # Output tensor name (arbitary)\n    dynamic_axes={\n        'inputs': {0: 'inputs',\n                   1: 'length'},\n        'outputs': {0: 'inputs'},\n    }\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T14:59:10.960894Z","iopub.execute_input":"2023-05-05T14:59:10.962548Z","iopub.status.idle":"2023-05-05T14:59:18.057885Z","shell.execute_reply.started":"2023-05-05T14:59:10.962484Z","shell.execute_reply":"2023-05-05T14:59:18.056842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"converting from onnx to tensorflow...\")\nonnx_feat_gen = onnx.load(onnx_feat_gen_path)\ntf_rep = onnx_tf.backend.prepare(onnx_feat_gen)\ntf_rep.export_graph(tf_feat_gen_path)\ndel tf_rep\n# %%\nprint(\"converting from tensorflow to tflite\")\nconverter = tf.lite.TFLiteConverter.from_saved_model(tf_feat_gen_path)\nconverter.optimizations = [tf.lite.Optimize.DEFAULT]\nconverter.target_spec.supported_types = [tf.float16]\ntflite_model = converter.convert()\n# Save the model\nwith open(tflite_model_path, 'wb') as f:\n    f.write(tflite_model)\nprint(\"converting finish!!\")","metadata":{"execution":{"iopub.status.busy":"2023-05-05T14:59:18.060639Z","iopub.execute_input":"2023-05-05T14:59:18.061385Z","iopub.status.idle":"2023-05-05T15:02:11.074567Z","shell.execute_reply.started":"2023-05-05T14:59:18.061342Z","shell.execute_reply":"2023-05-05T15:02:11.073363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tflite_runtime.interpreter.Interpreter(tflite_model_path)\ninterpreter.allocate_tensors()\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\n#%%\nif False:\n    prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n    ok = []\n    for i in tqdm(range(len(df))):\n        row = df.loc[i]\n        a = load_relevant_data_subset(os.path.join(INPUT_PATH, row.path))\n        output = prediction_fn(inputs=a)\n        z = output[\"outputs\"]\n        sign = np.argmax(z)\n        ok.append(row.label == sign)\n        if len(ok) % 1000 == 0:\n            print(np.mean(ok))\n    print(np.mean(ok))\n    # %%\n","metadata":{"execution":{"iopub.status.busy":"2023-05-05T15:02:39.130088Z","iopub.execute_input":"2023-05-05T15:02:39.130548Z","iopub.status.idle":"2023-05-05T15:02:47.633629Z","shell.execute_reply.started":"2023-05-05T15:02:39.130505Z","shell.execute_reply":"2023-05-05T15:02:47.631785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip $tflite_model_path","metadata":{"execution":{"iopub.status.busy":"2023-05-05T15:02:11.093425Z","iopub.execute_input":"2023-05-05T15:02:11.094207Z","iopub.status.idle":"2023-05-05T15:02:13.559643Z","shell.execute_reply.started":"2023-05-05T15:02:11.094153Z","shell.execute_reply":"2023-05-05T15:02:13.558076Z"},"trusted":true},"execution_count":null,"outputs":[]}]}