{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import copy\nimport torch\nimport math\nimport numpy as np\nimport torch.nn as nn\nfrom typing import Optional, Any\nimport torch.nn.functional as F\nfrom torch import Tensor\n\n\ndef _get_clones(mod, n):\n    return nn.ModuleList([copy.deepcopy(mod) for _ in range(n)])\n\ndef data_pre1(x):\n    \"取出54个点\"\n    col0 = [489,490,491,494,496,497,500,501,502,503,504,505]\n    col1 = [i+468 for i in range(21)]   # left_hand\n    col2 = [j+522 for j in range(21)]   # right_hand\n    x_pose = x[:,col0,:2]\n    x_left = x[:,col1,:2]\n    x_right = x[:,col2,:2]\n    \n    x_left = x_left - x_left[~torch.isnan(x_left)].mean(0,keepdim=True)\n    x_left = x_left / x_left[~torch.isnan(x_left)].std(0,keepdim=True)\n    \n    x_right = x_right - x_right[~torch.isnan(x_right)].mean(0,keepdim=True)\n    x_right = x_right / x_right[~torch.isnan(x_right)].std(0,keepdim=True)\n    \n    x_pose = x_pose - x_pose[~torch.isnan(x_pose)].mean(0,keepdim=True)\n    x_pose = x_pose / x_pose[~torch.isnan(x_pose)].std(0,keepdim=True)\n    \n    x_out = torch.cat([x_pose,x_left,x_right],dim=1)\n    x_out = torch.where(torch.isnan(x_out), torch.full_like(x_out, 0.0), x_out)\n   \n    return x_out\n\ndef data_pre2(x):\n    \"取出54个点\"\n    col0 = [489,490,491,494,496,497,500,501,502,503,504,505]\n    col1 = [i+468 for i in range(21)]   # left_hand\n    col2 = [j+522 for j in range(21)]   # right_hand\n    x_pose = x[:,col0,:2]\n    x_left = x[:,col1,:2]\n    x_right = x[:,col2,:2]\n    \n    x_left = torch.where(torch.isnan(x_left), torch.full_like(x_left, 0.0), x_left)\n    x_right = torch.where(torch.isnan(x_right), torch.full_like(x_right, 0.0), x_right)\n\n    x_left = x_left - x_left.mean(1, keepdim=True)\n    x_left = x_left / x_left.std(dim=1, keepdim=True)\n    \n    x_right = x_right - x_right.mean(1, keepdim=True)\n    x_right = x_right / x_right.std(dim=1, keepdim=True)\n    \n    x_out = torch.cat([x_pose,x_left,x_right],dim=1)\n    x_out = torch.where(torch.isnan(x_out), torch.full_like(x_out, 0.0), x_out)\n   \n    return x_out\n\ndef data_pre3(x):\n    \"取出54个点\"\n    col0 = [489,490,491,494,496,497,500,501,502,503,504,505]\n    col1 = [i+468 for i in range(21)]   # left_hand\n    col2 = [j+522 for j in range(21)]   # right_hand\n    x_pose = x[:,col0,:2]\n    x_left = x[:,col1,:2]\n    x_right = x[:,col2,:2]\n    \n    x_out = torch.cat([x_pose,x_left,x_right],dim=1)\n    x_out = torch.where(torch.isnan(x_out), torch.full_like(x_out, 0.0), x_out)\n   \n    return x_out\n\nclass MultiheadAttention(nn.Module):\n    def __init__(self, d_model, num_heads):\n        super(MultiheadAttention, self).__init__()\n        self.d_model = d_model\n        self.num_heads = num_heads\n        self.head_dim = d_model // num_heads\n\n        self.q_linear = nn.Linear(d_model, d_model, bias=True)\n        self.k_linear = nn.Linear(d_model, d_model, bias=True)\n        self.v_linear = nn.Linear(d_model, d_model, bias=True)\n        self.out_linear = nn.Linear(d_model, d_model, bias=True)\n\n    def forward(self, query, key, value, mask=None):\n        # x: [batch_size, seq_len, embed_dim]\n\n        q = self.q_linear(query).view(query.size(0), -1, self.num_heads, self.head_dim).transpose(1, 2)  # [batch_size, num_heads, seq_len, head_dim]\n        k = self.k_linear(key).view(key.size(0), -1, self.num_heads, self.head_dim).transpose(1, 2)      # [batch_size, num_heads, seq_len, head_dim]\n        v = self.v_linear(value).view(value.size(0), -1, self.num_heads, self.head_dim).transpose(1, 2)  # [batch_size, num_heads, seq_len, head_dim]\n\n        # 计算注意力得分\n        scores = torch.matmul(q, k.transpose(-2, -1)) / (self.head_dim ** 0.5)   # [batch_size, num_heads, seq_len, seq_len]\n        if mask is not None:\n            mask = mask.unsqueeze(1).unsqueeze(1)\n            scores = scores.masked_fill(mask == 0, float('-inf'))\n        attn = torch.softmax(scores, dim=-1)\n\n        # 计算加权和\n        attn_out = torch.matmul(attn, v)  # [batch_size, num_heads, seq_len, head_dim]\n        attn_out = attn_out.transpose(1, 2).contiguous().view(query.size(0), -1, self.num_heads * self.head_dim)  # [batch_size, seq_len, embed_dim]\n\n        # 输出投影\n        attn_out = self.out_linear(attn_out)   # [batch_size, seq_len, embed_dim]\n        return attn_out\n\nclass TransformerEncoderLayer(torch.nn.Module):\n    def __init__(self, d_model, nhead, dim_feedforward, dropout):\n        super().__init__()\n        self.self_attn = MultiheadAttention(d_model, nhead)\n        self.linear1 = nn.Linear(d_model, dim_feedforward)\n        self.dropout = nn.Dropout(dropout)\n        self.linear2 = nn.Linear(dim_feedforward, d_model)\n        self.norm1 = nn.LayerNorm(d_model)\n        self.norm2 = nn.LayerNorm(d_model)\n\n    def forward(self, src, src_mask=None, src_key_padding_mask=None):\n        # self-attention\n        src2 = self.self_attn(src, src, src)\n        src = src + self.dropout(src2)\n        src = self.norm1(src)\n\n        # feedforward\n        src2 = self.linear2(self.dropout(F.relu(self.linear1(src))))\n        src = src + self.dropout(src2)\n        src = self.norm2(src)\n\n        return src\n\n\nclass TransformerEncoder(nn.Module):\n    def __init__(self, d_model, nhead, dim_feedforward, dropout, num_layers):\n        super().__init__()\n        self.layers = nn.ModuleList([TransformerEncoderLayer(d_model, nhead, dim_feedforward, dropout)\n                                     for i in range(num_layers)])\n\n    def forward(self, src, mask=None, src_key_padding_mask=None):\n        output = src\n        for layer in self.layers:\n            output = layer(output)\n        return output\n\n\nclass SPOTERTransformerDecoderLayer(nn.Module):\n    def __init__(self, d_model, nhead, dim_feedforward=1024, dropout=0.1):\n        super(SPOTERTransformerDecoderLayer, self).__init__()\n\n        self.multihead_attn = MultiheadAttention(d_model, nhead)\n        # Implementation of Feedforward model\n        self.linear1 = nn.Linear(d_model, dim_feedforward)\n        self.dropout = nn.Dropout(dropout)\n        self.linear2 = nn.Linear(dim_feedforward, d_model)\n        self.norm1 = nn.LayerNorm(d_model)\n        self.norm2 = nn.LayerNorm(d_model)\n        self.norm3 = nn.LayerNorm(d_model)\n        self.dropout1 = nn.Dropout(dropout)\n        self.dropout2 = nn.Dropout(dropout)\n        self.dropout3 = nn.Dropout(dropout)\n\n\n    def forward(self, tgt, memory):\n        tgt = tgt + self.dropout1(tgt)\n        tgt = self.norm1(tgt)\n        tgt2 = self.multihead_attn(tgt, memory, memory)\n        tgt = tgt + self.dropout2(tgt2)\n        tgt = self.norm2(tgt)\n        tgt2 = self.linear2(self.dropout(F.relu(self.linear1(tgt))))\n        tgt = tgt + self.dropout3(tgt2)\n        tgt = self.norm3(tgt)\n        return tgt\n\n\nclass TransformerDecoder(nn.Module):\n    def __init__(self, d_model, nhead, dim_feedforward, dropout, num_layers):\n        super().__init__()\n        self.layers = nn.ModuleList([SPOTERTransformerDecoderLayer(d_model, nhead, dim_feedforward, dropout)\n                                     for i in range(num_layers)])\n\n    def forward(self, tgt, memory):\n        output = tgt\n        for layer in self.layers:\n            output = layer(output, memory)\n\n        return output\n\nclass SPOTER1(nn.Module):\n\n    def __init__(self, num_classes, hidden_dim=108, nhead=9, dim_feedforward=1024, dropout=0.1, num_layers=1):\n        super().__init__()\n\n        self.row_embed = nn.Parameter(torch.rand(50, hidden_dim))\n        self.pos = nn.Parameter(torch.cat([self.row_embed[0].unsqueeze(0).repeat(1, 1, 1)], dim=-1).flatten(0, 1).unsqueeze(0))\n        self.class_query = nn.Parameter(torch.rand(1, hidden_dim))\n        self.Encoder = TransformerEncoder(hidden_dim, nhead, dim_feedforward, dropout, num_layers)\n        self.Decoder = TransformerDecoder(hidden_dim, nhead, dim_feedforward, dropout, num_layers)\n        self.linear_class = nn.Linear(hidden_dim, num_classes)\n\n\n    def forward(self, inputs):\n        inputs = data_pre1(inputs)\n        h = torch.unsqueeze(inputs.flatten(start_dim=1), 0).float()   # 将inputs的维度使用flatten降低,(54,2)--->(108);再使用unsqueeze进行升维变成(1,108),最后h为(1,T,108)\n        src = self.pos + h\n        tgt = self.class_query.unsqueeze(0)\n        menery = self.Encoder(src)\n        out = self.Decoder(tgt,menery)\n        sign = self.linear_class(out)\n        sign = sign.reshape(-1)\n        sign = F.softmax(sign,dim=0)\n        return sign\n    \n    \nclass SPOTER2(nn.Module):\n\n    def __init__(self, num_classes, hidden_dim=512, nhead=4, dim_feedforward=2048, dropout=0.1, num_layers=1):\n        super().__init__()\n\n        self.row_embed = nn.Parameter(torch.rand(50, hidden_dim))\n        self.pos = nn.Parameter(torch.cat([self.row_embed[0].unsqueeze(0).repeat(1, 1, 1)], dim=-1).flatten(0, 1).unsqueeze(0))\n        self.class_query = nn.Parameter(torch.rand(1, hidden_dim))\n        self.embeding = nn.Linear(108, hidden_dim)\n        self.Encoder = TransformerEncoder(hidden_dim, nhead, dim_feedforward, dropout, num_layers)\n        self.Decoder = TransformerDecoder(hidden_dim, nhead, dim_feedforward, dropout, num_layers)\n        self.linear_class = nn.Linear(hidden_dim, num_classes)\n\n\n    def forward(self, inputs):\n        inputs = data_pre2(inputs)\n        h = torch.unsqueeze(inputs.flatten(start_dim=1), 0).float()   # 将inputs的维度使用flatten降低,(54,2)--->(108);再使用unsqueeze进行升维变成(1,108),最后h为(1,T,108)\n        h = self.embeding(h)\n        src = self.pos + h\n        tgt = self.class_query.unsqueeze(0)\n        menery = self.Encoder(src)\n        out = self.Decoder(tgt,menery)\n        sign = self.linear_class(out)\n        sign = sign.reshape(-1)\n        sign = F.softmax(sign,dim=0) \n        return sign\n    \nclass SPOTER3(nn.Module):\n\n    def __init__(self, num_classes, hidden_dim=108, nhead=9, dim_feedforward=2048, dropout=0.1, num_layers=6):\n        super().__init__()\n\n        self.row_embed = nn.Parameter(torch.rand(50, hidden_dim))\n        self.pos = nn.Parameter(torch.cat([self.row_embed[0].unsqueeze(0).repeat(1, 1, 1)], dim=-1).flatten(0, 1).unsqueeze(0))\n        self.class_query = nn.Parameter(torch.rand(1, hidden_dim))\n        self.Encoder = TransformerEncoder(hidden_dim, nhead, dim_feedforward, dropout, num_layers)\n        self.Decoder = TransformerDecoder(hidden_dim, nhead, dim_feedforward, dropout, num_layers)\n\n        self.linear_class = nn.Linear(hidden_dim, num_classes)\n\n\n    def forward(self, inputs):\n        inputs = data_pre3(inputs)\n        h = torch.unsqueeze(inputs.flatten(start_dim=1), 0).float()   # 将inputs的维度使用flatten降低,(54,2)--->(108);再使用unsqueeze进行升维变成(1,108),最后h为(1,T,108)\n        src = self.pos + h\n        tgt = self.class_query.unsqueeze(0)\n        menery = self.Encoder(src)\n        out = self.Decoder(tgt,menery)\n        sign = self.linear_class(out)\n        sign = sign.reshape(-1)\n        sign = F.softmax(sign,dim=0) \n        return sign","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-11T08:49:41.046317Z","iopub.execute_input":"2023-04-11T08:49:41.046809Z","iopub.status.idle":"2023-04-11T08:49:41.128716Z","shell.execute_reply.started":"2023-04-11T08:49:41.046764Z","shell.execute_reply":"2023-04-11T08:49:41.127483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_pre6(x):\n    '''\n    取出54+16*2+4+20=110 个点\n    return:x_out,x_reye,x_leye,x_nose,x_slip\n    '''\n    POSE = [489, 490, 491, 494, 496, 497, 500, 501, 502, 503, 504, 505]  # pose 12\n    left_hand = [i + 468 for i in range(21)]  # left_hand 21\n    right_hand = [j + 522 for j in range(21)]  # right_hand 21\n    REYE = [33, 7, 163, 144, 145, 153, 154, 155, 133, 246, 161, 160, 159, 158, 157, 173]   #16\n    LEYE = [263, 249, 390, 373, 374, 380, 381, 382, 362, 466, 388, 387, 386, 385, 384, 398]  #16\n    NOSE = [1, 2, 98, 327]  #4\n    SLIP = [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 191, 80, 81, 82, 13, 312, 311, 310, 415] #20\n    \n    x_left = x[:, left_hand, :2]\n    x_right = x[:, right_hand, :2]\n    \n#     # 无效帧的处理\n#     left_isnan = torch.isnan(x_left).any(dim=2)\n#     valid_left_frames = torch.nonzero(~left_isnan.any(dim=1)).squeeze()\n#     # print(valid_left_frames)\n#     right_isnan = torch.isnan(x_right).any(dim=2)\n#     valid_right_frames = torch.nonzero(~right_isnan.any(dim=1)).squeeze()\n\n#     if valid_left_frames.numel() > valid_right_frames.numel():\n#         valid_frame = valid_left_frames\n#     else:\n#         valid_frame = valid_right_frames\n\n#     if valid_frame.numel() < 5 and valid_frame.numel() > 0:    # 有效帧的长度小于5时\n#         # print(valid_frame,pq_path)\n#         # print(valid_left_frames)\n#         if len(x) > 10:\n#             try:\n#                 num_frames_to_insert = 10 - valid_frame.numel()\n#                 insert_frame_idxs = torch.linspace(0, valid_frame[-1], num_frames_to_insert+2)[0:-1].round().long()\n#                 valid_frames = torch.cat([valid_frame,insert_frame_idxs]).tolist()\n#                 valid_frames = sorted(list(set(valid_frames)))\n#                 x = x[valid_frames]\n#             except:\n#                 print(valid_frame,len(x))\n                \n        \n#     elif valid_frame.numel() >= 5:\n#         x = x[valid_frame]\n        \n    x_left = x[:, left_hand, :2]\n    x_right = x[:, right_hand, :2]\n    x_pose = x[:, POSE, :2]\n    x_reye = x[:,REYE,:2]\n    x_leye = x[:,LEYE,:2]\n    x_nose = x[:,NOSE,:2]\n    x_slip = x[:,SLIP,:2]\n\n\n    x_left = x_left - x_left[~torch.isnan(x_left)].mean(0,keepdim=True)\n    x_left = x_left / x_left[~torch.isnan(x_left)].std(0,keepdim=True)\n    \n    x_right = x_right - x_right[~torch.isnan(x_right)].mean(0,keepdim=True)\n    x_right = x_right / x_right[~torch.isnan(x_right)].std(0,keepdim=True)\n    \n    x_out = torch.cat([x_pose, x_left, x_right], dim=1)\n    x_out = torch.where(torch.isnan(x_out), torch.full_like(x_out, 0.0), x_out)\n\n\n    x_reye = torch.where(torch.isnan(x_reye), torch.full_like(x_reye, 0.0), x_reye)\n    x_leye = torch.where(torch.isnan(x_leye), torch.full_like(x_leye, 0.0), x_leye)\n    x_nose = torch.where(torch.isnan(x_nose), torch.full_like(x_nose, 0.0), x_nose)\n    x_slip = torch.where(torch.isnan(x_slip), torch.full_like(x_slip, 0.0), x_slip)\n    \n    data_out =torch.cat([x_out,x_reye,x_leye,x_nose,x_slip], dim=1)  # (None,110,2)\n    data_out = torch.where(torch.isnan(data_out), torch.full_like(data_out, 0.0), data_out)\n\n    return data_out\nclass XEmbed(nn.Module):\n    def __init__(self,point_dim,embed_dim):\n        super().__init__()\n        self.v = nn.Sequential(\n            nn.Linear(point_dim, embed_dim*2, bias=True),\n            nn.LayerNorm(embed_dim*2),\n            nn.ReLU(inplace=True),\n            nn.Linear(embed_dim*2, embed_dim, bias=True),\n            nn.LayerNorm(embed_dim),\n            nn.ReLU(inplace=True),\n        )  \n    def forward(self, x):\n        v = self.v(x)\n        return v\nclass SPOTER6(nn.Module):\n    \"\"\"\n    Implementation of the SPOTER (Sign POse-based TransformER) architecture for sign language recognition from sequence\n    of skeletal data.\n    \"\"\"\n\n    def __init__(self, num_classes, hidden_dim=220, point_num = 110, nhead=4, dim_feedforward=1024, dropout=0.1, num_layers=2):\n        super().__init__()\n\n        self.x_embed = XEmbed(point_num*2,hidden_dim)\n        \n        self.row_embed = nn.Parameter(torch.rand(50, hidden_dim))\n        self.pos = nn.Parameter(torch.cat([self.row_embed[0].unsqueeze(0).repeat(1, 1, 1)], dim=-1).flatten(0, 1).unsqueeze(0))\n        self.class_query = nn.Parameter(torch.rand(1, hidden_dim))\n        self.Encoder = TransformerEncoder(hidden_dim, nhead, dim_feedforward, dropout, num_layers)\n        self.Decoder = TransformerDecoder(hidden_dim, nhead, dim_feedforward, dropout, num_layers)\n\n        self.linear_class = nn.Linear(hidden_dim, num_classes)\n\n\n    def forward(self, inputs):\n        # inputs: torch.size(None,110,2)\n        inputs = data_pre6(inputs)\n        h = torch.unsqueeze(inputs.flatten(start_dim=1), 0).float()   # 将inputs的维度使用flatten降低,(54,2)--->(108);再使用unsqueeze进行升维变成(1,108),最后h为(T,1,108)\n        # torch.Size([1, None, 220])\n        h = self.x_embed(h)\n        # torch.Size([1, None, 256])\n\n        src = self.pos + h\n        tgt = self.class_query.unsqueeze(0)\n        menery = self.Encoder(src)\n        out = self.Decoder(tgt,menery)\n        # out = F.dropout(out,p=0.1,training=self.training)\n        sign = self.linear_class(out)\n        \n        sign = sign.reshape(-1)\n        sign = F.softmax(sign,dim=0)\n        return sign\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:41.239868Z","iopub.execute_input":"2023-04-11T08:49:41.240652Z","iopub.status.idle":"2023-04-11T08:49:41.291861Z","shell.execute_reply.started":"2023-04-11T08:49:41.240614Z","shell.execute_reply":"2023-04-11T08:49:41.290633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"slr_model6 = SPOTER6(num_classes=250, hidden_dim=220,num_layers=2)\nmodel6_pth = \"/kaggle/input/train5-pth/train6_aug_best.pth\"\nslr_model6.load_state_dict(torch.load(model6_pth, map_location=torch.device('cpu')))","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:41.294377Z","iopub.execute_input":"2023-04-11T08:49:41.295220Z","iopub.status.idle":"2023-04-11T08:49:41.386951Z","shell.execute_reply.started":"2023-04-11T08:49:41.295179Z","shell.execute_reply":"2023-04-11T08:49:41.385164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"slr_model1 = SPOTER1(num_classes=250)\nmodel_pth1 = '/kaggle/input/train5-pth/all_train5_best.pth'\nslr_model1.load_state_dict(torch.load(model_pth1,map_location=torch.device('cpu')))\n#slr_model.train(False)\n# slr_model.eval()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:41.388834Z","iopub.execute_input":"2023-04-11T08:49:41.389702Z","iopub.status.idle":"2023-04-11T08:49:41.429817Z","shell.execute_reply.started":"2023-04-11T08:49:41.389643Z","shell.execute_reply":"2023-04-11T08:49:41.428940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"slr_model2 = SPOTER2(num_classes=250,hidden_dim=512)\nmodel_pth2 = '/kaggle/input/train5-pth/epoch_27_512dim.pth'\nslr_model2.load_state_dict(torch.load(model_pth2,map_location=torch.device('cpu')))\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:41.434580Z","iopub.execute_input":"2023-04-11T08:49:41.437353Z","iopub.status.idle":"2023-04-11T08:49:41.573764Z","shell.execute_reply.started":"2023-04-11T08:49:41.437315Z","shell.execute_reply":"2023-04-11T08:49:41.572808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"slr_model3 = SPOTER3(num_classes=250,hidden_dim=108)\nmodel_pth3 = '/kaggle/input/train3-pth/fold_0.pth'\nslr_model3.load_state_dict(torch.load(model_pth3,map_location=torch.device('cpu')))\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:41.577780Z","iopub.execute_input":"2023-04-11T08:49:41.578613Z","iopub.status.idle":"2023-04-11T08:49:41.739521Z","shell.execute_reply.started":"2023-04-11T08:49:41.578577Z","shell.execute_reply":"2023-04-11T08:49:41.738514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"REYE = [33, 7, 163, 144, 145, 153, 154, 155, 133,\n    246, 161, 160, 159, 158, 157, 173]\nLEYE = [\n    263, 249, 390, 373, 374, 380, 381, 382, 362,\n    466, 388, 387, 386, 385, 384, 398,\n]\nNOSE=[\n    1,2,98,327\n]\nSLIP = [\n    78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    191, 80, 81, 82, 13, 312, 311, 310, 415,\n]\nSPOSE = (np.array([\n    11,13,15,12,14,16,23,24,\n])+489).tolist()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:41.743804Z","iopub.execute_input":"2023-04-11T08:49:41.746039Z","iopub.status.idle":"2023-04-11T08:49:41.756146Z","shell.execute_reply.started":"2023-04-11T08:49:41.746001Z","shell.execute_reply":"2023-04-11T08:49:41.754716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_hflip_hand(lhand, rhand):\n    rhand[...,0] = 1- rhand[...,0]\n    lhand[...,0] = 1- lhand[...,0]\n    rhand, lhand = lhand,rhand\n    return lhand, rhand\n\ndef do_hflip_eye(leye, reye):\n    reye[...,0] = 1- reye[...,0]\n    leye[...,0] = 1- leye[...,0]\n    reye, leye = leye,reye\n    return leye, reye\n\ndef do_hflip_spose(spose):\n    spose[...,0] = 1-spose[...,0]\n    spose = spose[:,[3,4,5,0,1,2,7,6]]\n    return spose\n\ndef do_hflip_slip(slip):\n    slip[...,0] = 1- slip[...,0]\n    slip = slip[:,[10,9,8,7,6,5,4,3,2,1,0]+[19,18,17,16,15,14,13,12,11]]\n    return slip","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:41.757317Z","iopub.execute_input":"2023-04-11T08:49:41.757852Z","iopub.status.idle":"2023-04-11T08:49:41.778238Z","shell.execute_reply.started":"2023-04-11T08:49:41.757819Z","shell.execute_reply":"2023-04-11T08:49:41.777143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class XEmbed(nn.Module):\n    def __init__(self,point_dim,embed_dim):\n        super().__init__()\n        self.v = nn.Sequential(\n            nn.Linear(point_dim, embed_dim*2, bias=True),\n            nn.LayerNorm(embed_dim*2),\n            nn.ReLU(inplace=True),\n            nn.Linear(embed_dim*2, embed_dim, bias=True),\n            nn.LayerNorm(embed_dim),\n            nn.ReLU(inplace=True),\n        )  \n    def forward(self, x):\n#         B,L = x.shape\n        v = self.v(x)\n        return v\nx_embed = XEmbed(212,256)\nx = torch.randn([1,56,212])\ny = x_embed(x)\nprint(y.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:41.779633Z","iopub.execute_input":"2023-04-11T08:49:41.780716Z","iopub.status.idle":"2023-04-11T08:49:41.802202Z","shell.execute_reply.started":"2023-04-11T08:49:41.780678Z","shell.execute_reply":"2023-04-11T08:49:41.801322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Net3(nn.Module):\n    def __init__(self,model1,model2,model3):\n        super().__init__()\n        self.model1 = model1\n        self.model2 = model2\n        self.model3 = model3\n    def forward(self,x):\n        y1 = self.model1(x)\n        y2 = self.model2(x)\n        y3 = self.model3(x)\n        out = 0.3*y1 + 0.2*y2 + 0.5*y3\n        return out\n# slr_model = Net(slr_model1,slr_model2,slr_model3)\n# slr_model.train(False)\n\nclass Net2(nn.Module):\n    def __init__(self,model1,model2):\n        super().__init__()\n        self.model1 = model1\n        self.model2 = model2\n        \n    def forward(self,x):\n        y1 = self.model1(x)\n        y2 = self.model2(x)\n        out = 0.3*y1 + 0.7*y2 \n        return out\n\nslr_model = Net2(slr_model1,slr_model6)\n# slr_model.train(False)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:41.805913Z","iopub.execute_input":"2023-04-11T08:49:41.808383Z","iopub.status.idle":"2023-04-11T08:49:41.820458Z","shell.execute_reply.started":"2023-04-11T08:49:41.808348Z","shell.execute_reply":"2023-04-11T08:49:41.819498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"测试模型的准确率","metadata":{}},{"cell_type":"code","source":"\nimport tqdm\nimport numpy as np\nimport time\nimport json\nimport pandas as pd\n\n\ndef load_relevant_data_subset(pq_path):\n    ROWS_PER_FRAME = 543  # number of landmarks per frame\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\n\ndef load_dataset(file_location):\n    sign_dic = {}\n    label_dic_path = '/kaggle/input/asl-signs/sign_to_prediction_index_map.json'\n    with open(label_dic_path, 'r') as sign_f:\n        sign_dic = json.load(sign_f)\n    df = pd.read_csv(file_location, encoding=\"utf-8\")\n    df_signs = df['sign'].to_list()\n    df_labels = [sign_dic.get(i) for i in df_signs]   # 从字典返回labels\n    df_paths = df['path'].to_list()\n    return df_paths, df_labels\n\ndata_file = '/kaggle/input/asl-signs/train.csv'\ndf_paths, df_labels = load_dataset(data_file)\n\nsign_list = [0]*250\n\ndef test(slr_model,df_path,de_label):\n    slr_model.train(False)\n    frames = load_relevant_data_subset('/kaggle/input/asl-signs/'+df_path)\n    inputs = torch.Tensor(frames)\n#     inputs = data_pre(inputs)\n#     print(inputs)\n#     print(inputs.shape)\n    \n    output = slr_model(inputs)\n#     print(output)\n   \n    sign = torch.argmax(output)\n    \n    \n#     print(output)\n    print('pre:{}\\t true:{}'.format(sign,de_label))\n    if int(sign) == int(de_label):\n        sign_list[de_label] = sign_list[de_label]+ 1\n#         print(sign)\n        \nfor i in range(100):\n    \n    test(slr_model6,df_paths[i], df_labels[i])   \nprint(sign_list)\nprint(sum(sign_list))","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:41.828983Z","iopub.execute_input":"2023-04-11T08:49:41.829711Z","iopub.status.idle":"2023-04-11T08:49:43.154177Z","shell.execute_reply.started":"2023-04-11T08:49:41.829677Z","shell.execute_reply":"2023-04-11T08:49:43.153032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas\nimport numpy as np\nimport torch\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\ndef plot_confusion_matrix(model, df_paths, df_labels, device, num_classes):\n    true_labels, predicted_labels = [], []\n    pred_correct, pred_all = 0, 0\n    data_top1 = {i: [0, 0] for i in range(num_classes)}   # 存储每一类的数量和识别正确的数量\n\n    model.eval()\n\n    with torch.no_grad():\n        for i in range(len(df_paths)):\n            df_path = df_paths[i]\n            labels = df_labels[i]\n            inputs = load_relevant_data_subset('/kaggle/input/asl-signs/'+df_path)\n#             print(inputs)\n#             print(labels)\n            inputs = torch.Tensor(inputs)\n            inputs = inputs.to(device)\n            \n            outputs = model(inputs)\n            \n            pred = int(torch.argmax(\n                torch.nn.functional.softmax(outputs, dim=0)))\n            label = int(labels)\n            predicted_labels.append(pred)\n            true_labels.append(label)\n            if pred == label:\n                pred_correct += 1\n#                 print('pre:{}\\t true:{}'.format(pred,label))\n                \n            pred_all += 1\n\n    cm_pt = confusion_matrix(\n        true_labels, predicted_labels, labels=range(num_classes))\n\n    return cm_pt, pred_correct, pred_all\n# if torch.cuda.is_available():\n#     device = torch.device(\"cuda\")\n# else:\n#     device = torch.device(\"cpu\")\n# slr_model1.train(False)\n# slr_model1.to(device)\n# num_classes = 250\n# cm, pred_correct, pred_all = plot_confusion_matrix(slr_model1, df_paths, df_labels, device, num_classes)\n# print(pred_correct, pred_all, pred_correct/pred_all)\n# print(cm)\n# plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n# plt.colorbar()\n# tick_marks = np.arange(num_classes)\n# # plt.xticks(tick_marks, range(num_classes))\n# # plt.yticks(tick_marks, range(num_classes))\n# plt.xlabel('Predicted Label')\n# plt.ylabel('True Label')\n# plt.show()\n# plt.savefig(\"/kaggle/working/confusion_matrix.png\")\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:43.155813Z","iopub.execute_input":"2023-04-11T08:49:43.156484Z","iopub.status.idle":"2023-04-11T08:49:43.167612Z","shell.execute_reply.started":"2023-04-11T08:49:43.156443Z","shell.execute_reply":"2023-04-11T08:49:43.166354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **将pytorch模型转换为onnx模型**","metadata":{}},{"cell_type":"code","source":"input_onnx_file = \"/kaggle/working/model.onnx\"\ndef run_convert_onnx(slr_model): \n    torch.onnx.export(\n                slr_model,\n                #torch.jit.script(input_net),\n                #torch.jit.trace(input_net, torch.zeros(100,num_landmark,3)),          # model being run \n                torch.randn((60,543,3)),   # torch.randn((60,54,2))\n                input_onnx_file,             # where to save the model (can be a file or file-like object)\n                export_params = True,        # store the trained parameter weights inside the model file\n                opset_version = 12,          # the ONNX version to export the model to\n                do_constant_folding=True,    # whether to execute constant folding for optimization \n                input_names =  ['inputs'],    # the model's input names\n                output_names = ['outputs'],   # the model's output names\n                dynamic_axes={\n                    'inputs': {0: 'length'},\n                    #'output': {0: 'length'},\n                },\n                #verbose = True,\n            )\n\nrun_convert_onnx(slr_model)\nprint('model.onnx saved !!')","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:43.169374Z","iopub.execute_input":"2023-04-11T08:49:43.170099Z","iopub.status.idle":"2023-04-11T08:49:44.006072Z","shell.execute_reply.started":"2023-04-11T08:49:43.170059Z","shell.execute_reply":"2023-04-11T08:49:44.003965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **检查模型输出是否相同**\n导入tf模型","metadata":{}},{"cell_type":"code","source":"!pip install onnxruntime","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:44.007625Z","iopub.execute_input":"2023-04-11T08:49:44.008108Z","iopub.status.idle":"2023-04-11T08:49:53.404564Z","shell.execute_reply.started":"2023-04-11T08:49:44.008066Z","shell.execute_reply":"2023-04-11T08:49:53.403345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import onnxruntime as ort\nmodel_path = \"/kaggle/working/model.onnx\"\nort_session = ort.InferenceSession(model_path)\n\n# 定义测试输入\nimport numpy as np\ninput_tensor = torch.randn(60,543,3)\ninput_data = input_tensor.numpy()\n\n# # 检查onnx输出\n# print(ort_session.get_inputs())\n \n    \nort_inputs = {ort_session.get_inputs()[0].name: input_data}\nort_outputs = ort_session.run(None,ort_inputs )\n# print(ort_outputs)\n\n# 检查pytorch输出\n\n# slr_model.train(False)\n# slr_model.eval()\n# slr_output = slr_model(input_tensor)\n# print(slr_output.shape)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:53.406652Z","iopub.execute_input":"2023-04-11T08:49:53.407310Z","iopub.status.idle":"2023-04-11T08:49:53.516248Z","shell.execute_reply.started":"2023-04-11T08:49:53.407266Z","shell.execute_reply":"2023-04-11T08:49:53.515276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **将onnx模型转换为tf_model模型**","metadata":{}},{"cell_type":"code","source":"!pip install onnx_tf    # 安装依赖包","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:49:53.517574Z","iopub.execute_input":"2023-04-11T08:49:53.518426Z","iopub.status.idle":"2023-04-11T08:50:03.056238Z","shell.execute_reply.started":"2023-04-11T08:49:53.518388Z","shell.execute_reply":"2023-04-11T08:50:03.054944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from onnx_tf.backend import prepare\nimport onnx\nTF_PATH = \"tf_model\" # 保存tf模型的位置\nONNX_PATH = \"/kaggle/working/model.onnx\"   # onnx模型的path\nonnx_model = onnx.load(ONNX_PATH)  # load onnx model\ntf_rep = prepare(onnx_model)  # creating TensorflowRep object\ntf_rep.export_graph(TF_PATH)\nprint('tf.saved_model() passed !!')","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:50:03.059313Z","iopub.execute_input":"2023-04-11T08:50:03.059785Z","iopub.status.idle":"2023-04-11T08:50:23.964434Z","shell.execute_reply.started":"2023-04-11T08:50:03.059736Z","shell.execute_reply":"2023-04-11T08:50:23.963258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **测试tf_model模型是否准确**","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nmobilenet_save_path = '/kaggle/working/tf_model'\nloaded = tf.saved_model.load(mobilenet_save_path)\n# print(list(loaded.signatures.keys())) \ninfer = loaded.signatures[\"serving_default\"]\n# print(infer)\n# 检查tf输出\nx1=tf.constant(input_tensor)\nlabel = infer(x1)\n# print(label['outputs'])","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:50:23.966330Z","iopub.execute_input":"2023-04-11T08:50:23.966754Z","iopub.status.idle":"2023-04-11T08:50:30.440723Z","shell.execute_reply.started":"2023-04-11T08:50:23.966715Z","shell.execute_reply":"2023-04-11T08:50:30.439596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # **将转换好的.pb模型打包**","metadata":{}},{"cell_type":"code","source":"packagePath = '/kaggle/working/tf_model'\nzipPath = '/kaggle/working/'\n!zip tf_model.zip $model_path","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:50:30.442622Z","iopub.execute_input":"2023-04-11T08:50:30.443035Z","iopub.status.idle":"2023-04-11T08:50:32.189382Z","shell.execute_reply.started":"2023-04-11T08:50:30.442994Z","shell.execute_reply":"2023-04-11T08:50:32.188116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **将tf_model模型转为tflite模型**","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n# print(tf.__version__)\nsaved_model_dir = '/kaggle/working/tf_model'   # path to the SavedModel directory\ntflite_path = '/kaggle/working/model.tflite'   # path to the tflite model\n# Convert the model\nconverter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir) \n\nconverter.target_spec.supported_ops = [\n  tf.lite.OpsSet.TFLITE_BUILTINS,  # enable TensorFlow Lite ops.\n  tf.lite.OpsSet.SELECT_TF_OPS   # enable TensorFlow ops.\n]\ntflite_model = converter.convert()\nopen(tflite_path, \"wb\").write(tflite_model)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:50:32.191476Z","iopub.execute_input":"2023-04-11T08:50:32.191834Z","iopub.status.idle":"2023-04-11T08:50:38.170680Z","shell.execute_reply.started":"2023-04-11T08:50:32.191803Z","shell.execute_reply":"2023-04-11T08:50:38.169646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **测试转换的tflite模型是否准确**","metadata":{}},{"cell_type":"code","source":"!pip install tflite_runtime==2.9.1","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:50:38.172141Z","iopub.execute_input":"2023-04-11T08:50:38.172605Z","iopub.status.idle":"2023-04-11T08:50:48.631596Z","shell.execute_reply.started":"2023-04-11T08:50:38.172567Z","shell.execute_reply":"2023-04-11T08:50:48.630371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm\nimport tflite_runtime.interpreter as tflite\n\nmodel_path = '/kaggle/working/model.tflite'\ninterpreter = tflite.Interpreter(model_path)\nfound_signatures = list(interpreter.get_signature_list().keys())\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\n# input_tensor = torch.randn(50,543,3)\n# input_data = input_tensor.numpy()\n# print(input_data)\noutput = prediction_fn(inputs=input_data)\nsign = np.argmax(output[\"outputs\"])\nprint(output['outputs'])\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:50:48.634533Z","iopub.execute_input":"2023-04-11T08:50:48.635074Z","iopub.status.idle":"2023-04-11T08:50:48.738344Z","shell.execute_reply.started":"2023-04-11T08:50:48.635025Z","shell.execute_reply":"2023-04-11T08:50:48.737060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **submission**","metadata":{}},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\nimport pandas as pd\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:50:48.740039Z","iopub.execute_input":"2023-04-11T08:50:48.740576Z","iopub.status.idle":"2023-04-11T08:50:48.747740Z","shell.execute_reply.started":"2023-04-11T08:50:48.740536Z","shell.execute_reply":"2023-04-11T08:50:48.746550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm\nimport tflite_runtime.interpreter as tflite\nimport time\nt1 = time.time()\nmodel_path = '/kaggle/working/model.tflite'\ninterpreter = tflite.Interpreter(model_path)\nfound_signatures = list(interpreter.get_signature_list().keys())\npq_path = '/kaggle/input/asl-signs/train_landmark_files/18796/1020380433.parquet'\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\nframes = load_relevant_data_subset(pq_path)\n# print(frames)\noutput = prediction_fn(inputs=frames)\n# print(output)\nsign = np.argmax(output[\"outputs\"])\nprint(sign, output['outputs'].shape)\nt2 =time.time()\nprint('{:.2f}ms'.format((t2-t1)*1000))","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:50:48.749348Z","iopub.execute_input":"2023-04-11T08:50:48.750034Z","iopub.status.idle":"2023-04-11T08:50:48.816216Z","shell.execute_reply.started":"2023-04-11T08:50:48.749996Z","shell.execute_reply":"2023-04-11T08:50:48.815020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = 61/152\nb =round(a,3)\nc={1:b}\nprint(c)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:50:48.817772Z","iopub.execute_input":"2023-04-11T08:50:48.818374Z","iopub.status.idle":"2023-04-11T08:50:48.825073Z","shell.execute_reply.started":"2023-04-11T08:50:48.818331Z","shell.execute_reply":"2023-04-11T08:50:48.823800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nimport time\ndef load_dataset(file_location):\n    sign_dic = {}\n    label_dic_path = '/kaggle/input/asl-signs/sign_to_prediction_index_map.json'\n    with open(label_dic_path, 'r') as sign_f:\n        sign_dic = json.load(sign_f)\n    # 读入数据集中所有的data数据与label，返回np.array类型(读进内存中)\n    # Load the datset csv file\n    df = pd.read_csv(file_location, encoding=\"utf-8\")\n    df_signs = df['sign'].to_list()\n    df_labels = [sign_dic.get(i) for i in df_signs]   # 从字典返回labels\n    df_paths = df['path'].to_list()\n    return df_paths, df_labels\n\ndata_file = '/kaggle/input/asl-signs/train.csv'\ndf_paths, df_labels = load_dataset(data_file)\n# print(df_paths[:50])\nsign_list = [0]*250\nsign_all = [0]*250\n\ndef run_tflite(df_path,de_label):\n    df_path = '/kaggle/input/asl-signs/'+df_path\n    model_path = '/kaggle/working/model.tflite'\n    interpreter = tflite.Interpreter(model_path)\n    found_signatures = list(interpreter.get_signature_list().keys())\n    prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n    frames = load_relevant_data_subset(df_path)\n#     print(frames.shape)\n    output = prediction_fn(inputs=frames)\n    sign = np.argmax(output[\"outputs\"])\n#     print(output)\n    print('pre:{}\\t true:{}'.format(sign,de_label))\n    if sign==de_label:\n        sign_list[de_label] += 1\n    sign_all[de_label] +=1\nt1 = time.time()\nvideo_sum = 100\nfor i in range(900,1000):\n    run_tflite(df_paths[i], df_labels[i])  \nt2 = time.time()\nprint(sign_list)\nprint(sum(sign_list))\nprint(sum(sign_all))\nprint('cost {:.4f}'.format(t2-t1))\nprint('pre video {:.4f}'.format((t2-t1)/video_sum))","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:50:48.826852Z","iopub.execute_input":"2023-04-11T08:50:48.827313Z","iopub.status.idle":"2023-04-11T08:50:55.012751Z","shell.execute_reply.started":"2023-04-11T08:50:48.827269Z","shell.execute_reply":"2023-04-11T08:50:55.010636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\n!zip submission.zip  'model.tflite'\n!ls\n\nprint(f'submit ok')","metadata":{"execution":{"iopub.status.busy":"2023-04-11T08:50:55.014463Z","iopub.execute_input":"2023-04-11T08:50:55.014859Z","iopub.status.idle":"2023-04-11T08:50:57.735124Z","shell.execute_reply.started":"2023-04-11T08:50:55.014819Z","shell.execute_reply":"2023-04-11T08:50:57.733760Z"},"trusted":true},"execution_count":null,"outputs":[]}]}