{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#refer to discussion at https://www.kaggle.com/competitions/asl-signs/discussion/391265","metadata":{"execution":{"iopub.status.busy":"2023-03-30T12:31:34.075273Z","iopub.execute_input":"2023-03-30T12:31:34.075664Z","iopub.status.idle":"2023-03-30T12:31:34.081072Z","shell.execute_reply.started":"2023-03-30T12:31:34.075630Z","shell.execute_reply":"2023-03-30T12:31:34.079814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#class Config(object):\nif 1:\n\tnum_class  = 250\n\tmax_length = 256  \n\tpoint_dim  = 1302\n\tembed_dim  = 384  \n\tnum_head   = 4\n\tnum_block  = 1\n\tlabel_smoothing = 0.75\n#CFG = Config()\n\n\n\nimport numpy as np\nimport random\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\ndef pack_seq(\n\tseq,\n):\n\tlength = [min(len(s), max_length)  for s in seq]\n\tbatch_size = len(seq)\n\tK = seq[0].shape[1]\n\tL = max(length)\n\t#print(length)\n\n\tx = torch.zeros((batch_size, L, point_dim)).to(seq[0].device)\n\tx_mask = torch.zeros((batch_size, L)).to(seq[0].device)\n\tfor b in range(batch_size):\n\t\tl = length[b]\n\t\tx[b, :l] = seq[b][:l,:]\n\t\tx_mask[b, l:] = 1\n\tx_mask = (x_mask>0.5)\n\n\treturn x, x_mask\n\ndef positional_encoding(length, embed_dim):\n\tdim = embed_dim//2\n\tposition = np.arange(length)[:, np.newaxis]     # (seq, 1)\n\tdim = np.arange(dim)[np.newaxis, :]/dim   # (1, dim)\n\tangle = 1 / (10000**dim)         # (1, dim)\n\tangle = position * angle    # (pos, dim)\n\tpos_embed = np.concatenate(\n\t\t[np.sin(angle), np.cos(angle)],\n\t\taxis=-1\n\t)\n\tpos_embed = torch.from_numpy(pos_embed).float()\n\treturn pos_embed\n\n\nclass XEmbed(nn.Module):\n\tdef __init__(self,\n\t):\n\t\tsuper().__init__()\n\t\tself.v = nn.Sequential(\n\t\t\tnn.Linear(point_dim, embed_dim*2, bias=True),\n\t\t\tnn.LayerNorm(embed_dim*2),\n\t\t\tnn.ReLU(inplace=True),\n\t\t\tnn.Linear(embed_dim*2, embed_dim, bias=True),\n\t\t\tnn.LayerNorm(embed_dim),\n\t\t\tnn.ReLU(inplace=True),\n\t\t)\n\tdef forward(self, x, x_mask):\n\t\tB,L,_ = x.shape\n\t\tv = self.v(x)\n\t\tx = v\n\t\treturn x, x_mask\n\nclass TransformerBlock(nn.Module):\n\tdef __init__(self,\n\t    embed_dim,\n        num_head,\n        out_dim,\n\t):\n\t\tsuper().__init__()\n\t\tself.attn  = MyMultiHeadAttention(\n\t\t\tembed_dim=embed_dim,\n\t\t\tout_dim=embed_dim,\n\t\t\tqk_dim=embed_dim // num_head,\n\t\t\tv_dim=embed_dim // num_head,\n\t\t\tnum_head=num_head,\n\n\t\t)\n\t\tself.ffn   = FeedForward(embed_dim, out_dim)\n\t\tself.norm1 = nn.LayerNorm(embed_dim)\n\t\tself.norm2 = nn.LayerNorm(out_dim)\n\n\tdef forward(self, x, x_mask=None):\n\t\tx = x + self.attn((self.norm1(x)), x_mask)\n\t\tx = x + self.ffn((self.norm2(x)))\n\t\treturn x\n    \nclass MyMultiHeadAttention(nn.Module):\n\tdef __init__(self,\n\t\t\tembed_dim,\n\t\t\tout_dim,\n\t\t\tqk_dim,\n\t\t\tv_dim,\n\t\t\tnum_head,\n\t\t):\n\t\tsuper().__init__()\n\t\tself.embed_dim = embed_dim\n\t\tself.num_head  = num_head\n\t\tself.qk_dim = qk_dim\n\t\tself.v_dim  = v_dim\n\n\t\tself.q = nn.Linear(embed_dim, qk_dim*num_head)\n\t\tself.k = nn.Linear(embed_dim, qk_dim*num_head)\n\t\tself.v = nn.Linear(embed_dim, v_dim*num_head)\n\n\t\tself.out = nn.Linear(v_dim*num_head, out_dim)\n\t\tself.scale = 1/(qk_dim**0.5)\n\n\t#https://github.com/pytorch/pytorch/issues/40497\n\tdef forward(self, x, x_mask):\n\t\tB,L,dim = x.shape\n\t\t#out, _ = self.mha(x,x,x, key_padding_mask=x_mask)\n\t\tnum_head = self.num_head\n\t\tqk_dim = self.qk_dim\n\t\tv_dim = self.v_dim\n\n\t\tq = self.q(x)\n\t\tk = self.k(x)\n\t\tv = self.v(x)\n\t\tq = q.reshape(B, L, num_head, qk_dim).permute(0,2,1,3).contiguous()\n\t\tk = k.reshape(B, L, num_head, qk_dim).permute(0,2,3,1).contiguous()\n\t\tv = v.reshape(B, L, num_head, v_dim ).permute(0,2,1,3).contiguous()\n\n\t\tdot = torch.matmul(q, k) *self.scale  # H L L\n\t\tx_mask = x_mask.reshape(B,1,1,L).expand(-1,num_head,L,-1)\n\t\t#dot[x_mask]= -1e4\n\t\tdot.masked_fill_(x_mask, -1e4)\n\t\tattn = F.softmax(dot, -1)    # L L\n\n\t\tv = torch.matmul(attn, v)  # L H dim\n\t\tv = v.permute(0,2,1,3).reshape(B,L, v_dim*num_head).contiguous()\n\t\tout = self.out(v)\n\n\t\treturn out\n    \n\nclass FeedForward(nn.Module):\n\tdef __init__(self, embed_dim, hidden_dim):\n\t\tsuper().__init__()\n\t\tself.mlp = nn.Sequential(\n\t\t\tnn.Linear(embed_dim, hidden_dim),\n\t\t\tnn.ReLU(inplace=True),\n\t\t\tnn.Linear(hidden_dim, embed_dim),\n\t\t)\n\tdef forward(self, x):\n\t\treturn self.mlp(x)\n    \nclass Net(nn.Module):\n\n\tdef __init__(self, num_class=num_class):\n\t\tsuper().__init__()\n\t\tself.output_type = ['inference', 'loss']\n\n\t\tself.x_embed = XEmbed()\n\n\t\tpos_embed = positional_encoding(max_length, embed_dim)\n\t\tself.pos_embed = nn.Parameter(pos_embed) \n\t\tself.cls_embed = nn.Parameter(torch.zeros((1, embed_dim)))\n\n\t\tself.encoder = nn.ModuleList([\n\t\t\tTransformerBlock(\n\t\t\t\tembed_dim,\n\t\t\t\tnum_head,\n\t\t\t\tembed_dim,\n\t\t\t) for i in range(num_block)\n\t\t])\n\t\tself.logit = nn.Linear(embed_dim, num_class)\n\n\tdef forward(self, batch):\n\t\txyz = batch['xyz']\n\n\t\t#----\n\t\tx, x_mask = pack_seq(xyz)\n\t\tx, x_mask = self.x_embed(x, x_mask)\n\t\tB,L,_ = x.shape\n\n\t\tx = x + self.pos_embed[:L].unsqueeze(0)\n\t\tx = torch.cat([\n\t\t\tself.cls_embed.unsqueeze(0).repeat(B,1,1),\n\t\t\tx\n\t\t],1)\n\t\tx_mask = torch.cat([\n\t\t\ttorch.zeros(B,1).to(x_mask),\n\t\t\tx_mask\n\t\t],1)\n \n\t\tfor block in self.encoder:\n\t\t\tx = block(x,x_mask)\n\t\tx = F.dropout(x,p=0.4,training=self.training)\n \n\t\t#---\n\t\t#mask pool\n\t\tx_mask = x_mask.unsqueeze(-1)\n\t\tx_mask = 1-x_mask.float()\n\t\tlast = (x*x_mask).sum(1)/x_mask.sum(1)\n\t\tlogit = self.logit(last)\n\n\t\toutput = {}\n\t\tif 'loss' in self.output_type:\n\t\t\toutput['label_loss'] = F.cross_entropy(logit, batch['label'],label_smoothing=label_smoothing)#0.5\n\n\t\tif 'inference' in self.output_type:\n\t\t\toutput['sign'] = torch.softmax(logit,-1)\n\n\t\treturn output\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-03-30T12:31:34.087316Z","iopub.execute_input":"2023-03-30T12:31:34.087765Z","iopub.status.idle":"2023-03-30T12:31:34.123915Z","shell.execute_reply.started":"2023-03-30T12:31:34.087729Z","shell.execute_reply":"2023-03-30T12:31:34.122981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_check_net():\n\n\tlength = [12,16,20,180]\n\tbatch_size = len(length)\n\txyz = [\n\t\tnp.random.uniform(-1,1,(length[b],point_dim)) for b in range(batch_size)\n\t]\n\txyz_mask = [\n\t\tnp.random.uniform(0,1,(length[b])) for b in range(batch_size)\n\t]\n\t#---\n\tbatch = {\n\t\t'label' : torch.from_numpy( np.random.choice(250,(batch_size))).long(),#.cuda(),\n\t\t'xyz' : [torch.from_numpy(x).float() for x in xyz],\n\t\t'xyz_mask' : [torch.from_numpy(x).float() for x in xyz_mask],\n\t}\n\n\tnet = Net()#.cuda()\n\toutput = net(batch)\n\n\n\t#---\n\n\tprint('batch')\n\tfor k, v in batch.items():\n\t\tif k in ['label','x']:\n\t\t\tprint(f'{k:>32} : {v.shape} ')\n\t\tif k=='xyz':\n\t\t\tprint(f'{k:>32} : {v[0].shape} ')\n\t\t\tfor i in range(1,len(v)):\n\t\t\t\tprint(f'{\" \":>32} : {v[i].shape} ')\n\n\tprint('output')\n\tfor k, v in output.items():\n\t\tif 'loss' not in k:\n\t\t\tprint(f'{k:>32} : {v.shape} ')\n\tprint('loss')\n\tfor k, v in output.items():\n\t\tif 'loss' in k:\n\t\t\tprint(f'{k:>32} : {v.item()} ')\n            \nrun_check_net()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T12:31:34.125570Z","iopub.execute_input":"2023-03-30T12:31:34.126446Z","iopub.status.idle":"2023-03-30T12:31:34.394699Z","shell.execute_reply.started":"2023-03-30T12:31:34.126398Z","shell.execute_reply":"2023-03-30T12:31:34.393329Z"},"trusted":true},"execution_count":null,"outputs":[]}]}