{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#pytorch model\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.sampler import SequentialSampler\nimport pandas as pd\n\n#num_landmark = 543\nmax_length = 80\nnum_class  = 250\nnum_point  = 82  # LIP, LHAND, RHAND\n\ndef pack_seq(\n    seq,\n):\n    length = [len(s) for s in seq]\n    batch_size = len(seq)\n    num_landmark=seq[0].shape[1]\n\n    x = torch.zeros((batch_size, max(length), num_landmark, 3)).to(seq[0].device)\n    x_mask = torch.zeros((batch_size, max(length))).to(seq[0].device)\n    for b in range(batch_size):\n        L = length[b]\n        x[b, :L] = seq[b][:L]\n        x_mask[b, L:] = 1\n    x_mask = (x_mask>0.5)\n    x = x.reshape(batch_size,-1,num_landmark*3)\n    return x, x_mask\n\n\nclass FeedForward(nn.Module):\n    def __init__(self, embed_dim, hidden_dim):\n        super().__init__()\n        self.mlp = nn.Sequential(\n            nn.Linear(embed_dim, hidden_dim),\n            nn.ReLU(inplace=True),\n            nn.Linear(hidden_dim, embed_dim),\n        )\n    def forward(self, x):\n        return self.mlp(x)\n\n\n#https://pytorch.org/docs/stable/generated/torch.nn.MultiheadAttention.html\nclass MultiHeadAttention(nn.Module):\n    def __init__(self,\n            embed_dim,\n            num_head,\n            batch_first,\n        ):\n        super().__init__()\n        self.mha = nn.MultiheadAttention(\n            embed_dim,\n            num_heads=num_head,\n            bias=True,\n            add_bias_kv=False,\n            kdim=None,\n            vdim=None,\n            dropout=0.0,\n            batch_first=batch_first,\n        )\n\n    def forward(self, x, x_mask):\n        out, _ = self.mha(x,x,x, key_padding_mask=x_mask)\n        return out\n\n\ndef positional_encoding(length, embed_dim):\n    dim = embed_dim//2\n\n    position = np.arange(length)[:, np.newaxis]     # (seq, 1)\n    dim = np.arange(dim)[np.newaxis, :]/dim   # (1, dim)\n\n    angle = 1 / (10000**dim)         # (1, dim)\n    angle = position * angle    # (pos, dim)\n\n    pos_embed = np.concatenate(\n        [np.sin(angle), np.cos(angle)],\n        axis=-1\n    )\n    pos_embed = torch.from_numpy(pos_embed).float()\n    return pos_embed\n\nclass TransformerBlock(nn.Module):\n    def __init__(self,\n        embed_dim,\n        num_head,\n        out_dim,\n        batch_first=True,\n    ):\n        super().__init__()\n        self.attn  = MultiHeadAttention(embed_dim, num_head,batch_first)\n        self.ffn   = FeedForward(embed_dim, out_dim)\n        self.norm1 = nn.LayerNorm(embed_dim)\n        self.norm2 = nn.LayerNorm(out_dim)\n\n    def forward(self, x, x_mask):\n        x = x + self.attn((self.norm1(x)), x_mask)\n        x = x + self.ffn((self.norm2(x)))\n        return x\n\nclass Net(nn.Module):\n\n    def __init__(self, num_class=num_class):\n        super().__init__()\n        self.output_type = ['inference', 'loss']\n\n        num_block = 1\n        embed_dim = 1024\n        num_head  = 8\n\n        pos_embed = positional_encoding(max_length, embed_dim)\n        # self.register_buffer('pos_embed', pos_embed)\n        self.pos_embed = nn.Parameter(pos_embed)\n\n        self.cls_embed = nn.Parameter(torch.zeros((1, embed_dim)))\n        self.x_embed = nn.Sequential(\n            nn.Linear(num_point * 3, embed_dim, bias=False),\n        )\n\n        self.encoder = nn.ModuleList([\n            TransformerBlock(\n                embed_dim,\n                num_head,\n                embed_dim,\n            ) for i in range(num_block)\n        ])\n        self.logit = nn.Linear(embed_dim, num_class)\n\n    def forward(self, batch):\n        length = [len(x) for x in batch['xyz']]\n        xyz = batch['xyz']\n        \n\n        x, x_mask = pack_seq(xyz)\n        B,L,_ = x.shape\n        x = self.x_embed(x)\n        x = x + self.pos_embed[:L].unsqueeze(0)\n\n        x = torch.cat([\n            self.cls_embed.unsqueeze(0).repeat(B,1,1),\n            x\n        ],1)\n        x_mask = torch.cat([\n            torch.zeros(B,1).to(x_mask),\n            x_mask\n        ],1)\n\n\n        #x = F.dropout(x,p=0.25,training=self.training)\n        for block in self.encoder:\n            x = block(x,x_mask = x_mask)\n\n        cls = x[:,0]\n        cls = F.dropout(cls,p=0.4,training=self.training)\n        logit = self.logit(cls)\n\n        output = {}\n        if 'loss' in self.output_type:\n            output['label_loss'] = F.cross_entropy(logit, torch.tensor(batch['label']).to('cuda'))\n\n        if 'inference' in self.output_type:\n            output['sign'] = torch.softmax(logit,-1)\n\n        return output\n\nROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\nLIP = [\n            61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n            291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n            78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n            95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n        ]\nLHAND = np.arange(468, 489).tolist()\nRHAND = np.arange(522, 543).tolist()\ndef pre_process(xyz):\n    xyz = xyz - xyz[~torch.isnan(xyz)].mean(0,keepdims=True) #noramlisation to common mean\n    xyz = xyz / xyz[~torch.isnan(xyz)].std(0, keepdims=True)\n    \n    lip = xyz[:, LIP]\n    lhand = xyz[:, LHAND]\n    rhand = xyz[:, RHAND]\n    xyz = torch.cat([ #(none, 82, 3)\n        lip,\n        lhand,\n        rhand,\n    ],1)\n    xyz[torch.isnan(xyz)] = 0\n    xyz = xyz[:max_length]\n    return xyz\n\n\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2023-04-12T11:30:12.950423Z","iopub.execute_input":"2023-04-12T11:30:12.950706Z","iopub.status.idle":"2023-04-12T11:30:15.642466Z","shell.execute_reply.started":"2023-04-12T11:30:12.950678Z","shell.execute_reply":"2023-04-12T11:30:15.641231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SignDataset(Dataset):\n    def __init__(self, df, augment=None):\n        self.df = df\n        self.augment = augment\n        self.length = len(self.df)\n\n    def __str__(self):\n        num_participant_id = self.df.participant_id.nunique()\n        string = ''\n        string += f'\\tlen = {len(self)}\\n'\n        string += f'\\tnum_participant_id = {num_participant_id}\\n'\n        return string\n\n    def __len__(self):\n        return self.length\n\n    def __getitem__(self, index):\n        d = self.df.iloc[index]\n\n        pq_file = f'/kaggle/input/asl-signs/{d.path}'\n        xyz = load_relevant_data_subset(pq_file)\n        xyz = xyz - xyz[~np.isnan(xyz)].mean(0,keepdims=True) #noramlisation to common maen\n        xyz = xyz / xyz[~np.isnan(xyz)].std(0, keepdims=True)\n\n        #--\n        if self.augment is not None:\n            xyz = self.augment(xyz)\n\n        #--\n        xyz = torch.from_numpy(xyz).float()\n        xyz = pre_process(xyz)\n\n        r = {}\n        r['index'] = index\n        r['d'    ] = d\n        r['xyz'  ] = xyz\n        r['label'] = d.label\n        return r","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2023-04-12T11:30:15.644723Z","iopub.execute_input":"2023-04-12T11:30:15.645241Z","iopub.status.idle":"2023-04-12T11:30:15.660206Z","shell.execute_reply.started":"2023-04-12T11:30:15.645200Z","shell.execute_reply":"2023-04-12T11:30:15.659069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\ndf = pd.read_csv(\"/kaggle/input/asl-signs/train.csv\")\nlabel_map = json.load(open(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\", \"r\"))\ndf['label'] = df['sign'].map(label_map)","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2023-04-12T11:30:15.663521Z","iopub.execute_input":"2023-04-12T11:30:15.666040Z","iopub.status.idle":"2023-04-12T11:30:15.913481Z","shell.execute_reply.started":"2023-04-12T11:30:15.666001Z","shell.execute_reply":"2023-04-12T11:30:15.912490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split, GroupKFold, StratifiedGroupKFold\ndf.head()\nrandom_df = df.sample(frac=1)\nsplit = round(len(random_df) * 0.8)\ntrain_df, valid_df = random_df[:split], random_df[split:]\ntrain_df = train_df.reset_index()\nvalid_df = valid_df.reset_index()","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2023-04-12T11:30:15.916279Z","iopub.execute_input":"2023-04-12T11:30:15.916952Z","iopub.status.idle":"2023-04-12T11:30:16.956705Z","shell.execute_reply.started":"2023-04-12T11:30:15.916914Z","shell.execute_reply":"2023-04-12T11:30:16.955580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = SignDataset(train_df)\nvalid_set = SignDataset(valid_df)","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2023-04-12T11:30:16.961651Z","iopub.execute_input":"2023-04-12T11:30:16.964196Z","iopub.status.idle":"2023-04-12T11:30:16.972035Z","shell.execute_reply.started":"2023-04-12T11:30:16.964157Z","shell.execute_reply":"2023-04-12T11:30:16.970352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tensor_key = ['xyz', 'label']\ndef null_collate(batch):\n    batch_size = len(batch)\n    d = {}\n    key = batch[0].keys()\n    for k in key:\n        d[k] = [b[k] for b in batch]\n    d['label'] = torch.LongTensor(d['label'])\n    return d","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2023-04-12T11:30:16.976552Z","iopub.execute_input":"2023-04-12T11:30:16.979609Z","iopub.status.idle":"2023-04-12T11:30:16.989159Z","shell.execute_reply.started":"2023-04-12T11:30:16.979542Z","shell.execute_reply":"2023-04-12T11:30:16.987951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"pycharm":{"name":"#%%\n"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(\n        train_set,\n        batch_size=64,\n        drop_last=False,\n        pin_memory=False,\n        collate_fn=null_collate,\n    )","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2023-04-12T11:30:16.994470Z","iopub.execute_input":"2023-04-12T11:30:16.997170Z","iopub.status.idle":"2023-04-12T11:30:17.006076Z","shell.execute_reply.started":"2023-04-12T11:30:16.997135Z","shell.execute_reply":"2023-04-12T11:30:17.004862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_loader = DataLoader(\n        valid_set,\n        batch_size=64,\n        drop_last=False,\n        pin_memory=False,\n        collate_fn=null_collate,\n    )","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2023-04-12T11:30:17.011449Z","iopub.execute_input":"2023-04-12T11:30:17.013505Z","iopub.status.idle":"2023-04-12T11:30:17.021413Z","shell.execute_reply.started":"2023-04-12T11:30:17.013467Z","shell.execute_reply":"2023-04-12T11:30:17.020227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from time import time\nimport os\nfold_dir = '/kaggle/working'\ndef np_cross_entropy(probability, truth):\n    p = np.clip(probability, 1e-4, 1 - 1e-4)\n    logp = -np.log(p)\n    loss = logp[np.arange(len(logp)), truth]\n    loss = loss.mean()\n    return loss\ndef do_valid(net, valid_loader, iteration):\n\n    valid_num = 0\n    valid_sign = []\n    valid_loss = 0\n\n    net = net.eval()\n    start_timer = time()\n    for t, batch in enumerate(valid_loader):\n        net.output_type = ['inference']\n        with torch.no_grad():\n            with torch.cuda.amp.autocast(enabled=True):\n                batch_size = len(batch['index'])\n                batch['xyz'] = [xyz.cuda() for xyz in batch['xyz']]\n                batch['label'] = [label.cuda() for label in batch['label']]\n                output = net(batch)  # data_parallel(net, batch) #\n\n        valid_sign.append(output['sign'].cpu().numpy())\n        valid_num += batch_size\n\n                # ---\n        print('\\r %8d / %d  %s' % (\n        valid_num, len(valid_loader.dataset), str(time() - start_timer)), end='', flush=True)\n            # if valid_num==200*4: break\n\n            # print('')\n    print(f'valid_num: {valid_num}')\n    print(f'val_dataset size: {len(valid_loader.dataset)}' )\n    assert (valid_num == len(valid_loader.dataset))\n            # ------\n    truth = valid_loader.dataset.df.label.values\n    sign = np.concatenate(valid_sign)\n    predict = np.argsort(-sign, -1)\n    correct = predict == truth.reshape(valid_num, 1)\n    topk = correct.cumsum(-1).mean(0)[:5]\n\n    loss = np_cross_entropy(sign, truth)\n\n    return [loss, topk[0], topk[1], topk[4]]\n        \ndef run_train():\n    lr = 1e-4  # 0.0001\n    skip_save_epoch = 0\n    num_epoch = 50\n    ## setup  ----------------------------------------\n    for f in ['checkpoint', 'train', 'valid', 'backup']: os.makedirs(fold_dir + '/' + f, exist_ok=True)\n    # backup_project_as_zip(PROJECT_PATH, fold_dir +'/backup/code.train.%s.zip'%IDENTIFIER)\n\n    scaler = torch.cuda.amp.GradScaler(enabled=True)\n    net = Net()\n    # net.load_pretrain()\n\n#     initial_checkpoint = None #fold_dir +'/checkpoint/00072000.model.pth' #\n#     if initial_checkpoint is not None:\n#         f = torch.load(initial_checkpoint, map_location=lambda storage, loc: storage)\n#         start_iteration = f.get('iteration', 0)\n#         start_epoch = f.get('epoch', 0)\n#         state_dict = f['state_dict']\n#        print(net.load_state_dict(state_dict, strict=False))  # True\n\n    start_iteration = 0\n    start_epoch = 0\n\n    net.cuda()\n\n    ## optimiser ----------------------------------\n#     if 0:  ##freeze\n#         for p in net.encoder.parameters():   p.requires_grad = False\n#         # for p in net.decoder.parameters():   p.requires_grad = False\n#         pass\n\n#     def freeze_bn(net):\n#         for m in net.modules():\n#             if isinstance(m, nn.BatchNorm2d):\n#                 m.eval()\n#                 m.weight.requires_grad = False\n#                 m.bias.requires_grad = False\n\n    # freeze_bn(net)\n\n    # -----------------------------------------------\n\n    optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, net.parameters()), lr=lr)\n    # optimizer = Lookahead(RAdam(filter(lambda p: p.requires_grad, net.parameters()),lr=start_lr), alpha=0.5, k=5)\n\n    num_iteration = num_epoch * len(train_loader)\n    iter_log = len(train_loader) * 3\n    iter_valid = iter_log\n    iter_save = iter_log\n\n    ## start training here! ##############################################\n    # array([0.57142857, 0.42857143])\n    def message(mode='print'):\n        asterisk = ' '\n        if mode == ('print'):\n            loss = batch_loss\n        if mode == ('log'):\n            loss = train_loss\n            if (iteration % iter_save == 0): asterisk = '*'\n\n#         text = \\\n#             ('iteration: %08d%s epoch: %6.2f | ' % (rate, iteration, asterisk, epoch,)).replace('e-0', 'e-').replace('e+0',\n#                                                                                                            'e+') + \\\n#             'val loss: %4.3f  %4.3f  %4.4f  %4.3f   | ' % (*valid_loss,) + \\\n#             'loss: %4.3f  %4.3f  %4.3f  | ' % (*loss,) + \\\n#             'time: %s' % str(time() - start_timer)\n        text = f'learning_rate: {lr:.4f}, iteration: {iteration:.4f}, epoch:{epoch:.4f}, val loss: {valid_loss}, loss: {loss.round(decimals=4)}, time: {str(time() - start_timer)}'\n        return text\n\n# ----\n    valid_loss = np.zeros(4, np.float32)\n    train_loss = np.zeros(3, np.float32)\n    batch_loss = np.zeros_like(train_loss)\n    sum_train_loss = np.zeros_like(train_loss)\n    sum_train = 0\n\n    start_timer = time()\n    iteration = start_iteration\n    epoch = start_epoch\n    while iteration < num_iteration:\n        for t, batch in enumerate(train_loader):\n\n            if iteration % iter_save == 0:\n                if iteration != start_iteration:\n                    n = iteration if epoch > skip_save_epoch else 0\n                    torch.save({\n                        'state_dict': net.state_dict(),\n                        'iteration': iteration,\n                        'epoch': epoch,\n                    }, fold_dir + f'/checkpoint/{n:08d}.model.pth')\n                    pass\n\n            if (iteration % iter_valid == 0):  # or (t==len(train_loader)-1):\n                if iteration != start_iteration:\n                    valid_loss = do_valid(net, valid_loader, f'{iteration:08d}')  #\n                pass\n\n            if (iteration % iter_log == 0) or (iteration % iter_valid == 0):\n                print('\\r')\n\n            # learning rate schduler ------------\n            # adjust_learning_rate(optimizer, scheduler(epoch))\n            # scheduler.get_last_lr()[0] #get_learning_rate(optimizer)\n\n            # one iteration update  -------------\n            batch_size = len(batch['index'])\n            batch['xyz'] = [xyz.cuda() for xyz in batch['xyz']]\n            batch['label'] = [label.cuda() for label in batch['label']]\n            net.train()\n            net.output_type = ['loss', 'inference']\n            # with torch.autograd.set_detect_anomaly(True):\n            \n            output = net(batch)\n            loss0 = output['label_loss'].mean()\n\n            optimizer.zero_grad()\n            scaler.scale(loss0).backward()\n\n            scaler.step(optimizer)\n            scaler.update()\n\n            # print statistics  --------\n            batch_loss[:3] = [loss0.item(), 0, 0]\n            sum_train_loss += batch_loss\n            sum_train += 1\n            if t % 100 == 0:\n                train_loss = sum_train_loss / (sum_train + 1e-12)\n                sum_train_loss[...] = 0\n                sum_train = 0\n\n            print('\\r', end='')\n            print(message(mode='print'))\n            start_timer = time()\n            epoch += 1 / len(train_loader)\n            iteration += 1\n\n        # debug  --------\n        # if 1:\n        # if t % 100 == 0:\n        # \tshow_result(batch, output, resize=0.50)\n        # \tcv2.waitKey(1)\n\n        torch.cuda.empty_cache()","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2023-04-12T11:30:17.024933Z","iopub.execute_input":"2023-04-12T11:30:17.025923Z","iopub.status.idle":"2023-04-12T11:30:17.058141Z","shell.execute_reply.started":"2023-04-12T11:30:17.025886Z","shell.execute_reply":"2023-04-12T11:30:17.056996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run_train()","metadata":{"scrolled":true,"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2023-04-12T11:30:17.061792Z","iopub.execute_input":"2023-04-12T11:30:17.062106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"pycharm":{"name":"#%%\n"}},"execution_count":null,"outputs":[]}]}