{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import jpx_tokyo_market_prediction","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-04T22:41:50.766732Z","iopub.execute_input":"2022-07-04T22:41:50.767133Z","iopub.status.idle":"2022-07-04T22:41:50.804722Z","shell.execute_reply.started":"2022-07-04T22:41:50.767047Z","shell.execute_reply":"2022-07-04T22:41:50.803822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## models.py\nfrom torch.nn.utils import weight_norm\nfrom torch.utils.tensorboard import SummaryWriter\nimport torch\nfrom torch import nn\nfrom torch.autograd import Variable\nimport pandas as pd\nimport numpy as np\nimport math\n# !pip install einops\n# from einops import rearrange\nfrom warnings import simplefilter\nsimplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n\n\n# The code for implementing the generic TCN model is taken from  https://github.com/locuslab/TCN\n\nclass Chomp1d(nn.Module):\n    def __init__(self, chomp_size):\n        super(Chomp1d, self).__init__()\n        self.chomp_size = chomp_size\n\n    def forward(self, x):\n        if self.chomp_size == 0:\n            result = x.contiguous()\n        else:\n            result = x[:, :, :-self.chomp_size].contiguous()\n        return result\n\n\nclass GenericTemporalConvNet(nn.Module):\n    def __init__(self, num_inputs, num_channels, kernel_size=2,\n                 runs_folder=None, mode=None, num_classes=1, gpu=True):\n        super(GenericTemporalConvNet, self).__init__()\n        self.mode = mode\n        if runs_folder is not None:\n            self.tb_writer = SummaryWriter(runs_folder, purge_step=0)\n        self.num_classes = num_classes\n        self.num_channels = num_channels\n        self.kernel_size = kernel_size\n        self.num_inputs = num_inputs\n        self.gpu = gpu\n        self.runs_folder = runs_folder\n\n        if mode == \"classification\":\n            self.last_layer = nn.Linear(in_features=self.num_channels[-1], out_features=num_classes)\n            self.activation = nn.LogSoftmax(dim=1)\n            if self.gpu:\n                self.last_layer.cuda()\n                self.activation.cuda()\n\n    def forward(self, x):\n        if self.mode == \"classification\":\n            output = self.network(x)\n            fc_output = self.last_layer(output[:, :, -1])\n            return self.activation(fc_output)\n        else:\n            return self.network(x)\n\n    def fit(self, num_epoch, train_loader, optimizer, clip, loss_function, save_every_epoch=20, model_path=None,\n            valid_loader=None, scheduler=None, print_every_epoch=20):\n        best_val_loss = float('inf')\n        old_lr = [group['lr'] for group in optimizer.param_groups][0]\n        receptive_field = self.kernel_size * (2 ** (len(self.num_channels) - 1))\n        print(\"Starting LR: \" + str(old_lr))\n        mean_abs_grads = []\n        max_abs_grads = []\n        zero_grad_percentages = []\n\n        for epoch in range(num_epoch):\n            mean_abs_grad = 0\n            max_abs_grad = 0\n            zero_grad_percentage = 0\n            num_all_grads = 0\n            self.train()\n            total_loss = 0\n            for batch_idx, (x_, y_) in enumerate(train_loader):\n                # print(batch_idx)\n                # if x_.shape[-1] < receptive_field:\n                #     x_ = torch.cat((x_, torch.zeros((x_.shape[0], x_.shape[1], receptive_field - x_.shape[2]))), dim=-1)\n                x_ = Variable(x_)\n                y_ = Variable(y_)\n                if self.gpu:\n                    x_ = x_.cuda()\n                    y_ = y_.cuda()\n                optimizer.zero_grad()\n                output = self(x_)\n                loss = loss_function(output, y_)\n                loss.backward()\n                for param in self.parameters():\n                    if param.grad is not None:\n                        mean_abs_grad = mean_abs_grad + torch.sum(torch.abs(param.grad))\n                        if torch.max(torch.abs(param.grad)) > max_abs_grad:\n                            max_abs_grad = torch.max(torch.abs(param.grad))\n                        zero_grad_percentage = zero_grad_percentage + torch.sum(param.grad == 0)\n                        num_all_grads = num_all_grads + param.grad.numel()\n                if clip > 0:\n                    torch.nn.utils.clip_grad_norm_(self.parameters(), clip)\n                optimizer.step()\n                total_loss += loss.item()\n            cur_loss = total_loss / (batch_idx + 1)\n            if (epoch + 1) % print_every_epoch == 0:\n                print(\"Epoch: \" + str(epoch))\n                print('Loss: {:.6f}'.format(cur_loss))\n            if self.tb_writer is not None:\n                self.tb_writer.add_scalar(\"training/loss\", cur_loss, epoch)\n            to_save = ((epoch + 1) % save_every_epoch == 0)\n            if to_save and model_path is not None:\n                torch.save(self.state_dict(), model_path + \"_\" + str(epoch))\n            if valid_loader is not None:\n                with torch.no_grad():\n                    self.eval()\n                    test_loss = 0\n                    correct = 0\n                    counter = 0\n                    for batch_idx, (x_, y_) in enumerate(valid_loader):\n                        # if x_.shape[-1] < receptive_field:\n                        #     x_ = torch.cat((x_, torch.zeros((x_.shape[0], x_.shape[1], receptive_field - x_.shape[2]))),\n                        #                    dim=-1)\n                        x_ = Variable(x_)\n                        y_ = Variable(y_)\n                        if self.gpu:\n                            x_ = x_.cuda()\n                            y_ = y_.cuda()\n                        output = self(x_)\n                        loss = loss_function(output, y_)\n                        if self.mode == \"classification\":\n                            pred = output.data.max(1, keepdim=True)[1]\n                            correct += pred.eq(y_.data.view_as(pred)).cpu().sum()\n                        test_loss += loss.item()\n                    if test_loss / (batch_idx + 1) < best_val_loss and model_path is not None:\n                        torch.save(self.state_dict(), model_path + \"_best\")\n                        best_val_loss = test_loss / (batch_idx + 1)\n                        print(\"Best model is saved, epoch: \" + str(epoch))\n                        print(best_val_loss)\n                    if self.tb_writer is not None:\n                        self.tb_writer.add_scalar(\"validation/loss\", test_loss / (batch_idx + 1), epoch)\n                        if self.mode == \"classification\":\n                            self.tb_writer.add_scalar(\"validation/accuracy\", 100. * correct / len(valid_loader.dataset),\n                                                      epoch)\n            if scheduler is not None and valid_loader is not None:\n                scheduler.step(test_loss / (batch_idx + 1))\n                lr = [group['lr'] for group in optimizer.param_groups][0]\n                if lr != old_lr:\n                    print(\"LR is modified, new LR: \" + str(lr) + \", step: \" + str(epoch))\n                    old_lr = lr\n            mean_abs_grads.append((mean_abs_grad / num_all_grads).cpu())\n            max_abs_grads.append(max_abs_grad.cpu())\n            zero_grad_percentages.append((zero_grad_percentage / num_all_grads * 100).cpu())\n        if self.tb_writer is not None:\n            self.tb_writer.flush()\n            self.tb_writer.close()\n        # print(mean_abs_grads)\n        # print(max_abs_grads)\n        # print(zero_grad_percentages)\n        mean_tensor = torch.stack(mean_abs_grads)\n        max_tensor = torch.stack(max_abs_grads)\n        percent_tensor = torch.stack(zero_grad_percentages)\n        mean_df = pd.DataFrame(mean_tensor.numpy())\n        max_df = pd.DataFrame(max_tensor.numpy())\n        percent_df = pd.DataFrame(percent_tensor.numpy())\n        mean_df.to_csv(self.runs_folder + '/mean.csv')\n        max_df.to_csv(self.runs_folder + '/max.csv')\n        percent_df.to_csv(self.runs_folder + '/percent.csv')\n\n\nclass TemporalBlock(nn.Module):\n    def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2):\n        super(TemporalBlock, self).__init__()\n        self.kernel_size = kernel_size\n\n        self.conv1 = weight_norm(nn.Conv1d(n_inputs, n_outputs, kernel_size,\n                                           stride=stride, padding=padding, dilation=dilation))\n        self.chomp1 = Chomp1d(padding)\n        self.relu1 = nn.ReLU()\n        self.dropout1 = nn.Dropout(dropout)\n\n        self.conv2 = weight_norm(nn.Conv1d(n_outputs, n_outputs, kernel_size,\n                                           stride=stride, padding=padding, dilation=dilation))\n        self.chomp2 = Chomp1d(padding)\n        self.relu2 = nn.ReLU()\n        self.dropout2 = nn.Dropout(dropout)\n\n        self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1,\n                                 self.conv2, self.chomp2, self.relu2, self.dropout2)\n        self.downsample = nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None\n        self.relu = nn.ReLU()\n        self.init_weights()\n\n    def init_weights(self):\n        self.conv1.weight.data.normal_(0, 0.01)\n        self.conv2.weight.data.normal_(0, 0.01)\n\n        if self.downsample is not None:\n            self.downsample.weight.data.normal_(0, 0.01)\n\n    def forward(self, x):\n        out = self.net(x)\n        res = x if self.downsample is None else self.downsample(x)\n        return self.relu(out + res)\n\n\nclass TemporalBlockLast(nn.Module):\n    def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2):\n        super(TemporalBlockLast, self).__init__()\n        self.kernel_size = kernel_size\n\n        self.conv1 = weight_norm(nn.Conv1d(n_inputs, n_outputs, kernel_size,\n                                           stride=stride, padding=padding, dilation=dilation))\n        self.chomp1 = Chomp1d(padding)\n        self.relu1 = nn.ReLU()\n        self.dropout1 = nn.Dropout(dropout)\n\n        self.conv2 = weight_norm(nn.Conv1d(n_outputs, n_outputs, kernel_size,\n                                           stride=stride, padding=padding, dilation=dilation))\n        self.chomp2 = Chomp1d(padding)\n        self.relu2 = nn.ReLU()\n        self.dropout2 = nn.Dropout(dropout)\n\n        self.net = nn.Sequential(self.conv1, self.chomp1, self.dropout1,\n                                 self.conv2, self.chomp2, self.dropout2)\n        self.downsample = nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None\n        self.relu = nn.ReLU()\n        self.init_weights()\n\n    def init_weights(self):\n        self.conv1.weight.data.normal_(0, 0.01)\n        self.conv2.weight.data.normal_(0, 0.01)\n\n        if self.downsample is not None:\n            self.downsample.weight.data.normal_(0, 0.01)\n\n    def forward(self, x):\n        out = self.net(x)\n        res = x if self.downsample is None else self.downsample(x)\n        # aoutnp = out.detach().cpu().numpy()\n        return out + res\n\n\nclass TemporalConvNet(GenericTemporalConvNet):\n    def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2,\n                 runs_folder=None, mode=None, num_classes=1, gpu=True):\n        super().__init__(num_inputs, num_channels, kernel_size=kernel_size,\n                         runs_folder=runs_folder, mode=mode, num_classes=num_classes, gpu=gpu)\n        layers = []\n        num_levels = len(num_channels)\n        for i in range(num_levels):\n            dilation_size = 2 ** i\n            in_channels = num_inputs if i == 0 else num_channels[i - 1]\n            out_channels = num_channels[i]\n            if i == num_levels - 1:\n                layers += [TemporalBlockLast(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size,\n                                             padding=(kernel_size - 1) * dilation_size, dropout=dropout)]\n            else:\n                layers += [TemporalBlock(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size,\n                                         padding=(kernel_size - 1) * dilation_size, dropout=dropout)]\n\n        self.network = nn.Sequential(*layers)\n\n\nclass PositionalEncoding(nn.Module):\n\n    def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000):\n        super().__init__()\n        self.dropout = nn.Dropout(p=dropout)\n\n        position = torch.arange(max_len).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))\n        pe = torch.zeros(max_len, 1, d_model)\n        pe[:, 0, 0::2] = torch.sin(position * div_term)\n        pe[:, 0, 1::2] = torch.cos(position * div_term)\n        self.register_buffer('pe', pe)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        \"\"\"\n        Args:\n            x: Tensor, shape [seq_len, batch_size, embedding_dim]\n        \"\"\"\n        pe = self.pe.permute(1, 2, 0)[:, :, :x.shape[-1]]\n        x = torch.cat((x, pe.repeat(x.shape[0], 1, 1)), dim=1)\n        return x  # self.dropout(x)\n\n\nclass TemporalTransformerBlock(nn.Module):\n    def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2, n_heads=1,\n                 canonical=True, convolve=True, gpu=True):\n        super(TemporalTransformerBlock, self).__init__()\n        self.canonical = canonical\n        self.dilation = dilation\n        self.kernel_size = kernel_size\n        # self.num_channels = n_outputs\n        self.gpu = gpu\n\n        self.dk = 1\n        self.dv = n_outputs\n        self.num_heads = n_heads\n\n        self.num_channels = self.dv + 2 * self.num_heads\n        self.convolve = convolve\n\n        if self.convolve:\n            self.conv = weight_norm(nn.Conv1d(n_inputs, self.num_channels, kernel_size,\n                                              stride=stride, padding=padding, dilation=dilation))\n            self.chomp = Chomp1d(padding)\n        else:\n            self.conv = weight_norm(nn.Conv1d(n_inputs, self.num_channels, 1,\n                                              stride=stride, padding=0, dilation=dilation))\n            self.chomp = Chomp1d(0)\n        self.dropout = nn.Dropout(dropout)\n        self.relu1 = nn.ReLU()\n\n        # self.conv2 = weight_norm(nn.Conv1d(self.num_channels, self.num_channels, kernel_size,\n        #                                   stride=stride, padding=padding, dilation=dilation))\n        # self.chomp2 = Chomp1d(padding)\n        # self.dropout2 = nn.Dropout(dropout)\n        # self.relu2 = nn.ReLU()\n\n        self.softmax = nn.Softmax(dim=-1)\n        self.relu = nn.ReLU()\n\n        # self.lastconv = weight_norm(nn.Conv1d(self.kernel_size, 1, self.kernel_size, padding=self.kernel_size - 1))\n        self.lastconv_w = torch.diag(torch.ones(self.kernel_size) / self.kernel_size).unsqueeze(0)\n        if self.gpu:\n            self.lastconv_w = self.lastconv_w.cuda()\n        self.transconv_w = torch.eye(self.kernel_size).unsqueeze(0).repeat(self.num_channels, 1, 1)\n        if self.gpu:\n            self.transconv_w = self.transconv_w.cuda()\n        self.lastchomp = Chomp1d(self.kernel_size - 1)\n\n        self.downsample = nn.Conv1d(n_inputs, self.dv * self.num_heads, 1) if n_inputs != self.dv * self.num_heads else None\n        self.init_weights()\n        self.multihead_linear = nn.Linear(in_features=self.num_heads * self.dv, out_features=self.dv)\n        self.self_attn_norm = nn.LayerNorm(self.dv)\n\n    def init_weights(self):\n        self.conv.weight.data.normal_(0, 0.01)\n        # self.conv2.weight.data.normal_(0, 0.01)\n        # self.lastconv.weight.data.copy_(\n        #     torch.diag(torch.ones(self.kernel_size) / self.kernel_size))\n\n        if self.downsample is not None:\n            self.downsample.weight.data.normal_(0, 0.01)\n\n    def forward(self, x):\n        res = x if self.downsample is None else self.downsample(x)\n\n        conv_out = self.dropout(self.chomp(self.conv(x)))\n        # conv_out = self.dropout2(self.chomp2(self.conv2(conv_out)))\n        batch_size = conv_out.shape[0]\n        length = conv_out.shape[-1]\n\n        # conv_out = rearrange(conv_out, 'b (h o) l -> b h o l', h=self.num_heads)\n        if self.canonical:\n            q = conv_out[:, :self.num_heads, :].unsqueeze(2)\n            k = conv_out[:, self.num_heads:2 * self.num_heads, :].unsqueeze(2)\n            v = conv_out[:, -self.dv:, :].unsqueeze(1)\n            k = k.permute(0, 1, 3, 2)\n            attn_weights = torch.matmul(k, q)\n            # attn_weights = torch.einsum('b h k i, b h k j -> b h i j', k, q)\n            attn_probs = self.softmax(attn_weights)\n            out = torch.matmul(v, attn_probs)\n            # out = torch.einsum('b h v i, b h i j -> b h v j', v, attn_probs)\n            # v.shape:\n            # torch.Size([32, 1, 6, 784])\n            # attn_probs.shape:\n            # torch.Size([32, 2, 784, 784])\n            # out.shape\n            # torch.Size([32, 2, 6, 784])\n            # Here out has shape of batch, num_heads, dv, length! (Linear will be on num_heads!)\n            # Down will be changed!\n            # aoutnp = out.detach().cpu().numpy()\n            out = out.unsqueeze(-2)\n#             out = rearrange(out, \"b h v k l -> (b h v) k l\")\n            out = out.view(-1, out.shape[-2], out.shape[-1])\n            # print(\"Canonical!\")\n        else:\n            # new_k = torch.zeros_like(k)\n            # new_q = torch.zeros_like(q)\n            # new_v = torch.zeros_like(v)\n            # for j in range(self.dilation):\n            #     new_k[:, :, int(length / self.dilation) * j:int(length / self.dilation) * (j + 1)] = k[:, :,\n            #                                                                                       j::self.dilation]\n            #     new_q[:, :, int(length / self.dilation) * j:int(length / self.dilation) * (j + 1)] = q[:, :,\n            #                                                                                       j::self.dilation]\n            #     new_v[:, :, int(length / self.dilation) * j:int(length / self.dilation) * (j + 1)] = v[:, :,\n            #                                                                                       j::self.dilation]\n            # k = new_k\n            # q = new_q\n            # v = new_v\n            # for i in range(self.kernel_size):\n            #     if i == 0:\n            #         new_conv_out2 = conv_out.unsqueeze(0)\n            #     else:\n            #         new_conv_out2 = torch.cat((nn.functional.pad(conv_out, [self.dilation * i, 0])[:, :, :, :-self.dilation * i].unsqueeze(0), new_conv_out2), dim=0)\n            # new_conv_out = torch.zeros((conv_out.shape[0], conv_out.shape[1], self.kernel_size,\n            #                             conv_out.shape[2]))\n            # if self.gpu:\n            #     new_conv_out = new_conv_out.cuda()\n            # for i in range(self.kernel_size):\n            #     if i == 0:\n            #         new_conv_out[:, :, i, :] = conv_out[:, :, :]\n            #     else:\n            #         new_conv_out[:, :, i, self.dilation * i:] = conv_out[:, :, :-self.dilation * i]\n            # if self.dilation == 2 ** 6:\n            #     print(\"Stop!\")\n            # print(conv_out.shape)\n            conv_out = torch.nn.functional.conv_transpose1d(conv_out, self.transconv_w, dilation=self.dilation,\n                                                            groups=self.num_channels)\n#             conv_out = rearrange(conv_out, \"b (c k) l -> b c k l\", c=self.num_channels)\n            conv_out = conv_out.view(conv_out.shape[0], self.num_channels, -1, conv_out.shape[-1])\n            conv_out = conv_out[:, :, :, :length]\n            conv_out = torch.flip(conv_out, [-2])\n            # newnp = new_conv_out.detach().cpu().numpy()\n            # convnp = conv_out.detach().cpu().numpy()\n            conv_out = conv_out.permute(2, 0, 1, 3)\n            q = conv_out[:, :, :self.num_heads, :].unsqueeze(0)\n            k = conv_out[:, :, self.num_heads:2 * self.num_heads, :].unsqueeze(0)\n            v = conv_out[:, :, -self.dv:, :].unsqueeze(0)\n            k = k.permute(2, 3, 4, 1, 0)\n            q = q.permute(2, 3, 4, 0, 1)\n            # knp = k[0, 0, :, :, 0].detach().cpu().numpy()\n            # qnp = q[0, 0, :, 0, :].detach().cpu().numpy()\n            # k = k.view(k.shape[0], -1, self.kernel_size, k.shape[2])\n            # q = q.view(q.shape[0], -1, q.shape[1], self.kernel_size)\n            attn_weights = torch.matmul(k, q)  # / math.sqrt(self.dk)\n            # attn_weights = torch.einsum(\"i b h k l, j b h k l -> b h i j l\", k, q)\n            attn_probs = self.softmax(attn_weights)\n            # v = v.view(v.shape[0], -1, v.shape[1], self.kernel_size)\n            v = v.permute(2, 0, 4, 3, 1)\n            # v = self.relu(v)\n            # vnp = v[0, 0, :, 0, :].detach().cpu().numpy()\n            out = torch.matmul(v, attn_probs)\n            # This matmul doesn't work!\n            # out = torch.einsum(\"k b h v l, b h k j l -> b h v j l\", v, attn_probs)\n            # out = out.view(batch_size, self.num_channels, -1)\n\n            # Here out has shape of torch.Size([32, 2, 784, 6, 8]):\n            # batch, num heads, length, dv, kernel.\n\n#             out = rearrange(out, \"b h l v k -> (b h v) k l\")\n            out = out.view(-1, out.shape[-1], out.shape[-3])\n            # aoutnp1 = out.detach().cpu().numpy()\n            out = torch.flip(out, [-2])\n            # aoutnp2 = out.detach().cpu().numpy()\n            # Kernel smoothing with last conv, due to conv. local attention, we should reduce dim. due to kernel size!\n            out = nn.functional.conv1d(out, self.lastconv_w, padding=self.kernel_size - 1)\n            # print(out.shape)\n            out = self.lastchomp(out)\n            # print(out.shape)\n            # out = self.lastchomp(self.lastconv(out))\n            # aoutnp3 = out.detach().cpu().numpy()\n            # out = torch.sum(out, dim=-2, keepdim=False) / self.kernel_size\n            # out = nn.functional.pad(out, [self.kernel_size, self.kernel_size])\n            # out = torch.flip(out, [-2])\n            # new_out = torch.empty((out.shape[0], out.shape[1], out.shape[2], out.shape[-1] - self.kernel_size))\n            # if self.gpu:\n            #     new_out = new_out.cuda()\n            # # for i in range(new_out.shape[-1]):\n            # #     new_out[:, :, :, i] = torch.sum(torch.diagonal(out, offset=i, dim1=-2, dim2=-1), dim=-1, keepdim=False) / self.kernel_size\n            # # out = new_out[:, :, :, -length:]\n        # Here, out should have the shape (b h v) k l: batch*num_heads*dv, 1, length = torch.Size([384, 1, 784])\n#         out = rearrange(out, \"(b h v) k l -> b (v h k) l\", h=self.num_heads, v=self.dv)\n        out = out.view(-1, self.num_heads, self.dv, out.shape[-2], out.shape[-1])\n        out = out.view(out.shape[0], -1, out.shape[-1])\n        out = out + res\n        out = self.multihead_linear(out.permute(0, 2, 1))\n        out = self.self_attn_norm(out).permute(0, 2, 1)\n        # k = 1 in rearranges here.\n        # result = rearrange(result, \"(b v) k l -> b (v k) l\", b=batch_size, v=self.dv)\n        # aresultnp = result.detach().cpu().numpy()\n        # aprob = attn_probs[0].detach().cpu().numpy()\n        return out\n\n\nclass TemporalTransformer(GenericTemporalConvNet):\n    def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2,\n                 runs_folder=None, mode=None, num_classes=1, gpu=True, canonical=True, n_heads=1, pos=0, conv=True):\n        super().__init__(num_inputs, num_channels, kernel_size=kernel_size,\n                         runs_folder=runs_folder, mode=mode, num_classes=num_classes, gpu=gpu)\n        layers = []\n        if pos > 0:\n            layers += [PositionalEncoding(d_model=pos, dropout=dropout, max_len=1000)]\n        num_levels = len(num_channels)\n        for i in range(num_levels):\n            dilation_size = 2 ** i\n            in_channels = num_inputs + pos if i == 0 else num_channels[i - 1]\n            out_channels = num_channels[i]\n            layers += [\n                TemporalTransformerBlock(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size,\n                                         padding=(kernel_size - 1) * dilation_size, dropout=dropout,\n                                         canonical=canonical, gpu=gpu, n_heads=n_heads, convolve=conv)]\n\n        self.network = nn.Sequential(*layers)\n\n\nclass GatedTemporalBlock(nn.Module):\n    def __init__(\n            self,\n            n_inputs,\n            n_outputs,\n            kernel_size,\n            stride,\n            dilation,\n            padding,\n            dropout=0.1,\n    ):\n        super(GatedTemporalBlock, self).__init__()\n        self.kernel_size = kernel_size\n        self.conv1 = weight_norm(\n            nn.Conv1d(\n                2 * n_inputs,\n                2 * n_outputs,\n                kernel_size,\n                stride=stride,\n                padding=padding,\n                dilation=dilation,\n            )\n        )\n        self.chomp1 = Chomp1d(padding)\n        self.dropout1 = nn.Dropout(dropout)\n\n        self.conv2 = weight_norm(\n            nn.Conv1d(\n                2 * n_outputs,\n                2 * n_outputs,\n                kernel_size,\n                stride=stride,\n                padding=padding,\n                dilation=dilation,\n            )\n        )\n        self.chomp2 = Chomp1d(padding)\n        self.dropout2 = nn.Dropout(dropout)\n        self.downsample = (\n            nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None\n        )\n        self.init_weights()\n\n    def init_weights(self):\n        self.conv1.weight.data.normal_(0, 0.01)\n        self.conv2.weight.data.normal_(0, 0.01)\n\n        if self.downsample is not None:\n            self.downsample.weight.data.normal_(0, 0.1)\n\n    def forward(self, x):\n        hidden, gate = self.conv1(x).chunk(2, dim=1)\n        hidden = hidden * torch.sigmoid(gate)\n        hidden = torch.cat((hidden, gate), dim=1)\n        hidden = self.chomp1(hidden)\n        hidden = self.dropout1(hidden)\n        hidden, gate = self.conv2(hidden).chunk(2, dim=1)\n        hidden = hidden * torch.sigmoid(gate)\n        hidden = torch.cat((hidden, gate), dim=1)\n        hidden = self.chomp2(hidden)\n        hidden = self.dropout2(hidden)\n        res = x if self.downsample is None else self.downsample(x)\n        return hidden + res\n\n\nclass GatedTemporalBlockLast(nn.Module):\n    def __init__(\n            self,\n            n_inputs,\n            n_outputs,\n            kernel_size,\n            stride,\n            dilation,\n            padding,\n            dropout=0.1,\n    ):\n        super(GatedTemporalBlockLast, self).__init__()\n        self.kernel_size = kernel_size\n        self.conv1 = weight_norm(\n            nn.Conv1d(\n                2 * n_inputs,\n                2 * n_outputs,\n                kernel_size,\n                stride=stride,\n                padding=padding,\n                dilation=dilation,\n            )\n        )\n        self.chomp1 = Chomp1d(padding)\n        self.dropout1 = nn.Dropout(dropout)\n\n        self.conv2 = weight_norm(\n            nn.Conv1d(\n                2 * n_outputs,\n                2 * n_outputs,\n                kernel_size,\n                stride=stride,\n                padding=padding,\n                dilation=dilation,\n            )\n        )\n        self.chomp2 = Chomp1d(padding)\n        self.dropout2 = nn.Dropout(dropout)\n\n        self.downsample = (\n            nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None\n        )\n        self.relu = nn.ReLU()\n        self.init_weights()\n\n    def init_weights(self):\n        self.conv1.weight.data.normal_(0, 0.01)\n        self.conv2.weight.data.normal_(0, 0.01)\n\n        if self.downsample is not None:\n            self.downsample.weight.data.normal_(0, 0.1)\n\n    def forward(self, x):\n        hidden, gate = self.conv1(x).chunk(2, dim=1)\n        hidden = hidden * torch.sigmoid(gate)\n        hidden = torch.cat((hidden, gate), dim=1)\n        hidden = self.chomp1(hidden)\n        hidden = self.dropout1(hidden)\n        hidden, gate = self.conv2(hidden).chunk(2, dim=1)\n        hidden = hidden * torch.sigmoid(gate)\n        hidden = self.chomp2(hidden)\n        hidden = self.dropout2(hidden)\n        res = x.chunk(2, dim=1)[0] if self.downsample is None else self.downsample(x.chunk(2, dim=1)[0])\n        return hidden + res\n\n\nclass GatedTemporalBlockFirst(nn.Module):\n    def __init__(\n            self,\n            n_inputs,\n            n_outputs,\n            kernel_size,\n            stride,\n            dilation,\n            padding,\n            dropout=0.1,\n    ):\n        super(GatedTemporalBlockFirst, self).__init__()\n        self.kernel_size = kernel_size\n        self.conv1 = weight_norm(\n            nn.Conv1d(\n                n_inputs,\n                2 * n_outputs,\n                kernel_size,\n                stride=stride,\n                padding=padding,\n                dilation=dilation,\n            )\n        )\n        self.chomp1 = Chomp1d(padding)\n        self.dropout1 = nn.Dropout(dropout)\n\n        self.conv2 = weight_norm(\n            nn.Conv1d(\n                2 * n_outputs,\n                2 * n_outputs,\n                kernel_size,\n                stride=stride,\n                padding=padding,\n                dilation=dilation,\n            )\n        )\n        self.chomp2 = Chomp1d(padding)\n        self.dropout2 = nn.Dropout(dropout)\n\n        self.downsample = (\n            nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None\n        )\n        self.relu = nn.ReLU()\n        self.init_weights()\n\n    def init_weights(self):\n        self.conv1.weight.data.normal_(0, 0.01)\n        self.conv2.weight.data.normal_(0, 0.01)\n\n        if self.downsample is not None:\n            self.downsample.weight.data.normal_(0, 0.1)\n\n    def forward(self, x):\n        hidden, gate = self.conv1(x).chunk(2, dim=1)\n        hidden = hidden * torch.sigmoid(gate)\n        hidden = torch.cat((hidden, gate), dim=1)\n        hidden = self.chomp1(hidden)\n        hidden = self.dropout1(hidden)\n        hidden, gate = self.conv2(hidden).chunk(2, dim=1)\n        hidden = hidden * torch.sigmoid(gate)\n        hidden = torch.cat((hidden, gate), dim=1)\n        hidden = self.chomp2(hidden)\n        hidden = self.dropout2(hidden)\n        res = x if self.downsample is None else self.downsample(x)\n        return hidden + torch.cat((res, res), dim=1)\n\n\nclass GatedTemporalConvNet(GenericTemporalConvNet):\n    def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2,\n                 runs_folder=None, mode=None, num_classes=1, gpu=True):\n        super().__init__(num_inputs, num_channels, kernel_size=kernel_size,\n                         runs_folder=runs_folder, mode=mode, num_classes=num_classes, gpu=gpu)\n        layers = []\n        num_levels = len(num_channels)\n        for i in range(num_levels):\n            dilation_size = 2 ** i\n            in_channels = num_inputs if i == 0 else num_channels[i - 1]\n            out_channels = num_channels[i]\n            if i == num_levels - 1:\n                layers += [\n                    GatedTemporalBlockLast(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size,\n                                           padding=(kernel_size - 1) * dilation_size, dropout=dropout)]\n            elif i == 0:\n                layers += [\n                    GatedTemporalBlockFirst(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size,\n                                            padding=(kernel_size - 1) * dilation_size, dropout=dropout)]\n            else:\n                layers += [GatedTemporalBlock(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size,\n                                              padding=(kernel_size - 1) * dilation_size, dropout=dropout)]\n\n        self.network = nn.Sequential(*layers)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-04T22:41:50.849045Z","iopub.execute_input":"2022-07-04T22:41:50.849311Z","iopub.status.idle":"2022-07-04T22:41:53.121237Z","shell.execute_reply.started":"2022-07-04T22:41:50.849287Z","shell.execute_reply":"2022-07-04T22:41:53.120036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\n\ndef long_to_wide(df, norm=True, mins=None, maxs=None):\n    df = df.pivot(index='SecuritiesCode', columns='Date', values=df.columns[2:])\n    df.fillna(value=0, inplace=True)\n    df = df.stack(-2)\n    df = df.swaplevel(i=0, j=1)\n    if norm:\n        if mins is None:\n            mins = df.drop(\"Target\").swaplevel(i=0, j=1).min(axis=1).swaplevel(i=0, j=1)\n            maxs = df.drop(\"Target\").swaplevel(i=0, j=1).max(axis=1).swaplevel(i=0, j=1)\n        df.loc[[\"Open\", \"High\", \"Low\", \"Close\", \"Volume\"]] = df.loc[[\"Open\", \"High\", \"Low\", \"Close\", \"Volume\"]].subtract(mins, axis=0).div(maxs - mins, axis=0)\n        return df, mins, maxs\n    else:\n        return df\n\ndef generate_data(df, val_ratio=0.2, hbsize=1, sep=False):\n    df_X = pd.concat((df.loc[\"Target\"].shift(axis=1), df.drop(\"Target\")), axis=0).fillna(value=0)\n    df_y = df.loc[\"Target\"]\n    X = df_X.to_numpy().T\n    y = df_y.to_numpy().T\n    if sep:\n        X = X.reshape(-1, 6)\n        y = y.reshape(-1, 1)\n#     print(X.shape)\n    del df_X, df_y\n    list_X = []\n    for i in range(X.shape[0] - hbsize + 1):\n        list_X.append(X[i:i + hbsize])\n    X = np.asarray(list_X)\n#     X = X.reshape(X.shape[0]/5, X.shape[1]*5, X.shape[2])\n    del list_X\n    list_y = []\n    for i in range(y.shape[0] - hbsize + 1):\n        list_y.append(y[i:i + hbsize])\n    y = np.asarray(list_y)\n    del list_y\n    if val_ratio > 0:\n        X, val_X = train_test_split(X, test_size=val_ratio, shuffle=False)\n        y, val_y = train_test_split(y, test_size=val_ratio, shuffle=False)\n        return X, val_X, y, val_y\n    else:\n        return X, y\n    \ndef generate_loaders(X, val_X, y, val_y, batch_size=1):\n    X = torch.from_numpy(X).permute(0, 2, 1).type(torch.FloatTensor)\n    y = torch.from_numpy(y).permute(0, 2, 1).type(torch.FloatTensor)\n    train_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(X, y),\n                                               batch_size=batch_size, shuffle=True)\n    if val_X is not None:\n        val_X = torch.from_numpy(val_X).permute(0, 2, 1).type(torch.FloatTensor)\n        val_y = torch.from_numpy(val_y).permute(0, 2, 1).type(torch.FloatTensor)\n        val_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(val_X, val_y),\n                                             batch_size=batch_size, shuffle=False)\n    return train_loader, val_loader","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:41:53.123565Z","iopub.execute_input":"2022-07-04T22:41:53.126274Z","iopub.status.idle":"2022-07-04T22:41:54.188682Z","shell.execute_reply.started":"2022-07-04T22:41:53.126237Z","shell.execute_reply":"2022-07-04T22:41:54.187444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch.optim as optim\nimport torch.nn.functional as F\n\nif not os.path.exists(\"models\"):\n    os.mkdir(\"models\")\nif not os.path.exists(\"runs\"):\n    os.mkdir(\"runs\")","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:41:54.194330Z","iopub.execute_input":"2022-07-04T22:41:54.196839Z","iopub.status.idle":"2022-07-04T22:41:54.204780Z","shell.execute_reply.started":"2022-07-04T22:41:54.196766Z","shell.execute_reply":"2022-07-04T22:41:54.203826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/train_files/stock_prices.csv\", parse_dates=[\"Date\"])\ncols = [\"Date\", \"SecuritiesCode\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"Target\"]\ndf = prices[cols]\ndf, mins, maxs = long_to_wide(df, norm=True)\nhbsize = 128\nsep = True\nstock_list = df.loc[\"Open\"].index.tolist()\nmodel_name_list = []\ntest_X_list = []\nmodel_dict_list = []","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:41:54.210386Z","iopub.execute_input":"2022-07-04T22:41:54.213235Z","iopub.status.idle":"2022-07-04T22:42:03.856100Z","shell.execute_reply.started":"2022-07-04T22:41:54.213195Z","shell.execute_reply":"2022-07-04T22:42:03.854903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, stock in enumerate(stock_list):\n    if i % 100 == 0:\n        print(i)\n    train_X, val_X, train_y, val_y = generate_data(df.iloc[i*6:(i+1)*6], val_ratio=0.1, hbsize=hbsize, sep=sep)\n    train_loader, val_loader = generate_loaders(train_X, val_X, train_y, val_y, batch_size=int(2**18))\n    test_X_list.append(torch.tensor(val_X[-1:, 1:, :]).permute(0, 2, 1).type(torch.FloatTensor))\n\n    kernel_size = 8\n    tcn_channels = [[6] * 4, [1]]\n    clip = -1\n    lr = 5e-3\n    dropout = 0.2\n    scheduler_patience = 5\n    scheduler_factor = 0.3\n    scheduler_cooldown = 10\n    gpu = True\n    num_epochs = 1\n    model_folder = \"models\"\n    run_folder = \"runs\"\n    print_every_epoch = 500\n    save_every_epoch = 500\n    flat_tcn_channels = [item for sublist in tcn_channels for item in sublist]\n    num_inputs = train_X.shape[-1]\n    setting = f\"_{i}\"\n#     print(train_X.shape)\n    model_name = \"GTCN_\"\n    model_name += str(kernel_size) + \"_\" + str(flat_tcn_channels[0]) + \"x\" + str(len(flat_tcn_channels)) + \"_\"\n    model_name += str(clip) + \"_\" + str(lr) + \"_\" + str(scheduler_patience) + \"_\" + str(scheduler_factor) + \"_\"\n    model_name += str(scheduler_cooldown) + \"_\" + str(dropout)\n\n    model = GatedTemporalConvNet(num_inputs=num_inputs, num_channels=flat_tcn_channels, kernel_size=kernel_size,\n                                 dropout=dropout, runs_folder=os.path.join(run_folder, model_name + setting),\n                                 mode=None, num_classes=None, gpu=gpu)\n    if gpu:\n        model.cuda()\n    num_params = 0\n    for p in model.parameters():\n        num_params = num_params + p.numel()\n#     print(\"Num GTCN params: \" + str(num_params))\n    model_name += \"_\" + str(num_params)\n\n    optimizer = optim.Adam(params=model.parameters(), lr=lr)\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=scheduler_patience, factor=scheduler_factor,\n                                                     cooldown=scheduler_cooldown)\n    loss_function = F.mse_loss\n    model.fit(num_epoch=num_epochs, train_loader=train_loader, optimizer=optimizer, clip=clip,\n              loss_function=loss_function, save_every_epoch=save_every_epoch,\n              model_path=os.path.join(model_folder, model_name + setting),\n              valid_loader=val_loader, scheduler=scheduler, print_every_epoch=print_every_epoch)\n    model_name_list.append(model_name + setting + \"_best\")\n    PATH = f\"models/{model_name_list[i]}\"\n    model_dict_list.append(torch.load(PATH))","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:42:03.859392Z","iopub.execute_input":"2022-07-04T22:42:03.859778Z","iopub.status.idle":"2022-07-04T22:44:37.461987Z","shell.execute_reply.started":"2022-07-04T22:42:03.859742Z","shell.execute_reply":"2022-07-04T22:44:37.460894Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import jpx_tokyo_market_prediction\nenv = jpx_tokyo_market_prediction.make_env()\niter_test = env.iter_test()\nmodel.eval()\nmodel.cpu()\ncols = [\"Date\", \"SecuritiesCode\", \"Close\", \"High\", \"Low\", \"Open\", \"Volume\"]\n# PATH = f\"../kaggle/working/models/{model_name}_best\"\n# model.load_state_dict(torch.load(PATH))\noutputs = torch.zeros(2000)\ntest_targets = np.empty(2000).tolist()\nfor i in range(len(model_name_list)):\n    test_X = test_X_list[i]\n    test_targets[i] = torch.cat([torch.zeros((1, 1, 1)).type(torch.FloatTensor), test_X[:, :1, 1-hbsize:]], dim=2)\nfor (prices, options, financials, trades, secondary_prices, sample_prediction) in iter_test:\n    test_new_all = torch.tensor(long_to_wide(prices[cols], norm=True, mins=mins, maxs=maxs)[0].values).unsqueeze(0).type(torch.FloatTensor)\n    new_test_X_list = test_X_list.copy() \n    test_X_list = []\n    for i in range(len(model_name_list)):\n        test_X = new_test_X_list[i]\n        test_new = test_new_all[:, i*5:(i+1)*5, :]\n        test_X_feat = torch.cat([test_X[:, 1:, 1-hbsize:], test_new], dim=2)\n        test_X = torch.cat([test_targets[i], test_X_feat], dim=1)\n        test_X_list.append(test_X)\n        with torch.no_grad():\n            model.load_state_dict(model_dict_list[i])\n            outputs[i] = model(test_X)[0, 0, -1]\n            test_targets[i] = torch.cat([test_targets[i][:, :, 1-hbsize:], outputs[i].reshape(1, 1, -1)], dim=2)\n    sorted, indices = outputs.sort(descending=True)\n    rank_df = sample_prediction[\"Rank\"].copy()\n    rank_df.iloc[indices] = np.arange(len(sample_prediction))\n    sample_prediction[\"Rank\"] = rank_df\n    print(sample_prediction)\n    env.predict(sample_prediction)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:44:37.463298Z","iopub.execute_input":"2022-07-04T22:44:37.464702Z","iopub.status.idle":"2022-07-04T22:44:51.211016Z","shell.execute_reply.started":"2022-07-04T22:44:37.464658Z","shell.execute_reply":"2022-07-04T22:44:51.209948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices_test = pd.read_csv(\"../input/jpx-tokyo-stock-exchange-prediction/supplemental_files/stock_prices.csv\", parse_dates=[\"Date\"])\ncols = [\"Date\", \"SecuritiesCode\", \"Target\"]\ndf_test = prices_test[cols]","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:44:51.212449Z","iopub.execute_input":"2022-07-04T22:44:51.213036Z","iopub.status.idle":"2022-07-04T22:44:51.866694Z","shell.execute_reply.started":"2022-07-04T22:44:51.212996Z","shell.execute_reply":"2022-07-04T22:44:51.865684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"((df_test[df_test[\"Date\"] == \"2021-12-06\"][\"Target\"].values - np.array([test_target[0, 0, -2].numpy() for test_target in test_targets]))**2).mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:48:07.428379Z","iopub.execute_input":"2022-07-04T22:48:07.428849Z","iopub.status.idle":"2022-07-04T22:48:07.454250Z","shell.execute_reply.started":"2022-07-04T22:48:07.428809Z","shell.execute_reply":"2022-07-04T22:48:07.453334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"((df_test[df_test[\"Date\"] == \"2021-12-07\"][\"Target\"].values - np.array([test_target[0, 0, -1].numpy() for test_target in test_targets]))**2).mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T22:47:33.348593Z","iopub.execute_input":"2022-07-04T22:47:33.349029Z","iopub.status.idle":"2022-07-04T22:47:33.378679Z","shell.execute_reply.started":"2022-07-04T22:47:33.348994Z","shell.execute_reply":"2022-07-04T22:47:33.377769Z"},"trusted":true},"execution_count":null,"outputs":[]}]}