{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\n\ntorch.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n\nimport math\nimport psutil\nimport random\nimport pandas as pd\nimport numpy as np\nimport datatable as dt\nimport seaborn as sns\n\nfrom enum import IntEnum\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn.init import xavier_uniform_\nfrom torch.nn.init import constant_\nfrom torch.nn.init import xavier_normal_\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\nfrom pathlib import Path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_df = pd.read_pickle('/kaggle/input/data-plus/train.pkl')\n\nlen(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[:1000000]\nlen(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_df = train_df[train_df.content_type_id == False]\n\n#arrange by timestamp\ntrain_df = train_df.sort_values(['timestamp'], ascending=True).reset_index(drop = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df['timestamp']\ndel train_df['content_type_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train_df.iloc[:int(0.9 * len(train_df))]\ntest = train_df.iloc[int(0.9 * len(train_df)):]\nprint(f\"- Shape of Train dataset: {train.shape}\")\nprint(f\"- Shape of Test dataset: {test.shape}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_group = (\n    train[['user_id', 'content_id', 'answered_correctly']]\n    .groupby('user_id')\n    .apply(lambda r: (r['content_id'].values, r['answered_correctly'].values))\n)\n\ntest_group = (\n    test[['user_id', 'content_id', 'answered_correctly']]\n    .groupby('user_id')\n    .apply(lambda r: (r['content_id'].values, r['answered_correctly'].values))\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AKTDataset(Dataset):\n    def __init__(self, group, n_skills, max_seq=200, min_samples=1):\n        super(AKTDataset, self).__init__()\n        \n        self.max_seq = max_seq\n        self.n_skills = n_skills\n        self.samples = {}\n        self.user_ids = []\n        \n        for user_id in group.index:\n            q_, res_ = group[user_id]\n            if len(q_) < min_samples:\n                continue\n            \n            seq_length = len(q_)\n            if seq_length > self.max_seq:\n                initial = seq_length % self.max_seq\n                \n                if initial >= min_samples:\n                    self.user_ids.append(f\"{user_id}_0\")\n                    q = q_[:initial]\n                    res = res_[:initial]\n                    # qa = res.astype(int) * self.n_skills + q\n                    self.samples[f\"{user_id}_0\"] = (q, res)\n                    \n                for seq in range(seq_length // self.max_seq):\n                    self.user_ids.append(f\"{user_id}_{seq+1}\")\n                    start = initial + seq * self.max_seq\n                    end = start + self.max_seq\n                    q = q_[start:end]\n                    res = res_[start:end]\n                    # qa = res.astype(int) * self.n_skills + q\n                    self.samples[f\"{user_id}_{seq+1}\"] = (q, res)\n                    \n            else:\n                user_id = str(user_id)\n                self.user_ids.append(user_id)\n                self.samples[user_id] = (q_, res_)\n                \n    def __len__(self):\n        return len(self.user_ids)\n    \n    def __getitem__(self, index):\n        user_id = self.user_ids[index]\n        q_, res_ = self.samples[user_id]\n        seq_length = len(q_)\n        \n        q_ = q_ + 1\n        \n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n        res = np.zeros(self.max_seq, dtype=int)\n        \n        if seq_length >= self.max_seq:\n            q[:] = q_[-self.max_seq:]\n            res[:] = res_[-self.max_seq:]\n        else:\n            q[-seq_length:] = q_\n            res[-seq_length:] = res_\n        \n        q = q[1:]\n        res = res[1:]\n        qa = res.astype(int) * self.n_skills + q\n        \n        return q, qa, res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_question = 13523\nbatch_size = 36\nmax_seq = 200\nn_pid = -1\nn_blocks = 1\nd_model = 256\ndropout = 0.05\nkq_same = 1\nn_heads = 8\nd_ff = 2048\nl2 = 1e-5\nfinal_fc_dim = 512\n\nlr = 1e-5\nMAX_LEARNING_RATE = 2e-3\nepochs = 50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_question","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = AKTDataset(train_group, n_skills=n_question, max_seq = max_seq, min_samples=1)\ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=8)\n\nvalid_dataset = AKTDataset(test_group, n_skills=n_question, max_seq = max_seq)\nvalid_dataloader = DataLoader(valid_dataset, batch_size=24, shuffle=False, num_workers=8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define AKT Model|"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dim(IntEnum):\n    batch = 0\n    seq = 1\n    feature = 2\n\n\nclass AKT(nn.Module):\n    def __init__(self, n_question, n_pid, d_model, n_blocks,\n                 kq_same, dropout, model_type, final_fc_dim=512, n_heads=8, d_ff=2048,  l2=1e-5, separate_qa=False):\n        super().__init__()\n        \"\"\"\n        Input:\n            d_model: dimension of attention block\n            final_fc_dim: dimension of final fully connected net before prediction\n            n_heads: number of heads in multi-headed attention\n            d_ff : dimension for fully conntected net inside the basic block\n        \"\"\"\n        self.n_question = n_question\n        self.dropout = dropout\n        self.kq_same = kq_same\n        self.n_pid = n_pid\n        self.l2 = l2\n        self.model_type = model_type\n        self.separate_qa = separate_qa\n        embed_l = d_model\n        if self.n_pid > 0:\n            self.difficult_param = nn.Embedding(self.n_pid+1, 1)\n            self.q_embed_diff = nn.Embedding(self.n_question+1, embed_l)\n            self.qa_embed_diff = nn.Embedding(2 * self.n_question + 1, embed_l)\n        # n_question+1 ,d_model\n        self.q_embed = nn.Embedding(self.n_question+1, embed_l)\n        if self.separate_qa:\n            self.qa_embed = nn.Embedding(2*self.n_question+1, embed_l)\n        else:\n            self.qa_embed = nn.Embedding(2, embed_l)\n        # Architecture Object. It contains stack of attention block\n        self.model = Architecture(n_question=n_question, n_blocks=n_blocks, n_heads=n_heads, dropout=dropout,\n                                    d_model=d_model, d_feature=d_model / n_heads, d_ff=d_ff,  kq_same=self.kq_same, model_type=self.model_type)\n\n        self.out = nn.Sequential(\n            nn.Linear(d_model + embed_l,\n                      final_fc_dim), nn.ReLU(), nn.Dropout(self.dropout),\n            nn.Linear(final_fc_dim, 256), nn.ReLU(\n            ), nn.Dropout(self.dropout),\n            nn.Linear(256, 1)\n        )\n        self.reset()\n\n    def reset(self):\n        for p in self.parameters():\n            if p.size(0) == self.n_pid+1 and self.n_pid > 0:\n                torch.nn.init.constant_(p, 0.)\n\n    def forward(self, q_data, qa_data, pid_data=None):\n        # Batch First\n        q_embed_data = self.q_embed(q_data)  # BS, seqlen,  d_model# c_ct\n        if self.separate_qa:\n            # BS, seqlen, d_model #f_(ct,rt)\n            qa_embed_data = self.qa_embed(qa_data)\n        else:\n            qa_data = (qa_data-q_data)//self.n_question  # rt\n            # BS, seqlen, d_model # c_ct+ g_rt =e_(ct,rt)\n            qa_embed_data = self.qa_embed(qa_data)+q_embed_data\n\n        if self.n_pid > 0:\n            q_embed_diff_data = self.q_embed_diff(q_data)  # d_ct\n            pid_embed_data = self.difficult_param(pid_data)  # uq\n            q_embed_data = q_embed_data + pid_embed_data * \\\n                q_embed_diff_data  # uq *d_ct + c_ct\n            qa_embed_diff_data = self.qa_embed_diff(\n                qa_data)  # f_(ct,rt) or #h_rt\n            if self.separate_qa:\n                qa_embed_data = qa_embed_data + pid_embed_data * \\\n                    qa_embed_diff_data  # uq* f_(ct,rt) + e_(ct,rt)\n            else:\n                qa_embed_data = qa_embed_data + pid_embed_data * \\\n                    (qa_embed_diff_data+q_embed_diff_data)  # + uq *(h_rt+d_ct)\n            c_reg_loss = (pid_embed_data ** 2.).sum() * self.l2\n        else:\n            c_reg_loss = 0.\n\n        # BS.seqlen,d_model\n        # Pass to the decoder\n        # output shape BS,seqlen,d_model or d_model//2\n        d_output = self.model(q_embed_data, qa_embed_data)  # 211x512\n\n        concat_q = torch.cat([d_output, q_embed_data], dim=-1)\n        output = self.out(concat_q)\n\n        return output.squeeze(-1)\n\n\nclass Architecture(nn.Module):\n    def __init__(self, n_question,  n_blocks, d_model, d_feature,\n                 d_ff, n_heads, dropout, kq_same, model_type):\n        super().__init__()\n        \"\"\"\n            n_block : number of stacked blocks in the attention\n            d_model : dimension of attention input/output\n            d_feature : dimension of input in each of the multi-head attention part.\n            n_head : number of heads. n_heads*d_feature = d_model\n        \"\"\"\n        self.d_model = d_model\n        self.model_type = model_type\n\n        if model_type in {'akt'}:\n            self.blocks_1 = nn.ModuleList([\n                TransformerLayer(d_model=d_model, d_feature=d_model // n_heads,\n                                 d_ff=d_ff, dropout=dropout, n_heads=n_heads, kq_same=kq_same)\n                for _ in range(n_blocks)\n            ])\n            self.blocks_2 = nn.ModuleList([\n                TransformerLayer(d_model=d_model, d_feature=d_model // n_heads,\n                                 d_ff=d_ff, dropout=dropout, n_heads=n_heads, kq_same=kq_same)\n                for _ in range(n_blocks*2)\n            ])\n\n    def forward(self, q_embed_data, qa_embed_data):\n        # target shape  bs, seqlen\n        seqlen, batch_size = q_embed_data.size(1), q_embed_data.size(0)\n\n        qa_pos_embed = qa_embed_data\n        q_pos_embed = q_embed_data\n\n        y = qa_pos_embed\n        seqlen, batch_size = y.size(1), y.size(0)\n        x = q_pos_embed\n\n        # encoder\n        for block in self.blocks_1:  # encode qas\n            y = block(mask=1, query=y, key=y, values=y)\n        flag_first = True\n        for block in self.blocks_2:\n            if flag_first:  # peek current question\n                x = block(mask=1, query=x, key=x,\n                          values=x, apply_pos=False)\n                flag_first = False\n            else:  # dont peek current response\n                x = block(mask=0, query=x, key=x, values=y, apply_pos=True)\n                flag_first = True\n        return x\n\n\nclass TransformerLayer(nn.Module):\n    def __init__(self, d_model, d_feature,\n                 d_ff, n_heads, dropout,  kq_same):\n        super().__init__()\n        \"\"\"\n            This is a Basic Block of Transformer paper. It containts one Multi-head attention object. Followed by layer norm and postion wise feedforward net and dropout layer.\n        \"\"\"\n        kq_same = kq_same == 1\n        # Multi-Head Attention Block\n        self.masked_attn_head = MultiHeadAttention(\n            d_model, d_feature, n_heads, dropout, kq_same=kq_same)\n\n        # Two layer norm layer and two droput layer\n        self.layer_norm1 = nn.LayerNorm(d_model)\n        self.dropout1 = nn.Dropout(dropout)\n\n        self.linear1 = nn.Linear(d_model, d_ff)\n        self.activation = nn.ReLU()\n        self.dropout = nn.Dropout(dropout)\n        self.linear2 = nn.Linear(d_ff, d_model)\n\n        self.layer_norm2 = nn.LayerNorm(d_model)\n        self.dropout2 = nn.Dropout(dropout)\n\n    def forward(self, mask, query, key, values, apply_pos=True):\n        \"\"\"\n        Input:\n            block : object of type BasicBlock(nn.Module). It contains masked_attn_head objects which is of type MultiHeadAttention(nn.Module).\n            mask : 0 means, it can peek only past values. 1 means, block can peek only current and pas values\n            query : Query. In transformer paper it is the input for both encoder and decoder\n            key : Keys. In transformer paper it is the input for both encoder and decoder\n            Values. In transformer paper it is the input for encoder and  encoded output for decoder (in masked attention part)\n\n        Output:\n            query: Input gets changed over the layer and returned.\n\n        \"\"\"\n\n        seqlen, batch_size = query.size(1), query.size(0)\n        nopeek_mask = np.triu(\n            np.ones((1, 1, seqlen, seqlen)), k=mask).astype('uint8')\n        src_mask = (torch.from_numpy(nopeek_mask) == 0).to(device)\n        if mask == 0:  # If 0, zero-padding is needed.\n            # Calls block.masked_attn_head.forward() method\n            query2 = self.masked_attn_head(\n                query, key, values, mask=src_mask, zero_pad=True)\n        else:\n            # Calls block.masked_attn_head.forward() method\n            query2 = self.masked_attn_head(\n                query, key, values, mask=src_mask, zero_pad=False)\n\n        query = query + self.dropout1((query2))\n        query = self.layer_norm1(query)\n        if apply_pos:\n            query2 = self.linear2(self.dropout(\n                self.activation(self.linear1(query))))\n            query = query + self.dropout2((query2))\n            query = self.layer_norm2(query)\n        return query\n\n\nclass MultiHeadAttention(nn.Module):\n    def __init__(self, d_model, d_feature, n_heads, dropout, kq_same, bias=True):\n        super().__init__()\n        \"\"\"\n        It has projection layer for getting keys, queries and values. Followed by attention and a connected layer.\n        \"\"\"\n        self.d_model = d_model\n        self.d_k = d_feature\n        self.h = n_heads\n        self.kq_same = kq_same\n\n        self.v_linear = nn.Linear(d_model, d_model, bias=bias)\n        self.k_linear = nn.Linear(d_model, d_model, bias=bias)\n        if kq_same is False:\n            self.q_linear = nn.Linear(d_model, d_model, bias=bias)\n        self.dropout = nn.Dropout(dropout)\n        self.proj_bias = bias\n        self.out_proj = nn.Linear(d_model, d_model, bias=bias)\n        self.gammas = nn.Parameter(torch.zeros(n_heads, 1, 1))\n        torch.nn.init.xavier_uniform_(self.gammas)\n\n        self._reset_parameters()\n\n    def _reset_parameters(self):\n        xavier_uniform_(self.k_linear.weight)\n        xavier_uniform_(self.v_linear.weight)\n        if self.kq_same is False:\n            xavier_uniform_(self.q_linear.weight)\n\n        if self.proj_bias:\n            constant_(self.k_linear.bias, 0.)\n            constant_(self.v_linear.bias, 0.)\n            if self.kq_same is False:\n                constant_(self.q_linear.bias, 0.)\n            constant_(self.out_proj.bias, 0.)\n\n    def forward(self, q, k, v, mask, zero_pad):\n\n        bs = q.size(0)\n\n        # perform linear operation and split into h heads\n\n        k = self.k_linear(k).view(bs, -1, self.h, self.d_k)\n        if self.kq_same is False:\n            q = self.q_linear(q).view(bs, -1, self.h, self.d_k)\n        else:\n            q = self.k_linear(q).view(bs, -1, self.h, self.d_k)\n        v = self.v_linear(v).view(bs, -1, self.h, self.d_k)\n\n        # transpose to get dimensions bs * h * sl * d_model\n\n        k = k.transpose(1, 2)\n        q = q.transpose(1, 2)\n        v = v.transpose(1, 2)\n        # calculate attention using function we will define next\n        gammas = self.gammas\n        scores = attention(q, k, v, self.d_k,\n                           mask, self.dropout, zero_pad, gammas)\n\n        # concatenate heads and put through final linear layer\n        concat = scores.transpose(1, 2).contiguous()\\\n            .view(bs, -1, self.d_model)\n\n        output = self.out_proj(concat)\n\n        return output\n\n\ndef attention(q, k, v, d_k, mask, dropout, zero_pad, gamma=None):\n    \"\"\"\n    This is called by Multi-head atention object to find the values.\n    \"\"\"\n    scores = torch.matmul(q, k.transpose(-2, -1)) / \\\n        math.sqrt(d_k)  # BS, 8, seqlen, seqlen\n    bs, head, seqlen = scores.size(0), scores.size(1), scores.size(2)\n\n    x1 = torch.arange(seqlen).expand(seqlen, -1).to(device)\n    x2 = x1.transpose(0, 1).contiguous()\n\n    with torch.no_grad():\n        scores_ = scores.masked_fill(mask == 0, -1e32)\n        scores_ = F.softmax(scores_, dim=-1)  # BS,8,seqlen,seqlen\n        scores_ = scores_ * mask.float().to(device)\n        distcum_scores = torch.cumsum(scores_, dim=-1)  # bs, 8, sl, sl\n        disttotal_scores = torch.sum(\n            scores_, dim=-1, keepdim=True)  # bs, 8, sl, 1\n        position_effect = torch.abs(\n            x1-x2)[None, None, :, :].type(torch.FloatTensor).to(device)  # 1, 1, seqlen, seqlen\n        # bs, 8, sl, sl positive distance\n        dist_scores = torch.clamp(\n            (disttotal_scores-distcum_scores)*position_effect, min=0.)\n        dist_scores = dist_scores.sqrt().detach()\n    m = nn.Softplus()\n    gamma = -1. * m(gamma).unsqueeze(0)  # 1,8,1,1\n    # Now after do exp(gamma*distance) and then clamp to 1e-5 to 1e5\n    total_effect = torch.clamp(torch.clamp(\n        (dist_scores*gamma).exp(), min=1e-5), max=1e5)\n    scores = scores * total_effect\n\n    scores.masked_fill_(mask == 0, -1e32)\n    scores = F.softmax(scores, dim=-1)  # BS,8,seqlen,seqlen\n    if zero_pad:\n        pad_zero = torch.zeros(bs, head, 1, seqlen).to(device)\n        scores = torch.cat([pad_zero, scores[:, :, 1:, :]], dim=2)\n    scores = dropout(scores)\n    output = torch.matmul(scores, v)\n    return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = AKT(\n    n_question=n_question, n_pid=n_pid, n_blocks=n_blocks, \n    d_model=d_model, n_heads=n_heads, dropout=dropout, \n    kq_same=kq_same, model_type='akt', l2=l2, \n    final_fc_dim=final_fc_dim, d_ff=d_ff\n)\n\noptimizer = torch.optim.Adam(\n    model.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8\n)\n\ncriterion = nn.BCEWithLogitsLoss()\nscheduler = torch.optim.lr_scheduler.OneCycleLR(\n    optimizer,  max_lr=MAX_LEARNING_RATE, steps_per_epoch=len(train_dataloader), epochs=epochs\n)\n\nmodel.to(device)\ncriterion.to(device)\n\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, dataloader, optimizer, scheduler, criterion, device):\n    model.train()\n    \n    train_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n    \n    for item in dataloader:\n        q = item[0].to(device).long()\n        qa = item[1].to(device).long()\n        res = item[2].to(device).float()\n        target_mask = (q != 0)\n        \n        optimizer.zero_grad()\n        output = model(q, qa)\n        \n        loss = criterion(output, res)\n        \n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n        train_loss.append(loss.item())\n        \n        output = torch.masked_select(output, target_mask)\n        res = torch.masked_select(res, target_mask)\n        pred = (torch.sigmoid(output) >= 0.5).long()\n        \n        num_corrects += (pred == res).sum().item()\n        num_total += len(res)\n        \n        labels.extend(res.view(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n        \n    acc = num_corrects / num_total\n    auc = roc_auc_score(labels, outs)\n    loss = np.mean(train_loss)\n    \n    return loss, acc, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def validation(model, dataloader, criterion, device):\n    model.eval()\n    \n    valid_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n    \n    for item in dataloader:\n        q = item[0].to(device).long()\n        qa = item[1].to(device).long()\n        res = item[2].to(device).float()\n        target_mask = (q != 0)\n        \n        output = model(q, qa)\n        loss = criterion(output, res)\n        \n        valid_loss.append(loss.item())\n        \n        output = torch.masked_select(output, target_mask)\n        res = torch.masked_select(res, target_mask)\n        pred = (torch.sigmoid(output) >= 0.5).long()\n        \n        num_corrects += (pred == res).sum().item()\n        num_total += len(res)\n        \n        labels.extend(res.view(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n        \n    acc = num_corrects / num_total\n    auc = roc_auc_score(labels, outs)\n    loss = np.mean(valid_loss)\n    \n    return loss, acc, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_auc = 0\nmax_steps = 3\nstep = 0\n\nfrom tqdm.notebook import tqdm\n\nfor epoch in range(epochs):\n    loss, acc, auc = train_model(model, train_dataloader, optimizer, scheduler, criterion, device)\n    print(\"- Epoch - {}/{} train: - {:.3f} acc - {:.3f} auc - {:.3f}\".format(epoch+1, epochs, loss, acc, auc))\n    loss, acc, auc = validation(model, valid_dataloader, criterion, device)\n    print(\"- Epoch - {}/{} valid: - {:.3f} acc - {:.3f} auc - {:.3f}\".format(epoch+1, epochs, loss, acc, auc))\n    if auc > best_auc:\n        best_auc = auc\n        step = 0\n        torch.save(model.state_dict(), \"akt_model.pt\")\n    else:\n        step += 1\n        if step >= max_steps:\n            break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}