{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport pickle\nimport gc\nimport time\nimport itertools\n\nfrom tqdm import tqdm\nfrom sklearn.metrics import roc_auc_score, auc, roc_auc_score\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom time import time \nfrom collections import namedtuple\nRANDOM_SEED = 42\nnp.random.seed(RANDOM_SEED)\ngc.enable()\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport math\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\nimport joblib\nimport random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nskills = joblib.load(\"../input/data-generate/skills.pkl.zip\")\nn_skill = len(skills)\ngroup = pd.read_pickle(\"../input/group-with-label-smothing/group_with_labelsmoothing.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group[115]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\n%%time\nn=len(group.index)\nVALID_SAMPLES = 0\nvalid_indexes=np.random.choice(group.index, VALID_SAMPLES, replace=False)\ntrain_group = group[np.invert(group.index.isin(valid_indexes))]\nvalid_group =group[group.index.isin(valid_indexes)]\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQ=100\nclass RiiidDataset(Dataset):\n    def __init__(self,group,n_skill,max_seq=MAX_SEQ):\n        super(RiiidDataset,self).__init__()\n        self.max_seq = max_seq\n        self.n_skill = n_skill\n        self.samples = {}\n        self.user_ids = []\n        for user_id in group.index:\n            _,content_id_,answered_correctly_,part_id_, \\\n                prior_question_elapsed_time_,prior_lag_time_,\\\n            answered_correctly_smoothing_= group[user_id]\n            if len(content_id_)<5:\n                continue\n            if len(content_id_)>self.max_seq:\n                total_questions = len(content_id_)\n                initial = total_questions % self.max_seq\n                if initial >= 5:\n                    self.user_ids.append(f\"{user_id}_0\")\n                    self.samples[f\"{user_id}_0\"] = (content_id_[:initial],\n                                    answered_correctly_[:initial],part_id_[:initial],\n                                        prior_question_elapsed_time_[:initial],\n                                        prior_lag_time_[:initial],answered_correctly_smoothing_[:initial])\n                    \n                for seq in range(total_questions // self.max_seq):\n                    self.user_ids.append(f\"{user_id}_{seq+1}\")\n                    start = initial + seq * self.max_seq\n                    end = initial + (seq + 1) * self.max_seq\n                    self.samples[f\"{user_id}_{seq+1}\"] = (content_id_[start:end],\n                    answered_correctly_[start:end],part_id_[start:end], \n                        prior_question_elapsed_time_[start:end],prior_lag_time_[start:end],\n                       answered_correctly_smoothing_[start:end]  )\n                '''\n                initial = total_questions % self.max_seq\n                a=random.random()\n                if a>=0.75 and initial >= 50:\n                    \n                    self.user_ids.append(f\"{user_id}_0_50\")\n                    self.samples[f\"{user_id}_0_50\"] = (content_id_[:initial],\n                                    answered_correctly_[:initial],part_id_[:initial],\n                                    prior_question_elapsed_time_[:initial],\n                                   prior_lag_time_[:initial],answered_correctly_smoothing_[:initial]) \n                    for seq in range(total_questions // self.max_seq):\n                        self.user_ids.append(f\"{user_id}_{seq+1}_50\")\n                        start = initial + seq * self.max_seq\n                        end = initial + (seq + 1) * self.max_seq\n                        self.samples[f\"{user_id}_{seq+1}_50\"] = (content_id_[start:end],\n                        answered_correctly_[start:end],part_id_[start:end], \n                            prior_question_elapsed_time_[start:end],prior_lag_time_[start:end],\n                           answered_correctly_smoothing_[start:end]  )\n\n                if len(content_id_)>80:\n                    n_sample = total_questions // self.max_seq\n                    for i in range(n_sample):\n                        index=np.random.randint(0,len(content_id_)-1,150)\n                        index=sorted(list(set(index)))[:100]\n                        self.user_ids.append(f\"{user_id}_{seq+1}\"+str(i))\n                        self.samples[f\"{user_id}_{seq+1}\"+str(i)] = (content_id_[index],\n                                    answered_correctly_[index],part_id_[index],\n                                        prior_question_elapsed_time_[index],\n                                        prior_lag_time_[index],answered_correctly_smoothing_[index])  \n                    '''\n            else:\n                user_id = str(user_id)\n                self.user_ids.append(user_id)\n                self.samples[user_id] = (content_id_,answered_correctly_,\n                                         part_id_,prior_question_elapsed_time_,\n                                       prior_lag_time_,answered_correctly_smoothing_)\n\n    def __len__(self):\n        return len(self.user_ids)\n    \n    def __getitem__(self,index):\n        user_id = self.user_ids[index]\n        content_id_,answered_correctly_,\\\n        part_id_,prior_question_elapsed_time_,\\\n        prior_lag_time_,answered_correctly_smoothing_= self.samples[user_id]\n        \n        seq_len = len(content_id_)\n        \n        content_id=np.zeros(self.max_seq,dtype = int)\n        answered_correctly=np.zeros(self.max_seq,dtype = int)\n        part_id=np.zeros(self.max_seq,dtype = int)\n        prior_question_elapsed_time=np.zeros(self.max_seq,dtype = int)\n        prior_lag_time=np.zeros(self.max_seq,dtype = int)\n        \n        answered_correctly_smoothing=np.zeros(self.max_seq,dtype = float)\n        \n        if seq_len == self.max_seq:\n            content_id[:] = content_id_\n            answered_correctly[:] = answered_correctly_\n            part_id[:] = part_id_\n            prior_question_elapsed_time[:] = prior_question_elapsed_time_\n            prior_lag_time[:] = prior_lag_time_\n            answered_correctly_smoothing[:] =  answered_correctly_smoothing_\n\n        else:\n            content_id[-seq_len:] = content_id_\n            answered_correctly[-seq_len:] = answered_correctly_\n            part_id[-seq_len:] = part_id_\n            prior_question_elapsed_time[-seq_len:] = prior_question_elapsed_time_\n            prior_lag_time[-seq_len:] = prior_lag_time_\n            answered_correctly_smoothing[-seq_len:] =  answered_correctly_smoothing_\n\n\n        content_id_new=content_id[1:]\n        answered_correctly_new=answered_correctly[1:]\n        part_id_new=part_id[1:]\n        prior_question_elapsed_time_new=prior_question_elapsed_time[1:]\n        prior_lag_time_new = prior_lag_time[1:]\n        answered_correctly_smoothing_new=answered_correctly_smoothing[1:]\n        \n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = content_id[:-1].copy()\n        x += (answered_correctly[:-1] == 1) * self.n_skill\n \n        return x, content_id_new,part_id_new,\\\nprior_question_elapsed_time_new,prior_lag_time_new,answered_correctly_new,\\\nanswered_correctly_smoothing_new\n\n\n    \ndef collate_fn(batch):\n    x, content_id,part_id, prior_question_elapsed_time,\\\n    prior_lag_time,label,label_smoothing = zip(*batch)\n    \n    x=torch.Tensor(x).long()\n    content_id = torch.Tensor(content_id).long()\n    part_id = torch.Tensor(part_id).long()\n    prior_question_elapsed_time = torch.Tensor(prior_question_elapsed_time).long()\n    prior_lag_time = torch.Tensor(prior_lag_time).long()\n    label = torch.Tensor(label).float()\n    label_smoothing = torch.Tensor(label_smoothing).float()\n    # remember the order\n    return x,content_id,part_id, prior_question_elapsed_time,\\\n            prior_lag_time,label,label_smoothing","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ScaledDotProductAttention(nn.Module):\n    \n    def __init__(self, temperature, attn_dropout=0.1):\n        super().__init__()\n        self.temperature = temperature\n        self.dropout = nn.Dropout(attn_dropout)\n    \n    def forward(self, q, k, v, mask=None):\n        attn = torch.matmul(q / self.temperature, k.transpose(2, 3))\n        \n        if mask is not None:\n            attn = attn.masked_fill(mask == 0, -1e9)\n            \n        attn = self.dropout(F.softmax(attn, dim=-1))\n        output = torch.matmul(attn, v)\n        \n        return output, attn\nclass MultiHeadAttention(nn.Module):\n    def __init__(self,n_head,d_model,d_k,d_v,dropout=0.1):\n        super().__init__()\n        \n        self.n_head = n_head\n        self.d_k = d_k\n        self.d_v = d_v\n\n        self.w_qs = nn.Linear(d_model, n_head * d_k, bias=False)\n        self.w_ks = nn.Linear(d_model, n_head * d_k, bias=False)\n        self.w_vs = nn.Linear(d_model, n_head * d_v, bias=False)\n        self.fc = nn.Linear(n_head * d_v, d_model, bias=False)\n\n        self.attention = ScaledDotProductAttention(temperature=d_k ** 0.5)\n\n        self.dropout = nn.Dropout(dropout)\n        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)\n    \n    def forward(self, q, k, v, mask=None):\n        \n        d_k, d_v, n_head = self.d_k, self.d_v, self.n_head\n        sz_b, len_q, len_k, len_v = q.size(0), q.size(1), k.size(1), v.size(1)\n        \n        residual = q\n        \n        q = self.w_qs(q).view(sz_b, len_q, n_head, d_k)\n        k = self.w_ks(k).view(sz_b, len_k, n_head, d_k)\n        v = self.w_vs(v).view(sz_b, len_v, n_head, d_v)\n        \n        q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)\n        \n        if mask is not None:\n            mask = mask.unsqueeze(1)   # For head axis broadcasting.\n        \n        q, attn = self.attention(q, k, v, mask=mask)\n\n        # Transpose to move the head dimension back: b x lq x n x dv\n        # Combine the last two dimensions to concatenate all the heads together: b x lq x (n*dv)\n        q = q.transpose(1, 2).contiguous().view(sz_b, len_q, -1)\n        q = self.dropout(self.fc(q))\n        q += residual\n\n        q = self.layer_norm(q)\n\n        return q, attn\nclass PositionwiseFeedForward(nn.Module):\n    ''' A two-feed-forward-layer module '''\n\n    def __init__(self, d_in, d_hid, dropout=0.2):\n        super().__init__()\n        self.w_1 = nn.Linear(d_in, d_hid) # position-wise\n        self.w_2 = nn.Linear(d_hid, d_in) # position-wise\n        #self.layer_norm = nn.LayerNorm(d_in, eps=1e-6)\n        self.dropout = nn.Dropout(dropout)\n\n    def forward(self, x):\n\n        residual = x\n\n        x = self.w_2(F.relu(self.w_1(x)))\n        x = self.dropout(x)\n        x += residual\n        #x = self.layer_norm(x)\n\n        return x\nclass EncoderLayer(nn.Module):\n    ''' Compose with two layers '''\n\n    def __init__(self, d_model, d_inner, n_head, d_k, d_v, dropout=0.1):\n        super(EncoderLayer, self).__init__()\n        self.slf_attn = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)\n        self.pos_ffn = PositionwiseFeedForward(d_model, d_inner, dropout=dropout)\n\n    def forward(self, enc_input, slf_attn_mask=None):\n        enc_output, enc_slf_attn = self.slf_attn(\n            enc_input, enc_input, enc_input, mask=slf_attn_mask)\n        enc_output = self.pos_ffn(enc_output)\n        return enc_output, enc_slf_attn\n\nclass DecoderLayer(nn.Module):\n\n    def __init__(self, d_model, d_inner, n_head, d_k, d_v, dropout=0.1):\n        super(DecoderLayer, self).__init__()\n        self.slf_attn = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)\n        self.enc_attn = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)\n        self.pos_ffn = PositionwiseFeedForward(d_model, d_inner, dropout=dropout)\n\n    def forward(\n            self, dec_input, enc_output,\n            slf_attn_mask=None, dec_enc_attn_mask=None):\n        dec_output, dec_slf_attn = self.slf_attn(\n            dec_input, dec_input, dec_input, mask=slf_attn_mask)\n        dec_output, dec_enc_attn = self.enc_attn(\n            dec_output, enc_output, enc_output, mask=dec_enc_attn_mask)\n        dec_output = self.pos_ffn(dec_output)\n        return dec_output, dec_slf_attn, dec_enc_attn\n\nclass PositionalEncoding(nn.Module):\n    def __init__(self, d_hid, n_position=100):\n        super(PositionalEncoding, self).__init__()\n\n        # Not a parameter\n        self.register_buffer('pos_table', self._get_sinusoid_encoding_table(n_position, d_hid))\n\n    def _get_sinusoid_encoding_table(self, n_position, d_hid):\n        ''' Sinusoid position encoding table '''\n        # TODO: make it with torch instead of numpy\n\n        def get_position_angle_vec(position):\n            return [position / np.power(10000, 2 * (hid_j // 2) / d_hid) for hid_j in range(d_hid)]\n\n        sinusoid_table = np.array([get_position_angle_vec(pos_i) for pos_i in range(n_position)])\n        sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2])  # dim 2i\n        sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2])  # dim 2i+1\n\n        return torch.FloatTensor(sinusoid_table).unsqueeze(0)\n\n    def forward(self, x,y):\n        return x + self.pos_table[:, 1:x.size(1)+1].clone().detach(),\\\n    y + self.pos_table[:, :x.size(1)].clone().detach()\n\ndef future_mask(seq_length):\n    return (1 - torch.triu(torch.ones((1, seq_length, seq_length)), diagonal=1)).bool()\n\nclass Encoder(nn.Module):\n    ''' A encoder model with self attention mechanism. '''\n\n    def __init__(\n            self,d_model,n_layers, n_head, d_k, d_v, d_inner, dropout=0.1):\n\n        super().__init__()\n        self.dropout = nn.Dropout(p=dropout)\n        self.layer_stack = nn.ModuleList([\n            EncoderLayer(d_model, d_inner, n_head, d_k, d_v, dropout=dropout)\n            for _ in range(n_layers)])\n        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)\n\n    def forward(self, x,  return_attns=False):\n\n        enc_slf_attn_list = []  \n        enc_output = self.dropout(x)\n        enc_output = self.layer_norm(enc_output)\n        att_mask = future_mask(x.size(1)).to(device)\n\n        for enc_layer in self.layer_stack:\n            enc_output, enc_slf_attn = enc_layer(enc_output, slf_attn_mask=att_mask)\n            enc_slf_attn_list += [enc_slf_attn] if return_attns else []\n\n        if return_attns:\n            return enc_output, enc_slf_attn_list\n        return enc_output,_\n\nclass Decoder(nn.Module):\n    ''' A decoder model with self attention mechanism. '''\n\n    def __init__(\n            self,  d_model, n_layers, n_head, d_k, d_v,\n            d_inner,dropout=0.1):\n\n        super().__init__()\n        self.dropout = nn.Dropout(p=dropout)\n        self.layer_stack = nn.ModuleList([\n            DecoderLayer(d_model, d_inner, n_head, d_k, d_v, dropout=dropout)\n            for _ in range(n_layers)])\n        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)\n\n    def forward(self, x, enc_output,  return_attns=False):\n\n        dec_slf_attn_list, dec_enc_attn_list = [], []\n        \n        att_mask = future_mask(x.size(1)).to(device)\n\n        dec_output = self.dropout(x)\n        dec_output = self.layer_norm(dec_output)\n\n        for dec_layer in self.layer_stack:\n            dec_output, dec_slf_attn, dec_enc_attn = dec_layer(\n                dec_output, enc_output, slf_attn_mask=att_mask, dec_enc_attn_mask=att_mask)\n            dec_slf_attn_list += [dec_slf_attn] if return_attns else []\n            dec_enc_attn_list += [dec_enc_attn] if return_attns else []\n\n        if return_attns:\n            return dec_output, dec_slf_attn_list, dec_enc_attn_list\n        return dec_output,_,_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SAINTPLUS(nn.Module):\n    def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128, d_inner=128,\n            n_layers=2, n_head=8, d_k=64, d_v=64, dropout=0.1, n_position=100): \n        super(SAINTPLUS, self).__init__()\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n\n        self.embedding = nn.Embedding(2*n_skill+1, embed_dim)\n        self.pos_embedding = PositionalEncoding(embed_dim, n_position=100)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n        self.p_embedding = nn.Embedding(8, embed_dim)\n        self.t_embedding = nn.Embedding(301, embed_dim)\n        self.lag_embedding = nn.Embedding(150, embed_dim)\n\n        self.encoder = Encoder(\n            d_model=embed_dim,n_layers=n_layers, n_head=n_head,\\\n            d_k=d_k, d_v=d_v, d_inner=d_inner, dropout=dropout)\n        self.decoder = Decoder(\n           d_model=embed_dim,n_layers=n_layers, n_head=n_head,\\\n            d_k=d_k, d_v=d_v, d_inner=d_inner, dropout=dropout)\n        \n        self.dropout = nn.Dropout(0.2)\n        self.layer_normal = nn.LayerNorm(embed_dim) \n\n        self.ffn = PositionwiseFeedForward(d_in=embed_dim,d_hid=embed_dim,dropout=0.2)\n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self, x, question_ids,part_id, prior_question_elapsed_time,\n               prior_lag_time):\n        device = x.device        \n        x = self.embedding(x)\n        \n        e = self.e_embedding(question_ids)\n        e,x = self.pos_embedding(e,x)\n        p = self.p_embedding(part_id)\n        t = self.t_embedding(prior_question_elapsed_time)\n        t_lag =self.lag_embedding(prior_lag_time)\n        \n        enc_input = e+p\n        dec_input = x+t+t_lag\n        \n        \n        enc_output, att_weight = self.encoder(enc_input, False )\n        dec_output, *_ = self.decoder(dec_input,  enc_output, False )\n        \n        x = self.pred(dec_output)\n\n        return x.squeeze(-1), att_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = RiiidDataset(group, n_skill)\ndataloader = DataLoader(dataset, batch_size=224, shuffle=True,collate_fn=collate_fn)\n\nitem = dataset.__getitem__(5)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel = SAINTPLUS( n_skill, max_seq=MAX_SEQ, embed_dim=128, d_inner=64,\n            n_layers=5, n_head=8, d_k=64, d_v=64, dropout=0.1, n_position=MAX_SEQ)\n# optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.99, weight_decay=0.005)\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\ncriterion = nn.BCEWithLogitsLoss()\n#model.load_state_dict(torch.load(\"SAINT+20epoch_CTS.pt\", map_location=device))\ncriterion=criterion.to(device)\nmodel=model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(model,train_iterator,optim,criterion, device=\"cuda\"):\n    train_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n    tbar=tqdm(train_iterator)\n    for batch in tbar:\n        x,content_id,  part_id, prior_question_elapsed_time,\\\n        prior_lag_time,label,label_smoothing =batch\n        x=x.to(device)\n        content_id = content_id.to(device)\n        part_id = part_id.to(device)\n        prior_question_elapsed_time = prior_question_elapsed_time.to(device)\n        prior_lag_time = prior_lag_time.to(device)\n\n        label = label.to(device)\n        label_smoothing = label_smoothing.to(device)\n        '''\n        print(content_id.size(),part_id.size(),\\\n              prior_question_elapsed_time.size(), mask.size(), label.size() )\n        '''\n        with torch.set_grad_enabled(mode=True):\n            optimizer.zero_grad()\n            output,atten_weight = model(x, content_id,part_id,\n                           prior_question_elapsed_time,prior_lag_time)\n            loss = criterion(output, label_smoothing)\n            loss.backward()\n            optimizer.step()\n            \n            train_loss.append(loss.item())\n            output = output[:,-1]\n            label = label[:, -1] \n            pred = (torch.sigmoid(output) >= 0.5).long()\n            num_corrects += (pred == label).sum().item()\n            num_total += len(label)\n            \n            labels.extend(label.view(-1).data.cpu().numpy())\n            outs.extend(output.view(-1).data.cpu().numpy())\n            tbar.set_description('loss - {:.4f}'.format(loss))\n    acc = num_corrects / num_total\n    auc = roc_auc_score(labels, outs)\n    loss = np.mean(train_loss)\n\n    return loss, acc, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = 'cuda'\nepochs = 10 #HDKIM 20\nfor epoch in range(epochs):\n    loss, acc, auc = train_epoch(model, dataloader, optimizer, criterion, device)\n    print(\"epoch - {} train_loss - {:.2f} acc - {:.3f} auc - {:.3f}\".format(epoch, loss, acc, auc))\ntorch.save(model.state_dict(), \"SAINT+labelsmoothing_notaug_10epo_5lay_128emb.pt\")   ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}