{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import important libraries"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import riiideducation\nimport dask.dataframe as dd\nimport  pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import RobustScaler\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nimport warnings\nimport gc\nwarnings.filterwarnings('ignore')\nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read required Files"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain= pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                usecols=[1, 2, 3,4,7,8,9], dtype={'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','content_type_id': 'int8','answered_correctly':'int8',\n                                                  'prior_question_elapsed_time': 'float32','prior_question_had_explanation': 'object'}\n  \n                  )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain = train[train.content_type_id == False]\ntrain = train.sort_values(['timestamp'], ascending=True)\n\ntrain.drop(['timestamp','content_type_id'], axis=1,   inplace=True)\n\ntrain.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_c = train[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean','std','sum','skew'])\nresults_c.columns = [\"content_mean\",\"content_std\",\"content_sum\",\"content_skew\"]\n\nresults_u = train[['user_id','answered_correctly']].groupby(['user_id']).agg(['mean', 'sum','std','skew'])\nresults_u.columns = [\"user_mean\", 'user_sum','user_std','user_skew']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#reading in question df\nquestions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',\n                            usecols=[0,1, 3,4],\n                            dtype={'question_id': 'int16',\n                              'part': 'int8','bundle_id': 'int8','tags': 'str'}\n                          )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',\n                            usecols=[0,1, 3,4],\n                            dtype={'question_id': 'int16',\n                              'part': 'int8','bundle_id': 'int8','tags': 'str'}\n                          )\ntag = questions_df[\"tags\"].str.split(\" \", n = 10, expand = True) \ntag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']\n\nquestions_df =  pd.concat([questions_df,tag],axis=1).drop(['tags'],axis=1)\nquestions_df['tags1'] = pd.to_numeric(questions_df['tags1'], errors='coerce',downcast='integer').fillna(-1)\nquestions_df['tags2'] = pd.to_numeric(questions_df['tags2'], errors='coerce',downcast='integer').fillna(-1)\nquestions_df['tags3'] = pd.to_numeric(questions_df['tags3'], errors='coerce',downcast='integer').fillna(-1)\n#questions_df['tags4'] = pd.to_numeric(questions_df['tags4'], errors='coerce',downcast='integer').fillna(-1)\n#questions_df['tags5'] = pd.to_numeric(questions_df['tags5'], errors='coerce',downcast='integer')\n#questions_df['tags6'] = pd.to_numeric(questions_df['tags6'], errors='coerce',downcast='integer')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Sample data"},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_columns = ['prior_question_had_explanation','bundle_id','part','tags1','tags2','tags3']\n\ncont_columns = ['prior_question_elapsed_time', \"content_mean\",\"content_std\",\"content_sum\",\"content_skew\",\n                \"user_mean\", 'user_sum','user_std','user_skew']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nX=train.iloc[88000000:,:]\nX = pd.merge(X, results_u, on=['user_id'], how=\"left\")\nX = pd.merge(X, results_c, on=['content_id'], how=\"left\")\nX = pd.merge(X, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')\n\nX=X[X.answered_correctly!= -1 ]\nX=X.sort_values(['user_id'])\nX['prior_question_had_explanation']=X['prior_question_had_explanation'].fillna('False').map({\"True\":True,\"False\":False})\nX['prior_question_elapsed_time'].fillna(0,inplace=True)\n\nfor col in cont_columns:\n    X[col].fillna(X[col].mode(),inplace=True)\n\nY = X[[\"answered_correctly\"]]\nX = X.drop([\"answered_correctly\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing\n- label encoding\n- Robust scaler"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfeatures=cat_columns+cont_columns\n\ndef encode(df,cols):\n    enc =  {}\n    for col in cols:\n        print(col)\n        lbencoder = LabelEncoder()\n        lb = lbencoder.fit(df[col].values)\n        df[col]=lb.transform(df[col].values)\n        enc[col]=lb\n        \n    return df,enc\n\nX,enc_dict = encode(X,cat_columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scale_dict={}\nfix_missing={}\nfor col in cont_columns:\n    scaler = RobustScaler()\n    scale_dict[col]=scaler.fit(X[col].values.reshape(-1,1))\n    X[col] = scale_dict[col].transform(X[col].values.reshape(-1,1))\n    fix_missing[col] = X[col].mode().values[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Determining embedding dimension"},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_dims = [X[col].nunique() for col in cat_columns]\ncat_embs = [(dim, min(50,(dim+1)//2)) for dim in cat_dims]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_embs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RidDataset(Dataset):\n    def __init__(self, df,targets,cat_features,cont_features,mode='train'):\n        self.mode = mode\n        self.data_cont = df[cont_features].values\n        self.data_cat = df[cat_features].values\n        if mode=='train':\n            self.targets = targets.values \n    \n    def __len__(self):\n        return len(self.data_cont)\n    \n    def __getitem__(self, idx):\n        if self.mode == 'train':\n            return torch.FloatTensor(self.data_cont[idx]),torch.LongTensor(self.data_cat[idx]),torch.FloatTensor(self.targets[idx])\n        elif self.mode == 'test':\n            return torch.FloatTensor(self.data_cont[idx]), torch.LongTensor(self.data_cat[idx]),0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RidModel(nn.Module):\n    def __init__(self,emb_dims,no_of_cont):\n        super(RidModel, self).__init__()\n        \n        self.emb = nn.ModuleList([nn.Embedding(x,y) for x,y in emb_dims])\n        \n        no_of_embs = sum([y for x, y in emb_dims])\n        self.no_of_embs = no_of_embs\n        self.no_of_cont = no_of_cont\n        \n        \n        self.batch_norm1 = nn.BatchNorm1d(self.no_of_cont)\n        self.dropout1 = nn.Dropout(0.2)\n        self.dense1 = nn.utils.weight_norm(nn.Linear(no_of_cont, 128))\n        \n        self.batch_norm2 = nn.BatchNorm1d(128+no_of_embs)\n        self.dense2 = nn.utils.weight_norm(nn.Linear(128+no_of_embs, 32))\n         \n        self.batch_norm3 = nn.BatchNorm1d(32)\n        self.dense3 = nn.utils.weight_norm(nn.Linear(32, 16))\n        \n        self.batch_norm4 = nn.BatchNorm1d(16)\n        self.dense4 = nn.utils.weight_norm(nn.Linear(16, 1))\n        \n       \n    def forward(self, cont,cat):\n         \n        ## cat data part\n        x_cat = [emb_layer(cat[:,i]) for i,emb_layer in enumerate(self.emb)]\n        x_cat = torch.cat(x_cat,1)\n        x_cat = self.dropout1(x_cat)\n        ##cont data\n        x = self.batch_norm1(cont)\n        x = self.dropout1(x)\n        x = F.relu(self.dense1(x))\n        \n        ##concat\n        x = torch.cat([x,x_cat],1)\n        \n        ##rest of NN\n        x = self.batch_norm2(x)\n        x = F.relu(self.dense2(x))\n        \n        x = self.batch_norm3(x)\n        x = F.relu(self.dense3(x))\n        \n        \n        x = self.batch_norm4(x)\n        x = F.sigmoid(self.dense4(x))\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train,X_valid,y_train,y_valid = train_test_split(X[features],Y,test_size=0.15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X,Y,train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert X_train.shape[0]==y_train.shape[0]\nassert X_valid.shape[0]==y_valid.shape[0]\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"nepochs=5\ntrain_set = RidDataset(X_train,y_train,cat_columns,cont_columns,mode=\"train\")\nvalid_set = RidDataset(X_valid,y_valid,cat_columns,cont_columns,mode=\"train\")\nval_auc=[]\ndataloaders = {'train':DataLoader(train_set,batch_size=2**15,shuffle=True),\n              \"val\":DataLoader(valid_set,batch_size=2**15,shuffle=True)}\n\nmodel = RidModel(cat_embs,len(cont_columns)).to(DEVICE)\ncheckpoint_path = 'rid_model.pt'\noptimizer = optim.Adam(model.parameters())\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=5, eps=1e-4, verbose=True)\ncriterion = nn.BCELoss()\nbest_loss = {'train':np.inf,'val':np.inf}\nauc_score = {'train':0,'val':0.0}\n\nfor epoch in range(nepochs):\n            epoch_loss = {'train': 0.0, 'val': 0.0}\n            \n            for phase in ['train', 'val']:\n                if phase == 'train':\n                    model.train()\n                else:\n                    model.eval()\n                \n                running_loss = 0.0\n                auc=0.0\n                \n                for i,(x,y,z) in enumerate(dataloaders[phase]):\n                    x, y, z = x.to(DEVICE), y.to(DEVICE),z.to(DEVICE)\n                    optimizer.zero_grad()\n                    \n                    with torch.set_grad_enabled(phase=='train'):\n                        preds = model(x,y)\n                        loss = criterion(preds, z)\n                        auc = roc_auc_score(z.detach().cpu().numpy(),preds.detach().cpu().numpy())\n                        \n                        if phase=='train':\n                            loss.backward()\n                            optimizer.step()\n                    \n                    running_loss += loss.item() / len(dataloaders[phase])\n                    auc += auc/len(dataloaders[phase])\n                \n                epoch_loss[phase] = running_loss\n                auc_score[phase]=auc\n                \n            print(\"Epoch {}/{}   - loss: {:5.5f}   - val_loss: {:5.5f} -- AUC {:5.4f} --val AUC {:5.4f}\".format(epoch+1,\n                    nepochs, epoch_loss['train'], epoch_loss['val'],auc_score['train'],auc_score['val']))\n            val_auc.append(auc_score['val'])\n            scheduler.step(epoch_loss['val'])\n            \n            if epoch_loss['val'] < best_loss['val']:\n                best_loss = epoch_loss\n                torch.save(model.state_dict(), checkpoint_path)\n                \n \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Final validation AUC Score {np.mean(val_auc):5.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_df = pd.read_pickle('../input/riiid-cross-validation-files/cv1_valid.pickle')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class Iter_Valid:   \n#     def __init__(self, df, max_user=1000):\n#         df = df.reset_index(drop=True)\n#         self.df = df\n#         self.user_answer = df['user_answer'].astype(str).values\n#         self.answered_correctly = df['answered_correctly'].astype(str).values\n#         df['prior_group_responses'] = \"[]\"\n#         df['prior_group_answers_correct'] = \"[]\"\n#         self.sample_df = df[df['content_type_id'] == 0][['row_id']]\n#         self.sample_df['answered_correctly'] = 0\n#         self.len = len(df)\n#         self.user_id = df.user_id.values\n#         self.task_container_id = df.task_container_id.values\n#         self.content_type_id = df.content_type_id.values\n#         self.max_user = max_user\n#         self.current = 0\n#         self.pre_user_answer_list = []\n#         self.pre_answered_correctly_list = []\n\n#     def __iter__(self):\n#         return self\n    \n#     def fix_df(self, user_answer_list, answered_correctly_list, pre_start):\n#         df= self.df[pre_start:self.current].copy()\n#         sample_df = self.sample_df[pre_start:self.current].copy()\n#         df.loc[pre_start,'prior_group_responses'] = '[' + \",\".join(self.pre_user_answer_list) + ']'\n#         df.loc[pre_start,'prior_group_answers_correct'] = '[' + \",\".join(self.pre_answered_correctly_list) + ']'\n#         self.pre_user_answer_list = user_answer_list\n#         self.pre_answered_correctly_list = answered_correctly_list\n#         return df, sample_df\n\n#     def __next__(self):\n#         added_user = set()\n#         pre_start = self.current\n#         pre_added_user = -1\n#         pre_task_container_id = -1\n#         pre_content_type_id = -1\n#         user_answer_list = []\n#         answered_correctly_list = []\n#         while self.current < self.len:\n#             crr_user_id = self.user_id[self.current]\n#             crr_task_container_id = self.task_container_id[self.current]\n#             crr_content_type_id = self.content_type_id[self.current]\n#             if crr_user_id in added_user and (crr_user_id != pre_added_user or (crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)):\n#                 # known user(not prev user or (differnt task container and both question))\n#                 return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n#             if len(added_user) == self.max_user:\n#                 if  crr_user_id == pre_added_user and (crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):\n#                     user_answer_list.append(self.user_answer[self.current])\n#                     answered_correctly_list.append(self.answered_correctly[self.current])\n#                     self.current += 1\n#                     continue\n#                 else:\n#                     return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n#             added_user.add(crr_user_id)\n#             pre_added_user = crr_user_id\n#             pre_task_container_id = crr_task_container_id\n#             pre_content_type_id = crr_content_type_id\n#             user_answer_list.append(self.user_answer[self.current])\n#             answered_correctly_list.append(self.answered_correctly[self.current])\n#             self.current += 1\n#         if pre_start < self.current:\n#             return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n#         else:\n#             raise StopIteration()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validaten_flg = False\nif validaten_flg:\n    iter_test = Iter_Valid(target_df,max_user=1000)\n    predicted = []\n    def set_predict(df):\n        predicted.append(df)\nelse:\n    import riiideducation\n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    set_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n  \nmodel = RidModel(cat_embs,len(cont_columns)).to(DEVICE)\nmodel.load_state_dict(torch.load(checkpoint_path))\nmodel.eval()\n    \nfor (test_df, sample_prediction_df) in tqdm(iter_test):\n    preds=[]\n    \n    ##preprocess\n    test_df = pd.merge(test_df, results_u, on=['user_id'],  how=\"left\")\n    test_df = pd.merge(test_df, results_c, on=['content_id'],  how=\"left\")\n    test_df = test_df.loc[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df = pd.merge(test_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')\n    \n    test_df['prior_question_elapsed_time'].fillna(0,inplace=True)\n    test_df['prior_question_had_explanation'].fillna(False,inplace=True)\n    \n    for col in cat_columns[2:]:\n        test_df[col].fillna(questions_df[col].mode(),inplace=True)\n        \n    for col in cat_columns[1:]:\n        test_df[col] = test_df[col].astype(int)\n\n    ## cont features filling nan with mode\n    for col in cont_columns:\n        test_df[col].fillna(fix_missing[col],inplace=True)\n    \n    #print(test_df[col].isna().sum())\n    ## cat features encoding\n    for col in cat_columns:\n        try:\n            test_df[col] = enc_dict[col].transform(test_df[col])\n        except:\n            print(col)\n            break\n    \n    ## cont features scaling\n    for col in cont_columns:\n        test_df[col]=scale_dict[col].transform(test_df[col].values.reshape(-1,1))\n\n    #print(test_df[features].isna().sum())\n    \n    #print(test_df[features].isna().sum())\n    ##dataloader\n    train_set = RidDataset(test_df[features],None,cat_columns,cont_columns,mode=\"test\")\n    testloader = DataLoader(train_set,batch_size=32,shuffle=False)\n\n    ##predictions\n    for i,(x,y,z) in enumerate(testloader):\n        x,y = x.to(DEVICE),y.to(DEVICE)\n\n        with torch.no_grad():\n            batch_pred = model(x,y)\n\n        preds.append(batch_pred)\n\n    preds = torch.cat(preds, dim=0).cpu().numpy()\n    if(any(np.isnan(preds))):\n        print('nan found')\n        print(test_df[features].isna().sum())\n    ##\n    test_df['answered_correctly'] =  preds\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}