{"cells":[{"metadata":{},"cell_type":"markdown","source":"Copied notebook from Maunish dave, learning how to use Kaggle!\nhttps://www.kaggle.com/maunish/riiid-super-cool-eda-and-pytorch-baseline\n\nMade very small changes to original model to get a sense of leaderboard performance"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport gc\nimport time\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import TensorDataset, DataLoader\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dtype = {\n    'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float16',\n    'prior_question_had_explanation': 'boolean'\n}\n\ntrain_data = pd.read_csv('../input/riiid-test-answer-prediction/train.csv',\n                         usecols=dtype.keys(),\n                         dtype=dtype, index_col=0,\n                         nrows=10**6)\nque_data = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = train_data.sort_values(\"timestamp\").reset_index(drop=True)\ntrain_data[\"time_required_to_answer\"] = train_data.groupby('user_id')['prior_question_elapsed_time'].shift(-1)\ntrain_data['question_has_explanation'] = train_data.groupby('user_id')['prior_question_had_explanation'].shift(-1)\n\ntag = que_data[\"tags\"].str.split(\" \", n=10, expand=True)\ntag.columns = ['tags1', 'tags2', 'tags3', 'tags4', 'tags5', 'tags6']\n\nque_data = pd.concat([que_data, tag], axis=1).drop(['tags'], axis=1)\nque_data['tags1'] = pd.to_numeric(que_data['tags1'],\n                                  errors='coerce',\n                                  downcast='integer').fillna(-1)\nque_data['tags2'] = pd.to_numeric(que_data['tags2'],\n                                  errors='coerce',\n                                  downcast='integer').fillna(-1)\nque_data['tags3'] = pd.to_numeric(que_data['tags3'],\n                                  errors='coerce',\n                                  downcast='integer').fillna(-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.merge(train_data, que_data, left_on='content_id',\n                      right_on='question_id', how='left')\ntrain_data['timespend'] = train_data.groupby('user_id')['timestamp'].transform(lambda x: (x.max() - x.min())/1000)\ntrain_answered_question = train_data[train_data['answered_correctly']!=-1]\n\ngrouped_by_user_id = train_answered_question.groupby('user_id')\ndf1 = grouped_by_user_id.agg({\n    'answered_correctly': ['mean', 'count', 'std', 'median']\n}).copy()\ndf1.columns = ['mean_user_accuracy', 'questions_answered',\n               'std_user_accuracy', 'median_user_accuracy']\n\ndel grouped_by_user_id\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_content_id = train_answered_question.groupby(\"content_id\")\ndf2 = grouped_by_content_id.agg({\n    'answered_correctly': ['mean', 'count', 'std', 'median']\n}).copy()\ndf2.columns = ['mean_accuracy', 'questions_asked', 'std_accuracy',\n               'median_accuracy']\n\ndel grouped_by_content_id\ndel train_answered_question\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n            'mean_user_accuracy',\n            'questions_answered',\n            'std_user_accuracy',\n            'median_user_accuracy',\n            'mean_accuracy',\n            'questions_asked',\n            'std_accuracy',\n            'median_accuracy',\n            'prior_question_elapsed_time',\n            'time_required_to_answer',\n            'prior_question_had_explanation',\n            'question_has_explanation',\n            'timespend',\n            'bundle_id',\n            'tags1',\n            'tags2',\n            'tags3',\n]\ntarget_column = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = train_data[train_data[target_column] != -1]\ntrain_data = train_data.merge(df1, how='left', on='user_id')\ntrain_data = train_data.merge(df2, how='left', on='content_id')\ntrain_data['prior_question_had_explanation'] = train_data['prior_question_had_explanation'].fillna(value=False).astype(bool)\ntrain_data['question_has_explanation'] = train_data['question_has_explanation'].fillna(value=False).astype(bool)\n\ntrain_data = train_data.fillna(value = -1)\n\ntarget = train_data[target_column].values\ntrain_data = train_data[features]\ntrain_data = train_data.replace([np.inf, -np.inf], np.nan)\ntrain_data = train_data.fillna(-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\ntrain_data = scaler.fit_transform(train_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n  def __init__(self, input_size, output_size):\n    super(Model, self).__init__()\n    self.batch_norm1 = nn.BatchNorm1d(input_size)\n    self.dropout1 = nn.Dropout(0.5)\n    self.linear1 = nn.utils.weight_norm(nn.Linear(input_size, 256))\n  \n    self.batch_norm2 = nn.BatchNorm1d(256)\n    self.dropout2 = nn.Dropout(0.4)\n    self.linear2 = nn.utils.weight_norm(nn.Linear(256, 64))\n\n    self.batch_norm3 = nn.BatchNorm1d(64)\n    self.dropout3 = nn.Dropout(0.4)\n    self.linear3 = nn.utils.weight_norm(nn.Linear(64, output_size))\n\n  def forward(self, xb):\n    x = self.batch_norm1(xb)\n    x = self.dropout1(x)\n    x = F.leaky_relu(self.linear1(x))\n\n    x = self.batch_norm2(x)\n    x = self.dropout2(x)\n    x = F.leaky_relu(self.linear2(x))\n\n    x = self.batch_norm3(x)\n    x = self.dropout3(x)\n    return self.linear3(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = {\n    \"epochs\": 15,\n    \"train_batch_size\": 50_000,\n    \"valid_batch_size\": 50_000,\n    \"test_batch_size\": 50_000,\n    \"nfolds\": 3,\n    \"learning_rate\": 0.005,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run(plot_losses=True):\n  def train_loop(train_loader,\n                 model, loss_fn,\n                 device, optimizer, lr_scheduler=None):\n    model.train()\n    total_loss = 0\n    for i, (inputs, targets) in enumerate(train_loader):\n      inputs = inputs.to(device)\n      targets = targets.to(device)\n\n      optimizer.zero_grad()\n      outputs = model(inputs)\n\n      loss = loss_fn(outputs, targets)\n      loss.backward()\n\n      total_loss += loss.item()\n\n      optimizer.step()\n      if lr_scheduler != None:\n        lr_scheduler.step(loss.item())\n\n    total_loss /= len(train_loader)\n    return total_loss\n  \n  def valid_loop(valid_loader, model, loss_fn, device):\n    model.eval()\n    total_loss = 0\n    predictions = list()\n\n    for i, (inputs, targets) in enumerate(valid_loader):\n      inputs = inputs.to(device)\n      targets = targets.to(device)\n      outputs = model(inputs)\n\n      loss = loss_fn(outputs,targets)\n      predictions.extend(outputs.sigmoid().detach().cpu().numpy())\n\n      total_loss += loss.item()\n    \n    total_loss /= len(valid_loader)\n    return total_loss, np.array(predictions)\n  \n  kfold = StratifiedKFold(n_splits=config['nfolds'])\n\n  fold_train_losses = list()\n  fold_valid_losses = list()\n\n  device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n  print(f\"{device} is used\")\n\n  def loss_fn(outputs, targets):\n    targets = targets.view(-1, 1)\n    return nn.BCEWithLogitsLoss()(outputs, targets)\n\n  for k, (train_idx, valid_idx) in enumerate(kfold.split(train_data, target)):\n    x_train, x_valid, y_train, y_valid = train_data[train_idx,:], train_data[valid_idx,:], target[train_idx], target[valid_idx]\n    input_dim = x_train.shape[1]\n    output_dim = 1\n\n    model = Model(input_dim, output_dim)\n    model.to(device)\n\n    train_tensor = torch.tensor(x_train, dtype=torch.float)\n    y_train_tensor = torch.tensor(y_train, dtype=torch.float)\n\n    train_ds = TensorDataset(train_tensor, y_train_tensor)\n    train_dl = DataLoader(train_ds,\n                          batch_size = config[\"train_batch_size\"],\n                          shuffle=True,\n                          num_workers=4,\n                          pin_memory=True)\n    valid_tensor = torch.tensor(x_valid, dtype=torch.float)\n    y_valid_tensor = torch.tensor(y_valid, dtype=torch.float)\n\n    valid_ds = TensorDataset(valid_tensor, y_valid_tensor)\n    valid_dl = DataLoader(valid_ds, batch_size=config[\"valid_batch_size\"],\n                          shuffle=False,\n                          num_workers=4,\n                          pin_memory=True)\n    optimizer = optim.Adam(model.parameters(), lr=config['learning_rate'])\n    lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,\n                                                        mode='min',\n                                                        factor=0.1,\n                                                        patience=5,\n                                                        eps=1e-4,\n                                                        verbose=True)\n    print(f\"Fold {k}\")\n    best_loss = 999\n\n    train_losses = list()\n    valid_losses = list()\n    start = time.time()\n    for i in range(config[\"epochs\"]):\n      train_loss = train_loop(train_dl, model, loss_fn, device,\n                              optimizer, lr_scheduler=lr_scheduler)\n      valid_loss, predictions = valid_loop(valid_dl, model, loss_fn, device)\n      train_losses.append(train_loss)\n      valid_losses.append(valid_loss)\n      end = time.time()\n      epoch_time = end-start\n      start = end\n      score = roc_auc_score(y_valid, predictions)\n\n      print(f\"epoch:{i} Training loss:{train_loss} | Validation loss:{valid_loss} | Score: {score:.4f} | epoch time {epoch_time:.2f}\")\n\n      if valid_loss <= best_loss:\n        print(f\"Validation loss Decreased from {best_loss} to {valid_loss}\")\n        best_loss = valid_loss\n        torch.save(model.state_dict(), f'model{k}.bin')\n      \n    fold_train_losses.append(train_losses)\n    fold_valid_losses.append(valid_losses)\n  \n  if plot_losses == True:\n    plt.figure(figsize=(20, 14))\n    for i, (t, v) in enumerate(zip(fold_train_losses, fold_valid_losses)):\n      plt.subplot(2, 5, i+1)\n      plt.title(f\"Fold {i}\")\n      plt.plot(t, label=\"train_loss\")\n      plt.plot(v, label=\"valid_loss\")\n      plt.legend()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference(test):\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    all_prediction = np.zeros((test.shape[0], 1))\n    for i in range(config[\"nfolds\"]):\n        input_dim = test.shape[1]\n        output_dim = 1\n        model = Model(input_dim, output_dim)\n        model.load_state_dict(torch.load(f\"model{i}.bin\"))\n        predictions = list()\n        model.to(device)\n        test_tensor = torch.tensor(test, dtype=torch.float)\n        test_dl = DataLoader(test_tensor,\n                            batch_size=config[\"test_batch_size\"],\n                            shuffle=False)\n        \n        with torch.no_grad():\n            for i, inputs in enumerate(test_dl):\n                inputs = inputs.to(device,\n                                  dtype=torch.float)\n                outputs = model(inputs)\n                predictions.extend(outputs.sigmoid().cpu().detach().numpy())\n        \n        all_prediction += np.array(predictions)/config['nfolds']\n    \n    return all_prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_data, sample_prediction_df) in iter_test:\n    test_data = pd.merge(test_data, que_data, left_on='content_id',\n                        right_on='question_id', how='left')\n    test_data['timespend'] = test_data.groupby('user_id')['timestamp'].transform(lambda x: x.max() - x.min())\n    test_data['time_required_to_answer'] = test_data.groupby('user_id')['prior_question_elapsed_time'].shift(-1)\n    test_data['question_has_explanation'] = test_data.groupby('user_id')['prior_question_had_explanation'].shift(-1)\n    test_data = test_data.merge(df1, how='left', on='user_id')\n    test_data = test_data.merge(df2, how='left', on='content_id')\n    test_data['prior_question_had_explanation'] = test_data['prior_question_had_explanation'].fillna(value=False).astype(bool)\n    test_data['question_has_explanation'] = test_data['question_has_explanation'].fillna(value=False).astype(bool)\n    test_data.fillna(value=-1, inplace=True)\n    \n    test_transform = scaler.transform(test_data[features])\n    \n    test_data['answered_correctly'] = inference(test_transform)\n    \n    env.predict(test_data.loc[test_data['content_type_id']==0,\n                             ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}