{"cells":[{"metadata":{},"cell_type":"markdown","source":"Here  I have tried with LSTM and CNN using pytorch\n\nTODO: Ensembling both model results. "},{"metadata":{},"cell_type":"markdown","source":"Basic data modeling used from this kernel\nhttps://www.kaggle.com/yaroslavmavliutov/riiid-prediction-cnn-keras-0-751"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport os\nfrom tqdm import tqdm\n\nimport riiideducation\n\nfrom sklearn.metrics import roc_auc_score,accuracy_score\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import StandardScaler\n\nimport torch.nn as nn\nimport torch \nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures_csv = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/lectures.csv\")\nexample_test_csv = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/example_test.csv\")\ntrain_csv = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/train.csv\", low_memory=False, nrows=1000000)\nquestions_csv = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/questions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 0 if the event was a question being posed to the user, 1 if the event was the user watching a lecture. So, let's keep just the questions\ntrain_csv = train_csv[train_csv.content_type_id == 0]\n# read -1 as null, for lectures\ntrain_csv = train_csv[train_csv.answered_correctly != -1]\n\ntrain_csv = train_csv.sort_values(['timestamp'], ascending=True).reset_index(drop = True)\n\ncontent_mean_final = train_csv[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean'])\ncontent_mean_final.columns = [\"answered_correctly_content_mean\"]\n\nuser_mean_final = train_csv[['user_id','answered_correctly']].groupby(['user_id']).agg(['mean', 'sum', 'count'])\nuser_mean_final.columns = [\"answered_correctly_user_mean\", 'sum_correct', 'count']\n\n#saving value to fillna\nelapsed_time_mean_final = train_csv.prior_question_elapsed_time.mean()\n\ntrain_csv.drop(['timestamp', 'content_type_id'], axis=1, inplace=True)\n\n\nvalidation = pd.DataFrame()\nfor i in range(4):\n    last_records = train_csv.drop_duplicates('user_id', keep = 'last')\n    train_csv = train_csv[~train_csv.index.isin(last_records.index)]\n    validation = validation.append(last_records)\nX = pd.DataFrame()\nfor i in range(15):\n    last_records = train_csv.drop_duplicates('user_id', keep = 'last')\n    train_csv = train_csv[~train_csv.index.isin(last_records.index)]\n    X = X.append(last_records)\n\n\nresults_c = train_csv[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean'])\nresults_c.columns = [\"answered_correctly_content_mean\"]\n\nresults_u = train_csv[['user_id','answered_correctly']].groupby(['user_id']).agg(['mean', 'sum', 'count'])\nresults_u.columns = [\"answered_correctly_user_mean\", 'sum_correct', 'count']\n\nresult_time_mean = train_csv.prior_question_elapsed_time.mean()\n\ndel(train_csv)\n\nX = pd.merge(X, results_u, on=['user_id'], how=\"left\")\nX = pd.merge(X, results_c, on=['content_id'], how=\"left\")\n\nvalidation = pd.merge(validation, results_u, on=['user_id'], how=\"left\")\nvalidation = pd.merge(validation, results_c, on=['content_id'], how=\"left\")\n\ny = X['answered_correctly']\nX = X.drop(['answered_correctly'], axis=1)\n\ny_val = validation['answered_correctly']\nX_val = validation.drop(['answered_correctly'], axis=1)\n\nlencoder = LabelEncoder()\n\nX['prior_question_had_explanation'].fillna(False, inplace = True)\nX['prior_question_had_explanation_enc'] = lencoder.fit_transform(X['prior_question_had_explanation'])\nX['answered_correctly_user_mean'].fillna(0.5,  inplace=True)\nX['answered_correctly_content_mean'].fillna(0.5,  inplace=True)\nX['sum_correct'].fillna(0, inplace = True)\nX['count'].fillna(0, inplace = True)\nX['prior_question_elapsed_time'].fillna(result_time_mean, inplace = True)\n\nX_val['prior_question_had_explanation'].fillna(False, inplace = True)\nX_val['prior_question_had_explanation_enc'] = lencoder.fit_transform(X_val['prior_question_had_explanation'])\nX_val['answered_correctly_user_mean'].fillna(0.5,  inplace=True)\nX_val['answered_correctly_content_mean'].fillna(0.5,  inplace=True)\nX_val['sum_correct'].fillna(0, inplace = True)\nX_val['count'].fillna(0, inplace = True)\nX_val['prior_question_elapsed_time'].fillna(result_time_mean, inplace = True)\n\nX = X[['answered_correctly_user_mean', 'answered_correctly_content_mean', 'sum_correct', 'count',\n       'prior_question_elapsed_time','prior_question_had_explanation_enc']]\nX_val = X_val[['answered_correctly_user_mean', 'answered_correctly_content_mean', 'sum_correct', 'count',\n       'prior_question_elapsed_time','prior_question_had_explanation_enc']]\n\nscaler = StandardScaler()\nX = scaler.fit_transform(X)\nX_val = scaler.transform(X_val)\n\n\nX_train = X.reshape(X.shape[0], 1,X.shape[1])\nX_test = X_val.reshape(X_val.shape[0], 1,X_val.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RIIDModel(nn.Module):\n  def __init__(self):\n    super(RIIDModel,self).__init__()\n    self.cnn1 = nn.Conv1d(1,32,2)\n    self.cnn2 = nn.Conv1d(32,64,2)\n    self.fc1 = nn.Linear(256,32)\n    self.fc2 = nn.Linear(32,1)\n    self.drop = nn.Dropout(0.1)\n    self.relu = nn.ReLU()\n  \n  def forward(self,x):\n    out = self.relu(self.cnn1(x))\n    out = self.drop(self.relu(self.cnn2(out)))\n    out = out.view(out.shape[0],-1)\n    out = self.relu(self.fc1(out))\n    out = self.fc2(out)\n    return out    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RIIDModelLSTM(nn.Module):\n  def __init__(self,device):\n    super(RIIDModelLSTM,self).__init__()\n    self.n_layers = 2\n    self.lstm = nn.LSTM(6,6,num_layers = self.n_layers,batch_first=True,dropout=0.3)\n    self.fc1 = nn.Linear(6,32)\n    self.fc2 = nn.Linear(32,1)\n    self.drop = nn.Dropout(0.1)\n    self.relu = nn.ReLU()\n    self.device = device\n  \n  def init_hidden(self, batch_size):\n    c0 = torch.zeros((self.n_layers, batch_size, 6)).to(self.device)\n    h0 = torch.zeros((self.n_layers, batch_size, 6)).to(self.device)\n    return h0,c0\n  \n  def forward(self,x):\n    batch_size = x.shape[0]\n    h0,c0 = self.init_hidden(batch_size)\n    out,_ = self.lstm(x,(h0,c0))\n    out = self.drop(self.relu(self.fc1(out)))\n    out = self.drop(self.relu(self.fc2(out)))\n    return out  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = \"cpu\"\nmodel = RIIDModelLSTM(device)\nmodel.state_dict(torch.load(\"../input/lstm-model/model2.pt\",map_location=device))\nmodel.to(device)\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df = pd.merge(test_df, user_mean_final, on=['user_id'],  how=\"left\")\n    test_df = pd.merge(test_df, content_mean_final, on=['content_id'],  how=\"left\")\n    \n    test_df['answered_correctly_user_mean'].fillna(0.6,  inplace=True)\n    test_df['answered_correctly_content_mean'].fillna(0.6,  inplace=True)\n    test_df['sum_correct'].fillna(0.1, inplace=True)\n    test_df['count'].fillna(0.1, inplace=True)\n    test_df['prior_question_elapsed_time'].fillna(elapsed_time_mean_final, inplace = True)\n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df[\"prior_question_had_explanation_enc\"] = lencoder.transform(test_df[\"prior_question_had_explanation\"])\n\n    # fit transform cnn\n    X = scaler.transform(test_df[['answered_correctly_user_mean', 'answered_correctly_content_mean', 'sum_correct', 'count',\n                                  'prior_question_elapsed_time', 'prior_question_had_explanation_enc']])\n    X = X.reshape(X.shape[0], 1, X.shape[1])\n    X = torch.tensor(X,dtype=torch.float)\n    with torch.no_grad():\n        out = model(X)\n        out = nn.Sigmoid()(out).view(-1)\n    test_df['answered_correctly'] = out.cpu().detach().numpy()\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}