{"cells":[{"metadata":{},"cell_type":"markdown","source":"References:\n#keras test example\nhttps://www.kaggle.com/code1110/riiid-keras-logisitc-regression-for-analytics\n\n\n#pytorch baseline\nhttps://www.kaggle.com/maunish/riiid-super-cool-eda-and-pytorch-baseline"},{"metadata":{"trusted":true},"cell_type":"code","source":"# delete all files created by skorch checkpoint (if any)\n!rm -rf  /kaggle/working/skorch_chk/*","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/skorch090/skorch-0.9.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport gc\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn_pandas import DataFrameMapper\nfrom sklearn.model_selection import train_test_split\n\nfrom skorch import NeuralNetClassifier\nfrom skorch.helper import predefined_split\nfrom skorch.callbacks import ProgressBar, EarlyStopping, Checkpoint\nfrom skorch.dataset import Dataset\nfrom skorch.callbacks import LRScheduler\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, TensorDataset, Dataset\nimport torch.optim as optim\nimport torch.nn.functional as F\n\nfrom glob import glob\n\n# custom\nimport riiideducation\n\n#supress warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GENERAL_BATCH_SIZE = 2400\nTIME_STEP_SIZE = 19","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CATEGORAL_COLUMNS = ['content_id', 'task_container_id']\nCONTINUOUS_COLUMNS = ['prior_question_had_explanation','prior_question_elapsed_time']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"folder_path = '../input/riiid-test-answer-prediction/'\ntrain_csv = folder_path + 'train.csv'\n# test_csv =  folder_path + 'example_test.csv'\n# lec_csv  =  folder_path + 'lectures.csv'\n# que_csv =   folder_path + 'questions.csv'\n# sample_csv =    folder_path + 'example_sample_submission.csv'\n\ndtype = {\n    'content_id': 'int16',\n    'task_container_id': 'int16',\n    'answered_correctly': 'int8',\n    'prior_question_had_explanation': 'int8',\n    'prior_question_elapsed_time': 'float32'\n}\n\ntest_dtype = {\n    'content_id': 'int16',\n    'task_container_id': 'int16',\n    'prior_question_had_explanation': 'int8',\n    'prior_question_elapsed_time': 'float32'\n}\n\n# train_data = pd.read_csv(train_csv,\n#                          usecols = dtype.keys(),\n#                          dtype=dtype,\n#                          low_memory=False,\n#                          nrows=10**4)\n\ntrain_data = pd.read_parquet(\"../input/riiid-train-data-multiple-formats/riiid_train.parquet\")\nprint(\"Train size:\", train_data.shape)\n\n# test_data = pd.read_csv(test_csv)\n# lec_data = pd.read_csv(lec_csv)\n# que_data = pd.read_csv(que_csv)\n# sample = pd.read_csv(sample_csv)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Basic data fixing**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# for now set to hard coded value, I need to recheck this value once we load all the data\n#train_data[\"prior_question_elapsed_time\"].mean()\n\n#TIME_MEAN_FOR_ELAPSED = 21000.0\nTIME_MEAN_FOR_ELAPSED = np.floor(train_data[\"prior_question_elapsed_time\"].mean())\nTIME_MEAN_FOR_ELAPSED","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fill_missing_values(data):\n    data['prior_question_had_explanation'] = data['prior_question_had_explanation'].fillna(0).astype(np.int8)\n    data[\"prior_question_elapsed_time\"] = data[\"prior_question_elapsed_time\"].fillna(TIME_MEAN_FOR_ELAPSED)\n    \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def change_column_type_to_categorical(data):\n    data[CATEGORAL_COLUMNS] = data[CATEGORAL_COLUMNS].astype('category')\n    \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datafram_mapper = DataFrameMapper([(train_data[['prior_question_elapsed_time']].columns, MinMaxScaler(feature_range=(-1, 1)))])\n\ndef normalize_data(mapper, data, is_train=True):\n    if is_train:    \n        mapper = mapper.fit(data.copy())\n    \n    scaled_features = mapper.transform(data.copy())\n    scaled_features_df = pd.DataFrame(scaled_features, index=data.index, columns=data[['prior_question_elapsed_time']].columns)\n    data = pd.concat([data.drop('prior_question_elapsed_time', axis = 1), scaled_features_df], axis=1)\n    \n    # clean up\n    del scaled_features\n    del scaled_features_df\n\n    gc.collect()\n\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = train_data[dtype.keys()]\ntrain_data = fill_missing_values(train_data)\ntrain_data = change_column_type_to_categorical(train_data)\ntrain_data = normalize_data(datafram_mapper, train_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"last_train_rows_for_pred = train_data[test_dtype.keys()].tail(TIME_STEP_SIZE - 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Get categorical column embeddding count**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# embedded_cols = {n: len(col.cat.categories) for n,col in train_data[CATEGORAL_COLUMNS].items()}\n# embedded_cols","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Determining size of embedding**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#t = train_data['task_container_id'].astype(pd.CategoricalDtype(ordered=True))\n#t = train_data['task_container_id'].astype(pd.CategoricalDtype(ordered=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#t.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# embedded_cols['content_id'] = 32737\n# embedded_cols['task_container_id'] = 10000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedded_cols = {'content_id': 32737, 'task_container_id': 10000}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedded_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# try different minimum sizes - choosing 50 based on fastai\nembedding_sizes = [(n_categories, min(50, (n_categories+1)//2)) for _,n_categories in embedded_cols.items()]\nembedding_sizes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_actual_features = train_data.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Dataset and data loader**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Riid_RNN_Dataset(Dataset):\n    def __init__(self, data):\n        self.categorical_data = data.loc[:, CATEGORAL_COLUMNS].to_numpy(dtype=np.int64)\n        self.continuous_data = data.loc[:, CONTINUOUS_COLUMNS].to_numpy(dtype=np.float32)\n        self.targets = data['answered_correctly'].to_numpy(dtype=np.float32)\n        \n        self.data_length = len(self.targets) - TIME_STEP_SIZE\n\n    def __getitem__(self, index):\n        X1 = self.categorical_data[index: index + TIME_STEP_SIZE]\n        X2 = self.continuous_data[index: index + TIME_STEP_SIZE]\n        y = self.targets[index + TIME_STEP_SIZE, np.newaxis]\n\n        return (X1, X2), y\n\n    def __len__(self):\n        return self.data_length","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"split_train, split_validation = train_test_split(train_data, test_size=0.20, shuffle=False)\n\ntrain_dataset = Riid_RNN_Dataset(split_train)\nvalidation_dataset = Riid_RNN_Dataset(split_validation)\n\n# train_dataloader = DataLoader(train_dataset, batch_size=GENERAL_BATCH_SIZE, shuffle=False, num_workers=0)\n# validation_dataloader = DataLoader(validation_dataset, batch_size=GENERAL_BATCH_SIZE, shuffle=False, num_workers=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test code\n# dataloader = DataLoader(train_dataset, batch_size=GENERAL_BATCH_SIZE, shuffle=False, num_workers=0)\n# for i, batch in enumerate(dataloader):\n#         print(i, batch[0][0].shape, batch[0][1].shape, batch[1].shape)\n#         #print(batch[0][1])\n#         break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_data\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RNN(nn.Module):\n  def __init__(self, n_contineous_inputs, n_hidden, n_rnnlayers, n_outputs, embedding_sizes):\n    super(RNN, self).__init__()\n    \n    self.embeddings = nn.ModuleList([nn.Embedding(categories, size) for categories,size in embedding_sizes])\n    n_emb = sum(e.embedding_dim for e in self.embeddings) #length of all embeddings combined\n    self.n_emb, self.n_cont = n_emb, n_contineous_inputs\n    \n    self.emb_drop = nn.Dropout(0.3)\n    \n    self.D = self.n_emb + self.n_cont\n    self.M = n_hidden\n    self.K = n_outputs\n    self.L = n_rnnlayers\n    \n    #print(f'RNN LSTM input size is: {self.D}')\n\n    self.rnn = nn.LSTM(\n        input_size=self.D,\n        hidden_size=self.M,\n        num_layers=self.L,\n        batch_first=True)\n    self.fc = nn.Linear(self.M, self.K)\n  \n  def forward(self, X):\n    X_categorical, X_continuous = (*X,)\n        \n    # initial hidden states\n    h0 = torch.zeros(self.L, X_categorical.size(0), self.M).to(device)\n    c0 = torch.zeros(self.L, X_categorical.size(0), self.M).to(device)\n\n    #print(f'categorical type: {X_categorical.dtype} & shape : {X_categorical.shape}')\n    #print(f'continuous type: {X_continuous.dtype} & shape : {X_continuous.shape}')    \n\n# below code is not needed\n#     x = [print(embedding) for col_idx,embedding in enumerate(self.embeddings)]\n#     x = [print(self.get_unique_categorical_data(X_categorical, col_idx).shape) for col_idx,embedding in enumerate(self.embeddings)]\n\n#     for col_idx,embedding in enumerate(self.embeddings):\n#         print(col_idx)\n#         print(f'input min & max: {torch.min(self.get_unique_categorical_data(X_categorical, col_idx))} & {torch.max(self.get_unique_categorical_data(X_categorical, col_idx))}')\n#         print(f'')\n#         t = embedding(self.get_unique_categorical_data(X_categorical, col_idx))\n# end of not needed code\n    \n    # categorial columns are first 2 columns in X\n    #x = [embedding(self.get_unique_categorical_data(X_categorical, col_idx)) for col_idx,embedding in enumerate(self.embeddings)]\n    x = [embedding(X_categorical[:, :,col_idx]) for col_idx,embedding in enumerate(self.embeddings)]\n    #print(f'default shape: {x[0].shape} & {x[1].shape}')\n    x = torch.cat(x, 2)\n    #print(f'after merge shape: {x.shape}')\n    x = self.emb_drop(x) # I can remove dropout if this is unable to compile\n\n    #concatentate last 2 columns (that are contineous columns - first two are categorical)\n    x = torch.cat([x, X_continuous], 2)    \n    #print(f'RNN forward input size is: {x.shape}')\n        \n    # get RNN unit output\n    out, _ = self.rnn(x, (h0, c0))\n\n    # we only want h(T) at the final time step\n    out = self.fc(out[:, -1, :])\n    return out\n\n  def get_unique_categorical_data(self, x, col_idx):\n    x = x.reshape(-1, 2)\n    _, idx = np.unique(x[:, col_idx], return_index=True)\n    return x[np.sort(idx), col_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    contineous_inputs=len(CONTINUOUS_COLUMNS)\n    hidden_units=80\n    layers=3\n    target_output=1\n    lr=1e-2\n    #lr=5e-2\n    weight_decay=1e-6\n    batch_size=GENERAL_BATCH_SIZE\n    epochs=5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"early_loss_stop = EarlyStopping(threshold=0)\nchkPoint = Checkpoint(dirname='skorch_chk',f_params='params_{last_epoch[epoch]}.pt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" def iterator_train(dataset, **kwargs):\n      return DataLoader(train_dataset, **kwargs)\n\n def iterator_valid(dataset, **kwargs):\n      return DataLoader(validation_dataset, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = NeuralNetClassifier(\n    RNN,\n    module__n_contineous_inputs=CFG.contineous_inputs,\n    module__n_hidden=CFG.hidden_units,\n    module__n_rnnlayers=CFG.layers,\n    module__n_outputs=CFG.target_output,\n    module__embedding_sizes=embedding_sizes,\n    \n    batch_size=CFG.batch_size,\n    max_epochs=CFG.epochs,\n\n    # any value besides '0' will give a runtime error in PyTorch \n    #(https://github.com/pytorch/pytorch/issues/28820)\n    iterator_train__num_workers=0,\n    iterator_valid__num_workers=0,\n    iterator_train__shuffle=False,\n    iterator_valid__shuffle=False,\n\n    iterator_train = iterator_train,\n    iterator_valid = iterator_valid,\n    \n    train_split=predefined_split(validation_dataset),\n    #train_split=None,\n\n    criterion=nn.BCEWithLogitsLoss,\n    optimizer=torch.optim.Adam,\n    lr=CFG.lr,\n    optimizer__weight_decay=CFG.weight_decay,\n    callbacks=[\n        ProgressBar(), \n        early_loss_stop,\n        chkPoint,\n         ('lr_scheduler',\n                 LRScheduler(policy=optim.lr_scheduler.ReduceLROnPlateau,\n                             mode='min', \n                             factor=0.15, \n                             patience=3, \n                             verbose=True, \n                             eps=1e-4))\n    ],\n    device=device  # comment to train on cpu\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net.fit(train_dataset, y=None)\n#net.fit(np.zeros(len(train_dataset)), y=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del train_dataset\n# del validation_dataset\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot training & validation loss values\ndef plot_history(history):\n    plt.plot(history[:, 'train_loss'])\n    plt.plot(history[:, 'valid_loss'])\n    plt.title('Model loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Val'], loc='upper right', frameon=False)\n    plt.show()\n    \nplot_history(net.history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Load the best run**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# to load the last checkpoint and use it for prediction\n\nnet.initialize()\nnet.load_params(checkpoint=chkPoint)\n#test_outputs = net.evaluation_step(X_test).sigmoid().cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Prediction**"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\n\nfirst_test_pred = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_datframe_compatible(test_dataframe):\n    return test_dataframe[test_dtype.keys()].copy().astype(test_dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_final_test_dataset(data):\n    categorical_data = data.loc[:, CATEGORAL_COLUMNS].to_numpy(dtype=np.int64)\n    continuous_data = data.loc[:, CONTINUOUS_COLUMNS].to_numpy(dtype=np.float32)\n    result1 = np.zeros((len(data) - TIME_STEP_SIZE + 1, TIME_STEP_SIZE, 2), dtype=np.int64)\n    result2 = np.zeros((len(data) - TIME_STEP_SIZE + 1, TIME_STEP_SIZE, 2), dtype=np.float32)\n    \n    for index in range(len(data) - TIME_STEP_SIZE + 1):\n        result1[index] = categorical_data[index: index + TIME_STEP_SIZE]\n        result2[index] = continuous_data[index: index + TIME_STEP_SIZE]\n\n    return (result1, result2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for test_df, sample_prediction_df in iter_test:\n    test_dataset = fill_missing_values(test_df)\n    test_dataset = make_datframe_compatible(test_dataset)\n    test_dataset = change_column_type_to_categorical(test_dataset)\n    test_dataset = normalize_data(datafram_mapper, test_dataset)\n\n    if first_test_pred:\n        merged_test_dataset = last_train_rows_for_pred.append(test_dataset, ignore_index=True)\n    else:\n        merged_test_dataset = previous_test_dataset.append(test_dataset, ignore_index=True)\n    \n    print(f'shape of final dataset: {merged_test_dataset.shape}')\n    \n    merged_test_dataset = create_final_test_dataset(merged_test_dataset)\n    \n#     t = net.evaluation_step(merged_test_dataset).sigmoid().cpu().numpy()\n#     print(t[:1])\n    test_df['answered_correctly'] = net.evaluation_step(merged_test_dataset).sigmoid().cpu().numpy()\n        \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n    \n    previous_test_dataset = test_dataset.copy()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}