{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Riiid! Answer Correctness Prediction - Intro\nTeam: Rohit\n\nAuthor:Rohit Kumar Hansdah\n\nLast edited: 20-Dec-2020\n\nThis model was designed to use an LSTM network, with L1 penalty on the recurrent and input weights for regularization. Since data is formatted as a time series, inputs are build on a per user basis. When training, a number of samples is pulled from each user. When testing, a database is saved and updated at each group/batch and takes into account new users. This database is in the variable 'test_user_dict'.\n\nThe hyperparameters are: the number of time steps in a series (variable timesteps), the number of samples per user (variable sampleperuser), the number of neurons in the LSTM (variable lstm_neurons), the learning rate (variable eta), and the number of training epochs (variable totalepoch)."},{"metadata":{"_uuid":"03acb6a0-86dd-400f-96eb-18a978ea3cfe","_cell_guid":"b1ee63f9-e3fb-41b6-8b5f-0533d1268e12","trusted":true},"cell_type":"code","source":"# Script for competition \"Riiid! Answer Correctness Prediction\"\n# Competition summary can be found here:\n# https://www.kaggle.com/c/riiid-test-answer-prediction/overview\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nimport pandas as pd\nimport numpy as np\nimport time\nimport math\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import regularizers\nfrom sklearn.preprocessing import StandardScaler\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import auc\n\nimport riiideducation","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Set Processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load training data from csv file\n#Data types are explicitly specified to minimize memory usage\nsubset_size = int(5E6)\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', low_memory=False, nrows=subset_size, \n                       dtype={'row_id': 'int64',\n                              'timestamp': 'int64',\n                              'user_id': 'int32',\n                              'content_id': 'int16',\n                              'content_type_id': 'int8',\n                              'task_container_id': 'int16',\n                              'user_answer': 'int8',\n                              'answered_correctly': 'int8',\n                              'prior_question_elapsed_time': 'float32', \n                              'prior_question_had_explanation': 'boolean',\n                             }\n                      )\nprint(train_df.dtypes)\nprint('From .csv loading train_df: rows, columns =', train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Quick glimpse at the training data\ntrain_df.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Remove unused rows and columns\ntrain_df = train_df.loc[train_df.content_type_id==0].reset_index(drop=True)\ntrain_df.drop(['row_id', 'content_type_id', 'user_answer'], axis=1, inplace=True)\nprint('After dropping train_df: rows, columns =', train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define data frame preprocesser (removes nans and converts bools)\nmean_pqet = train_df['prior_question_elapsed_time'].mean()\ndef df_preprocess(in_df):\n    in_df['prior_question_elapsed_time'].fillna(value=mean_pqet, inplace=True)\n    in_df['prior_question_had_explanation'] = in_df[\n        'prior_question_had_explanation'\n    ].fillna(value=False).astype(bool).map({True:1, False:0})\n    in_df.fillna(value=0, inplace=True)\n    \ndf_preprocess(train_df)\nprint('After preprocessing train_df: rows, columns', train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Group by user and how many rows they have in the dataset\ntrain_by_user_id = train_df.groupby(by='user_id')\nuser_list = list(train_by_user_id.groups.keys())\nprint('There are', len(user_list), 'users')\nprint('Average rows per user:', train_by_user_id.count().mean()[0])\nprint('Min rows of one user:', train_by_user_id.count().min()[0])\nprint('Max rows of one user:', train_by_user_id.count().max()[0])\n\n#Delete all large variables not used past this point to conserve memory\ndel train_by_user_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Group by question, and create dictionary\n#Dictionary has the question's mean, standard deviation, and skew\ntrain_by_content_id = train_df.groupby(by='content_id')\ncontent_id_dict = train_by_content_id.agg({'answered_correctly': [np.mean, np.std, 'skew']}).copy()\ncontent_id_dict.columns = ['content_mean', 'content_std', 'content_skew']\ncontent_id_dict.fillna(value=0, inplace=True)\n\n#Delete all large variables not used past this point to conserve memory\ndel train_by_content_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Group by container id, and create dictionary\n#Dictionary has the container's mean, standard deviation, and skew\ntrain_by_container = train_df.groupby(by='task_container_id')\ncontainer_dict = train_by_container.agg({'answered_correctly': [np.mean, np.std, 'skew']}).copy()\ncontainer_dict.columns = ['container_mean', 'container_std', 'container_skew']\ncontainer_dict.fillna(value=0, inplace=True)\n\n#Delete all large variables not used past this point to conserve memory\ndel train_by_container","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#From the questions metadata, create dictionary of part and tags\nquestions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nquestions_df.drop(['bundle_id', 'correct_answer'], axis=1, inplace=True)\nquestions_df.fillna(value=0, inplace=True)\nquestions_df = questions_df.rename(columns={'question_id':'content_id'})\n\n#Questions will have between 0 to 5 tags\ntag_list = [[], [], [], [], []]\ntag_counts = {}\nfor thisstr in questions_df.tags:\n    temp_list = str(thisstr).split(' ')\n    temp_list = [int(i) for i in temp_list]\n    temp_list.sort()\n    for thistag in temp_list:\n        if (not(thistag in tag_counts)):\n            tag_counts[thistag] = 0\n        tag_counts[thistag] = tag_counts[thistag] + 1\n    temp_list = temp_list * math.ceil(5/len(temp_list))\n    for i in range(5):\n        tag_list[i].append(temp_list[i])\n\nfor i in range(5):\n    questions_df['tag_'+str(i)] = tag_list[i]\nquestions_df.drop(['tags'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#See the frequency of each tag\nplt.figure(figsize=(12,6))\nplt.plot(*zip(*sorted(tag_counts.items())))\nplt.xlabel('Tag')\nplt.ylabel('Count of questions with that tag')\nplt.title('Tag Count')\nplt.grid(True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define function to merge all dictionaries onto dataset\ndef merge_all_dict(in_df):\n    temp_df = in_df.merge(content_id_dict, how='left', on='content_id')\n    temp_df = temp_df.merge(container_dict, how='left', on='task_container_id')\n    temp_df = temp_df.merge(questions_df, how='left', on='content_id')\n    return temp_df.fillna(value=0)\n    \ntrain_df = merge_all_dict(train_df)\nprint('After merge, (rows, columns) =', train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define the labels of the input vector\n#Columns that need to be scaled are first\nscaled_labels = [\n    'prior_question_elapsed_time',\n    'content_id',\n    'task_container_id',\n    'part'\n]\ntag_labels = ['tag_'+str(i) for i in range(5)]\nstat_labels = [\n    'content_mean',\n    'content_std',\n    'content_skew',\n    'container_mean',\n    'container_std',\n    'container_skew'\n]\nprior_labels = [\n    'prior_question_had_explanation',\n    'prior_answer_correct'\n]\nts_labels = scaled_labels + tag_labels + stat_labels + prior_labels\nsindex = len(scaled_labels)\ntindex = len(tag_labels)\n\n#Define input shapes\ntimesteps = 100\nfeatures = len(ts_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create scaler for first set of labels\nprint('Creating scaler for preprocessing:')\nscaler = StandardScaler()\nscaler.fit(train_df[scaled_labels].to_numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define function that will get the relevant features for time series\n#Will scale all features that need it\ndef build_ts_array(in_df):\n    out_np = in_df[ts_labels[:-1]].to_numpy()\n    out_np[:, 0:sindex] = scaler.transform(out_np[:, 0:sindex])\n    out_np[:, sindex:sindex+tindex] = out_np[:, sindex:sindex+tindex] / len(tag_counts)\n    #Prior answer correct needs to shifted by 1 row\n    temp_np = in_df['answered_correctly'].shift(periods=1, axis=0, fill_value=0.5).to_numpy()\n    return np.hstack((out_np, np.expand_dims(temp_np, axis=1)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define function that returns samples of a given user\n#Data will be front padded if the number of rows is less than timesteps\ndef user_sample_array(in_df, inuser, numsamples):\n    temp_df = in_df.loc[in_df.user_id==inuser].reset_index(drop=True)\n    x_list = []\n    y_list = []\n    #.iloc includes the first argument, but excludes the second\n    #Thus range starts at len() instead of len()-1\n    for i in range(len(temp_df), 0, -math.ceil(len(temp_df) / numsamples)):\n        next_x = build_ts_array(temp_df.iloc[max(0, i-timesteps):i])\n        next_y = temp_df['answered_correctly'].iloc[i-1]\n        if (len(next_x) < timesteps):\n            next_x = np.vstack((\n                np.zeros((timesteps - len(next_x), features)),\n                next_x\n            ))\n        x_list.append(next_x)\n        y_list.append(next_y)\n    return x_list, y_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create the training set\nx_train_list = []\ny_train_list = []\n\n#Building the set, each sample is the time series of one user\ntime_start = time.time()\nprint('Start creating training set at', time.ctime())\nsampleperuser = 10\nfor thisuser in user_list:\n    new_x_list, new_y_list = user_sample_array(train_df, thisuser, sampleperuser)\n    for xsample, ysample in zip(new_x_list, new_y_list):\n        x_train_list.append(xsample)\n        y_train_list.append(ysample)\n\nx_train = np.array(x_train_list)\ny_train = np.array(y_train_list)\n\ntime_finish = time.time()\nprint('Finished at', time.ctime())\nprint('Time elapsed:', int(time_finish - time_start), 'sec')\nprint('Shape of x_train:', x_train.shape)\nprint('Shape of y_train:', y_train.shape)\n\n#Delete all large variables not used past this point to conserve memory\ndel x_train_list, y_train_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Show example array for training\nfor i, thislabel in enumerate(ts_labels):\n    print('{:>32}'.format(thislabel), end=': ')\n    for val in x_train[0, -5:, i]:\n        print('{:>8.4f}'.format(val), end=' ')\n    print('')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Shuffle and split for validation\nx_train, x_val, y_train, y_val = train_test_split(\n    x_train,\n    y_train,\n    test_size = 1/10,\n    random_state = 17,\n    shuffle = True\n)\nprint('After split:')\nprint('# of training data:', x_train.shape[0])\nprint('# of validation data:', x_val.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Delete all large variables not used past this point to conserve memory\ndel train_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Creation and Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define hyperparameters\neta = 1E-1\ntotalepoch = 30\n\nlstm_neurons = 60\n\ninput_reg = 1E-5\nlstm_reg = 1E-5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Build model\nfrom tensorflow.keras.optimizers import Adam\n\nmodel = keras.models.Sequential()\nmodel.add(layers.Masking(mask_value=0., input_shape=(timesteps, features)))\nmodel.add(layers.LSTM(\n    lstm_neurons,\n    recurrent_regularizer=regularizers.l1(lstm_reg),\n    kernel_regularizer=regularizers.l1(input_reg)\n))\nmodel.add(layers.Dense(1, activation='sigmoid'))\n\nopt_adam = Adam(learning_rate=eta)\nmodel.compile(\n    optimizer=opt_adam,\n    loss='mse',\n    metrics=['accuracy']\n)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Fit model to training set\ntime_start = time.time()\nprint('Training start at', time.ctime())\nhistory = model.fit(\n    x_train,\n    y_train,\n    batch_size=1024,\n    epochs=totalepoch,\n    validation_data=(x_val, y_val),\n    verbose=0\n)\ntime_finish = time.time()\nprint('Training finished at', time.ctime())\nprint('Time elapsed:', int(time_finish - time_start), 'sec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(history.history)[['accuracy','val_accuracy']].plot(figsize=(12,6))\nplt.grid(True)\nplt.gca().set_ylim(0.6, 0.8)\nplt.title('Accuracy')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(history.history)[['loss','val_loss']].plot(figsize=(12,6))\nplt.grid(True)\nplt.gca().set_ylim(0.1, 0.3)\nplt.title('Loss/Error')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Apply model on the validation set, and show ROC curve\ny_val_pred = model.predict(x_val).ravel()\nfpr_val, tpr_val, thresh_val = roc_curve(y_val.astype(np.uint8), y_val_pred)\nauc_val = auc(fpr_val, tpr_val)\nplt.figure(figsize=(12,6))\nplt.plot([0, 1], [0, 1], 'k--')\nplt.plot(fpr_val, tpr_val, label='Val (area = {:.5f})'.format(auc_val))\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Show example prediction\nprint('{:>10}'.format('Expected'), '{:>11}'.format('Prediction'))\nfor i in range(15):\n    print('{:>10.7f}'.format(y_val[i]), '{:>11.7f}'.format(y_val_pred[i]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Get feature input weights (relative influence)\nweights = model.get_weights()\ninput_w = [0]*len(ts_labels)\nfor i in range(len(ts_labels)):\n    temp_array = weights[0][i,::4]\n    temp_array = [abs(j) for j in temp_array]\n    input_w[i] = np.mean(temp_array)\nscaled_w = 100*(input_w/max(input_w))\n\nprint('{:>30}'.format('FEATURE'), '{:>7}'.format('WEIGHT'), '{:>8}'.format('RELATIVE'))\nfor i, val in enumerate(input_w):\n    print(\n        '{:>30}'.format(ts_labels[i]),\n        '{:>7.4f}'.format(val),\n        '{:>8}'.format(scaled_w[i].astype(int))\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Delete all large variables not used past this point to conserve memory\ndel x_train, y_train, x_val, y_val","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# The environment can only be created once, per Riiid rules\n# If you need to rerun the script, first kill the session/kernel\nenv = riiideducation.make_env()\n\n#Load test data. Each batch of tests is along the principle axis, and must be followed in a strict order (to emulate the passage of time).\n#An iteration of test must call 'env.predict()' else there will be an error\niter_test = env.iter_test()\n\n#Apply model, and place results in prediction_df with the 'env.predict()' function, such as below\n#>>env.predict(prediction_df)\n#After the iter_test tuple is exhausted, the environment will output the final prediction in submission.csv\ntest_user_dict = {}\nold_user_list = []\nold_row_list = []\nold_test_df = pd.DataFrame({'empty': [0]})\nfor (test_df, prediction_df) in iter_test:\n    \n    #Preprocessing\n    test_df = test_df.reset_index(drop=True)\n    #Get answers from previous group\n    pgac_list = eval(str(test_df['prior_group_answers_correct'].iloc[0]))\n    #Drop columns\n    test_df.drop(['prior_group_responses', 'prior_group_answers_correct'], axis=1, inplace=True)\n    df_preprocess(test_df)\n    #Get user list and row list for this group\n    new_user_list = test_df.user_id.tolist()\n    new_row_list = test_df.row_id.tolist()\n    #Merge dictionaries\n    test_df = merge_all_dict(test_df)\n    #Create answer column\n    test_df['answered_correctly'] = [0.5]*len(test_df)\n    test_df.fillna(value=0, inplace=True)\n    \n    #Update answered_correctly in previous group, then insert to dictionary\n    if(pgac_list):\n        pgac_dict = pd.DataFrame.from_dict({'row_id': old_row_list, 'answered_correctly': pgac_list})\n        old_test_df = old_test_df.merge(pgac_dict, how='left', on='row_id')\n        for thisindex, (thisrow, thisuser) in enumerate(zip(old_row_list, old_user_list)):\n            if(old_test_df.iloc[thisindex]['content_type_id']==0):\n                if(not(thisuser in test_user_dict)):\n                    test_user_dict[thisuser] = {'rows': []}\n                test_user_dict[thisuser][thisrow] = old_test_df.iloc[thisindex].tolist()\n                test_user_dict[thisuser]['rows'].append(thisrow)\n                if(len(test_user_dict[thisuser]['rows']) > timesteps):\n                    row_remove = test_user_dict[thisuser]['rows'].pop(0)\n                    test_user_dict[thisuser].pop(row_remove)    \n    \n    #Predict for each user in group\n    x_test_list = []\n    for thisindex, (thisrow, thisuser) in enumerate(zip(new_row_list, new_user_list)):\n        if(thisuser in test_user_dict):\n            temp_dict = dict(test_user_dict[thisuser])\n            temp_dict.pop('rows')\n            database_df = pd.DataFrame.from_dict(temp_dict, orient='index', columns=test_df.columns)\n            user_df = pd.concat([database_df, test_df.loc[test_df.row_id==thisrow]])\n        else:\n            user_df = test_df.loc[test_df.row_id==thisrow]\n        new_test_x, new_test_y = user_sample_array(user_df, thisuser, 1)\n        x_test_list.append(new_test_x[0])\n        \n    x_test = np.array(x_test_list)\n    pred_list = model.predict(x_test)\n    update_df = pd.DataFrame({'row_id': new_row_list, 'prediction': pred_list.ravel()})\n    update_df.fillna(0.5, inplace=True)\n\n    #Save new -> old\n    old_test_df = test_df.drop(['answered_correctly'], axis=1)\n    old_user_list = list(new_user_list)\n    old_row_list = list(new_row_list)\n    \n    #Call env.predict()\n    prediction_df = prediction_df.merge(update_df, how='left', on='row_id')\n    prediction_df.drop(['answered_correctly'], axis=1, inplace=True)\n    prediction_df.rename(columns={'prediction': 'answered_correctly'}, inplace=True)\n    prediction_df = prediction_df.astype({'answered_correctly': 'float64'})\n    prediction_df.fillna(0.5, inplace=True)\n    env.predict(prediction_df)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}