{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Riiid! Answer Correctness Prediction - Intro\nTeam: EE258_F20_StudentPredictor\n\nAuthor: Rendale Mark Taas\n\nLast edited: 31-Dec-2020\n\nThis model was designed to use a complex network with an LSTM to extract time dependant trends. Since data is formatted as a time series, inputs are build on a per user basis. When training, a number of samples is pulled from each user. When testing, a database is saved and updated at each group/batch and takes into account new users. This database is in the variable 'test_user_dict'.\n\nThe hyperparameters are: the number of time steps in a series (variable timesteps), the number of samples per user (variable sampleperuser), the number of neurons per layer (variable lstm_neurons, ct_neurons, and dense_neurons), the learning rate (variable eta), and the number of training epochs (variable totalepoch)."},{"metadata":{"_uuid":"03acb6a0-86dd-400f-96eb-18a978ea3cfe","_cell_guid":"b1ee63f9-e3fb-41b6-8b5f-0533d1268e12","trusted":true},"cell_type":"code","source":"# Script for competition \"Riiid! Answer Correctness Prediction\"\n# Competition summary can be found here:\n# https://www.kaggle.com/c/riiid-test-answer-prediction/overview\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nimport pandas as pd\nimport numpy as np\nimport time\nimport math\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import regularizers\nfrom sklearn.preprocessing import StandardScaler\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import auc\n\nimport riiideducation","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Set Processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load training data from csv file\n#Data types are explicitly specified to minimize memory usage\nsubset_size = int(6E6)\ntrain_df = pd.read_csv(\n    '/kaggle/input/riiid-test-answer-prediction/train.csv',\n    low_memory=False,\n    nrows=subset_size,\n    dtype={\n        'row_id': 'int64',\n        'timestamp': 'int64',\n        'user_id': 'int32',\n        'content_id': 'int16',\n        'content_type_id': 'int8',\n        'task_container_id': 'int16',\n        'user_answer': 'int8',\n        'answered_correctly': 'int8',\n        'prior_question_elapsed_time': 'float32', \n        'prior_question_had_explanation': 'boolean'\n    }\n)\nprint(train_df.dtypes)\nprint('From .csv loading train_df: rows, columns =', train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Quick glimpse at the training data\ntrain_df.iloc[80:100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Remove unused columns\ntrain_df.drop(['user_answer'], axis=1, inplace=True)\nprint('After dropping train_df: rows, columns =', train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define data frame preprocesser (removes nans and converts bools)\nmean_pqet = train_df['prior_question_elapsed_time'].mean()\ndef df_preprocess(in_df):\n    in_df['prior_question_elapsed_time'].fillna(value=mean_pqet, inplace=True)\n    in_df['prior_question_had_explanation'] = in_df[\n        'prior_question_had_explanation'\n    ].fillna(value=False).astype(bool).map({True:1, False:0})\n    in_df.fillna(value=0, inplace=True)\n    \ndf_preprocess(train_df)\nprint('After preprocessing train_df: rows, columns', train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Group by user and how many rows they have in the dataset\ntrain_by_user_id = train_df.groupby(by='user_id')\nuser_list = list(train_by_user_id.groups.keys())\nprint('There are', len(user_list), 'users')\nprint('Average rows per user:', train_by_user_id.count().mean()[0])\nprint('Min rows of one user:', train_by_user_id.count().min()[0])\nprint('Max rows of one user:', train_by_user_id.count().max()[0])\n\n#Delete all large variables not used past this point to conserve memory\ndel train_by_user_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Group by question, and create dictionary\n#Dictionary has the question's mean, standard deviation, and skew\ntrain_by_content_id = train_df.loc[train_df.content_type_id==0].groupby(by='content_id')\ncontent_id_dict = train_by_content_id.agg({'answered_correctly': [np.mean, np.std, 'skew']}).copy()\ncontent_id_dict.columns = ['content_mean', 'content_std', 'content_skew']\ncontent_id_dict.fillna(value=0, inplace=True)\n\n#Delete all large variables not used past this point to conserve memory\ndel train_by_content_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Group by container id, and create dictionary\n#Dictionary has the container's mean, standard deviation, and skew\ntrain_by_container = train_df.loc[train_df.content_type_id==0].groupby(by='task_container_id')\ncontainer_dict = train_by_container.agg({'answered_correctly': [np.mean, np.std, 'skew']}).copy()\ncontainer_dict.columns = ['container_mean', 'container_std', 'container_skew']\ncontainer_dict.fillna(value=0, inplace=True)\n\n#Delete all large variables not used past this point to conserve memory\ndel train_by_container","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#From the questions metadata, create dictionary of part and tags\nprint('Start building questions_df at', time.ctime())\ntime_start = time.time()\n\nquestions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nquestions_df.drop(['bundle_id', 'correct_answer'], axis=1, inplace=True)\n#Tagless questions are given the tag '-1'\nquestions_df.fillna(value=-1, inplace=True)\nquestions_df.rename(columns={'question_id':'content_id'}, inplace=True)\ntotal_q = len(questions_df)\n\ntag_list = []\nfirst_tag_list = []\ntag_counts = {}\nfor thisindex, thisstr in enumerate(questions_df.tags):\n    temp_list = str(thisstr).split(' ')\n    temp_list = [int(i) for i in temp_list]\n    tag_list.append(temp_list)\n    first_tag_list.append(temp_list[0])\n    for thistag in temp_list:\n        if (not(thistag in tag_counts)):\n            tag_counts[thistag] = 0\n        tag_counts[thistag] = tag_counts[thistag] + 1\n\nquestions_df['tags'] = tag_list\nquestions_df['first_tag'] = first_tag_list\n\ntag_dict = {}\nfor thistag in tag_counts:\n    temp_df = questions_df.loc[\n        [(thistag in questions_df.tags[i]) for i in range(total_q)]\n    ]\n    temp_list = temp_df.content_id.tolist()\n    tag_dict[thistag] = temp_list\n\n#Delete all large variables not used past this part to conserve memory\ndel tag_list, first_tag_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#See the frequency of each tag\nplt.figure(figsize=(12,6))\nplt.plot(*zip(*sorted(tag_counts.items())))\nplt.xlabel('Tag')\nplt.ylabel('Count of questions with that tag')\nplt.title('Tag Count')\nplt.grid(True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Get statistics per tag\nstats_tag_dict = {}\nfor thistag in tag_dict:\n    temp_df = train_df.loc[train_df.content_type_id==0]\n    temp_df = temp_df.loc[temp_df['content_id'].isin(tag_dict[thistag])]\n    tag_weight = len(temp_df)\n    temp_df = temp_df.agg({'answered_correctly': [np.mean, np.std, 'skew']})\n    if(not(thistag in stats_tag_dict)):\n        stats_tag_dict[thistag] = {}\n    stats_tag_dict[thistag]['tag_mean'] = temp_df.loc['mean'][0]\n    stats_tag_dict[thistag]['tag_std'] = temp_df.loc['std'][0]\n    stats_tag_dict[thistag]['tag_skew'] = temp_df.loc['skew'][0]\n    stats_tag_dict[thistag]['weight'] = tag_weight\n\n#For each question find the weighted average statistic\ncomb_tag_stat_list = []\nfor thislist in questions_df.tags:\n    mean_list = []\n    std_list = []\n    skew_list = []\n    weight_list = []\n    for thistag in thislist:\n        mean_list.append(stats_tag_dict[thistag]['tag_mean'])\n        std_list.append(stats_tag_dict[thistag]['tag_std'])\n        skew_list.append(stats_tag_dict[thistag]['tag_skew'])\n        weight_list.append(stats_tag_dict[thistag]['weight'])\n    if(weight_list[0]==0):\n        comb_tag_stat_list.append(np.array([0.5, 0, 0]))\n    else:\n        temp_array = np.array([mean_list, std_list, skew_list])\n        comb_tag_stat_list.append(np.average(temp_array, axis=1, weights=weight_list))\n\nquestions_df = questions_df.merge(\n    pd.DataFrame(comb_tag_stat_list),\n    left_index=True,\n    right_index=True\n)\nquestions_df.rename(\n    columns={\n        0: 'tag_mean',\n        1: 'tag_std',\n        2: 'tag_skew'\n    },\n    inplace=True\n)\nquestions_df.drop(['tags'], axis=1, inplace=True)\n\ntime_finish = time.time()\nprint('Finished questions_df at', time.ctime())\nprint('Time elapsed:', int(time_finish - time_start), 'sec')\n\n#Delete all large variables not used past this part to conserve memory\ndel stats_tag_dict, tag_dict, tag_counts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#From the lectures metadata, create dictionary of part and tags\nlectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')\nlectures_df.fillna(value=0)\nlectures_df.drop(['type_of'], axis=1, inplace=True)\nlectures_df.rename(columns={'lecture_id': 'content_id', 'tag': 'first_tag'}, inplace=True)\nlectures_df['tag_mean'] = -1\nlectures_df['tag_std'] = 0\nlectures_df['tag_skew'] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define function to merge all dictionaries onto dataset\ndef merge_all_dict(in_df):\n    temp_df = in_df.merge(content_id_dict, how='left', on='content_id')\n    temp_df = temp_df.merge(container_dict, how='left', on='task_container_id')\n    temp_df.loc[\n        temp_df.content_type_id==1,\n        ['container_mean', 'content_mean']\n    ] = -1\n    temp_df.loc[\n        temp_df.content_type_id==1,\n        ['container_std', 'container_skew', 'content_std', 'content_skew']\n    ] = 0\n    \n    leconly_df = temp_df.loc[temp_df.content_type_id==1]\n    leconly_df = leconly_df.merge(lectures_df, how='left', on='content_id')[\n        ['row_id', 'first_tag', 'part', 'tag_mean', 'tag_std', 'tag_skew']\n    ]\n    queonly_df = temp_df.loc[temp_df.content_type_id==0]\n    queonly_df = queonly_df.merge(questions_df, how='left', on='content_id')[\n        ['row_id', 'first_tag', 'part', 'tag_mean', 'tag_std', 'tag_skew']\n    ]\n    \n    temp_df = temp_df.merge(leconly_df, how='left', on='row_id').combine_first(queonly_df)\n    \n    return temp_df.fillna(value=0).reset_index(drop=True)\n\ntrain_df = merge_all_dict(train_df)\nprint('After merge, (rows, columns) =', train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Quick glimpse at the data after processing\ntrain_df.iloc[80:100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define the labels of the input vector\n#Columns that need to be scaled are first\nscaled_labels = [\n    'timestamp',\n    'prior_question_elapsed_time',\n    'content_id',\n    'part',\n    'first_tag',\n    'task_container_id'\n]\nstat_labels = [\n    'content_mean',\n    'content_std',\n    'content_skew',\n    'container_mean',\n    'container_std',\n    'container_skew',\n    'tag_mean',\n    'tag_std',\n    'tag_skew'\n]\nother_labels = [\n    'prior_question_had_explanation',\n    'content_type_id',\n    'answered_correctly'\n]\nall_labels = scaled_labels + stat_labels + other_labels\nsindex = len(scaled_labels)\n\n#Define input shapes\ntimesteps = 50\nfeatures = len(all_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create scaler for first set of labels\nprint('Creating scaler for preprocessing:')\nscaler = StandardScaler()\nscaler.fit(train_df[scaled_labels].to_numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define function that will get the relevant features for time series\n#Will scale all features that need it\ndef build_ts_array(in_df):\n    out_np = in_df[all_labels].to_numpy()\n    out_np[:, 0:sindex] = scaler.transform(out_np[:, 0:sindex])\n    return out_np\n\n#Define function that will get features for current time sample\ndef build_ct_array(in_df):\n    out_np = in_df[all_labels[:-2]].to_numpy()\n    out_np[:, 0:sindex] = scaler.transform(out_np[:, 0:sindex])\n    return out_np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define function that returns samples of a given user\n#Data will be front padded if the number of rows is less than timesteps\ndef user_sample_array(in_df, inuser, numsamples):\n    temp_df = in_df.loc[in_df.user_id==inuser].reset_index(drop=True)\n    ts_list = []\n    ct_list = []\n    y_list = []\n    #.iloc includes the first argument, but excludes the second, thus ts is properly shifted\n    if(len(temp_df)>1):\n        for i in range(len(temp_df)-1, 0, -math.ceil(len(temp_df) / numsamples)):\n            if(temp_df.iloc[i].content_type_id==1):\n                continue\n            next_ts = build_ts_array(temp_df.iloc[max(0, i-timesteps):i])\n            next_ct = build_ct_array(temp_df.iloc[i:i+1])\n            next_y = temp_df['answered_correctly'].iloc[i]\n            if (len(next_ts) < timesteps):\n                next_ts = np.vstack((\n                    np.zeros((timesteps - len(next_ts), features)),\n                    next_ts\n                ))\n            ts_list.append(next_ts)\n            ct_list.append(next_ct)\n            y_list.append(next_y)\n    else:\n        if(temp_df.content_type_id[0]==0):\n            next_ts = np.zeros((timesteps, features))\n            next_ct = build_ct_array(temp_df)\n            next_y = temp_df['answered_correctly'].iloc[0]\n            ts_list.append(next_ts)\n            ct_list.append(next_ct)\n            y_list.append(next_y)\n    return ts_list, ct_list, y_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create the training set\nxts_train_list = []\nxct_train_list = []\ny_train_list = []\n\n#Building the set, each sample is the time series of one user\ntime_start = time.time()\nprint('Start creating training set at', time.ctime())\nsampleperuser = 5\nfor thisuser in user_list:\n    ts_list, ct_list, y_list = user_sample_array(train_df, thisuser, sampleperuser)\n    for tssample, ctsample, ysample in zip(ts_list, ct_list, y_list):\n        xts_train_list.append(tssample)\n        xct_train_list.append(ctsample)\n        y_train_list.append(ysample)\n\nxts_train = np.array(xts_train_list)\nxct_train = np.array(xct_train_list)\ny_train = np.array(y_train_list)\nxct_train = np.reshape(xct_train, (len(y_train), features-2))\n\ntime_finish = time.time()\nprint('Finished at', time.ctime())\nprint('Time elapsed:', int(time_finish - time_start), 'sec')\nprint('Shape of xts_train:', xts_train.shape)\nprint('Shape of xct_train:', xct_train.shape)\nprint('Shape of y_train:', y_train.shape)\n\n#Delete all large variables not used past this point to conserve memory\ndel xts_train_list, xct_train_list, y_train_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Show example time sample array for training\nprint('Time series example:')\nfor i, thislabel in enumerate(all_labels):\n    print('{:>32}'.format(thislabel), end=': ')\n    for val in xts_train[0, -5:, i]:\n        print('{:>8.4f}'.format(val), end=' ')\n    print('')\nprint('\\nCurrent time example:')\nfor i, thislabel in enumerate(all_labels[:-2]):\n    print('{:>32}'.format(thislabel), end=': ')\n    print('{:>8.4f}'.format(xct_train[0, i]), end=' ')\n    print('')\nprint('\\nExpected result from model:', y_train[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Shuffle and split for validation\nxts_train, xts_val, xct_train, xct_val, y_train, y_val = train_test_split(\n    xts_train,\n    xct_train,\n    y_train,\n    test_size = 1/10,\n    random_state = 17,\n    shuffle = True\n)\nprint('After split:')\nprint('# of training data:', xts_train.shape[0])\nprint('# of validation data:', xts_val.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Delete all large variables not used past this point to conserve memory\ndel train_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Creation and Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define hyperparameters\neta = 1E-2\ntotalepoch = 50\n\nlstm_neurons = 50\nct_neurons = 10\ndense_neurons = 10\n\nlstm_r_reg = 1E-5\nlstm_i_reg = 5E-4\nct_reg = 5E-4\ncomb_reg = 1E-5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Build model\nfrom tensorflow.keras.optimizers import Adam\n\n#Create inputs\nts_input = keras.Input(shape=(timesteps, features), name='ts_input')\nct_input = keras.Input(shape=(features-2), name='ct_input')\n\n#Time series path\nts_mask = layers.Masking(\n    mask_value=0.,\n    name='ts_mask'\n)(ts_input)\nts_state = layers.LSTM(\n    lstm_neurons,\n    recurrent_regularizer=regularizers.l1(lstm_r_reg),\n    kernel_regularizer=regularizers.l1(lstm_i_reg),\n    name='ts_lstm'\n)(ts_mask)\n\n#Current time path\nct_state = layers.Dense(\n    ct_neurons,\n    activation='relu',\n    kernel_regularizer=regularizers.l1(ct_reg),\n    name='ct_dense'\n)(ct_input)\n\n#Combined path\nstatemerge = layers.Concatenate(\n    name='comb_concat'\n)([ts_state, ct_state])\ncomb_dense = layers.Dense(\n    dense_neurons,\n    activation='relu',\n    kernel_regularizer=regularizers.l1(comb_reg),\n    name='comb_dense'\n)(statemerge)\nans_pred = layers.Dense(\n    1,\n    activation='sigmoid',\n    name='ans_pred'\n)(comb_dense)\n\n#Model compile\nmodel = keras.Model(\n    inputs=[ts_input, ct_input],\n    outputs=[ans_pred],\n    name='advancedmodel'\n)\n\nopt_adam = Adam(learning_rate=eta)\nmodel.compile(\n    optimizer=opt_adam,\n    loss='mse',\n    metrics=['AUC']\n)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Fit model to training set\ntime_start = time.time()\nprint('Training start at', time.ctime())\nhistory = model.fit(\n    (xts_train, xct_train),\n    y_train,\n    batch_size=1024,\n    epochs=totalepoch,\n    validation_data=((xts_val, xct_val), y_val),\n    verbose=0\n)\ntime_finish = time.time()\nprint('Training finished at', time.ctime())\nprint('Time elapsed:', int(time_finish - time_start), 'sec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(history.history)[['auc','val_auc']].plot(figsize=(12,6))\nplt.grid(True)\nplt.gca().set_ylim(0.7, 0.82)\nplt.title('Area Under Curve')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(history.history)[['loss','val_loss']].plot(figsize=(12,6))\nplt.grid(True)\nplt.gca().set_ylim(0.18, 0.26)\nplt.title('Loss/Error')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Apply model on the validation set, and show ROC curve\ny_val_pred = model.predict((xts_val, xct_val)).ravel()\nfpr_val, tpr_val, thresh_val = roc_curve(y_val.astype(np.uint8), y_val_pred)\nauc_val = auc(fpr_val, tpr_val)\nplt.figure(figsize=(10,10))\nplt.plot([0, 1], [0, 1], 'k--')\nplt.plot(fpr_val, tpr_val, label='Val (area = {:.5f})'.format(auc_val))\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='center right')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Show example prediction\nprint('{:>10}'.format('Expected'), '{:>11}'.format('Prediction'))\nfor i in range(10):\n    print('{:>10.7f}'.format(y_val[i]), '{:>11.7f}'.format(y_val_pred[i]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weights = model.trainable_weights\nlstm_input_w = [0]*features\nfor i in range(features):\n    temp_array = weights[0][i,::4]\n    temp_array = [abs(j) for j in temp_array]\n    lstm_input_w[i] = np.mean(temp_array)\nlstm_input_w_scaled = 1000*(lstm_input_w/max(lstm_input_w))\n\nprint('Time series/LSTM feature relevance')\nprint('{:>30}'.format('FEATURE'), '{:>9}'.format('WEIGHT'), '{:>8}'.format('RELATIVE'))\nfor i, val in enumerate(lstm_input_w):\n    print(\n        '{:>30}'.format(all_labels[i]),\n        '{:>9.6f}'.format(val),\n        '{:>8}'.format(lstm_input_w_scaled[i].astype(int))\n    )\n    \ndense_input_w = [0]*(features-2)\nfor i in range(features-2):\n    temp_array = weights[3][i]\n    temp_array = [abs(j) for j in temp_array]\n    dense_input_w[i] = np.mean(temp_array)\ndense_input_w_scaled = 1000*(dense_input_w/max(dense_input_w))\n\nprint('\\nCurrent time feature relevance')\nprint('{:>30}'.format('FEATURE'), '{:>9}'.format('WEIGHT'), '{:>8}'.format('RELATIVE'))\nfor i, val in enumerate(dense_input_w):\n    print(\n        '{:>30}'.format(all_labels[i]),\n        '{:>9.6f}'.format(val),\n        '{:>8}'.format(dense_input_w_scaled[i].astype(int))\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Delete all large variables not used past this point to conserve memory\ndel xts_train, xct_train, y_train, xts_val, xct_val, y_val","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# The environment can only be created once, per Riiid rules\n# If you need to rerun the script, first kill the session/kernel\nenv = riiideducation.make_env()\n\n#Load test data. Each batch of tests is along the principle axis, and must be followed in a strict order (to emulate the passage of time).\n#An iteration of test must call 'env.predict()' else there will be an error\niter_test = env.iter_test()\n\n#Apply model, and place results in prediction_df with the 'env.predict()' function, such as below\n#>>env.predict(prediction_df)\n#After the iter_test tuple is exhausted, the environment will output the final prediction in submission.csv\ntest_user_dict = {}\nold_user_list = []\nold_row_list = []\nold_test_df = pd.DataFrame({'empty': [0]})\nfor (test_df, prediction_df) in iter_test:\n    \n    #Preprocessing\n    test_df = test_df.reset_index(drop=True)\n    #Get answers from previous group\n    pgac_list = eval(str(test_df['prior_group_answers_correct'].iloc[0]))\n    #Drop columns\n    test_df.drop(\n        ['prior_group_responses', 'prior_group_answers_correct'],\n        axis=1,\n        inplace=True\n    )\n    df_preprocess(test_df)\n    #Get user list and row list for this group\n    new_user_list = test_df.user_id.tolist()\n    new_row_list = test_df.row_id.tolist()\n    #Merge dictionaries\n    test_df = merge_all_dict(test_df)\n    #Create answer column\n    test_df['answered_correctly'] = [0.5]*len(test_df)\n    test_df.fillna(value=0, inplace=True)\n    \n    #Update answered_correctly in previous group, then insert to dictionary\n    if(pgac_list):\n        pgac_dict = pd.DataFrame.from_dict(\n            {'row_id': old_row_list, 'answered_correctly': pgac_list}\n        )\n        old_test_df = old_test_df.merge(pgac_dict, how='left', on='row_id')\n        for thisindex, (thisrow, thisuser) in enumerate(zip(old_row_list, old_user_list)):\n            if(not(thisuser in test_user_dict)):\n                test_user_dict[thisuser] = {'rows': []}\n            test_user_dict[thisuser][thisrow] = old_test_df.iloc[thisindex].tolist()\n            test_user_dict[thisuser]['rows'].append(thisrow)\n            if(len(test_user_dict[thisuser]['rows']) > timesteps):\n                row_remove = test_user_dict[thisuser]['rows'].pop(0)\n                test_user_dict[thisuser].pop(row_remove)    \n    \n    #Predict for each user in group\n    xts_test_list = []\n    xct_test_list  =[]\n    prediction_rows = []\n    for thisrow, thisuser in zip(new_row_list, new_user_list):\n        temp_df = test_df.loc[test_df.row_id==thisrow]\n        if(temp_df.content_type_id.iloc[0]==0):\n            if(thisuser in test_user_dict):\n                temp_dict = dict(test_user_dict[thisuser])\n                temp_dict.pop('rows')\n                database_df = pd.DataFrame.from_dict(\n                    temp_dict,\n                    orient='index',\n                    columns=test_df.columns\n                )\n                user_df = pd.concat([database_df, temp_df])\n            else:\n                user_df = temp_df\n            next_ts, next_ct, next_y = user_sample_array(user_df, thisuser, 1)\n            xts_test_list.append(next_ts[0])\n            xct_test_list.append(next_ct[0])\n            prediction_rows.append(thisrow)\n        \n    xts_test = np.array(xts_test_list)\n    xct_test = np.array(xct_test_list)\n    xct_test = np.reshape(xct_test, (xts_test.shape[0], features-2))\n    pred_list = model.predict((xts_test, xct_test)).ravel()\n    update_df = pd.DataFrame({'row_id': prediction_rows, 'prediction': pred_list})\n    update_df.fillna(0.5, inplace=True)\n\n    #Save new -> old\n    old_test_df = test_df.drop(['answered_correctly'], axis=1)\n    old_user_list = list(new_user_list)\n    old_row_list = list(new_row_list)\n    \n    #Call env.predict()\n    prediction_df = prediction_df.merge(update_df, how='left', on='row_id')\n    prediction_df.drop(['answered_correctly'], axis=1, inplace=True)\n    prediction_df.rename(columns={'prediction': 'answered_correctly'}, inplace=True)\n    prediction_df = prediction_df.astype({'answered_correctly': 'float64'})\n    prediction_df.fillna(0.5, inplace=True)\n    env.predict(prediction_df)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}