{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#Imports\nimport numpy as np       #Numpy for numerical computations\nimport pandas as pd      #Pandas for data manipulations\nimport riiideducation    #Package for the competition API\nimport seaborn as sns    #Seaborn for data vizualisation\nimport os\nimport gc                #For garbage collector\n\n#Import data\nfor dirname, _, filenames in os.walk('/kaggle/input/riiid-test-answer-prediction'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Reading data, using a pickle to read it faster (15 seconds more or less)\nfull_train = pd.read_pickle(\"../input/train-gzip/riiid_train.gzip\")\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nlectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"full_train = full_train[['row_id','user_id','content_id','content_type_id','answered_correctly']]\ntrain = full_train.groupby('user_id').tail(500)\ntest = full_train.groupby('user_id').tail(4)\ntrain = train.drop(test.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Dictionnary for questions average\nquestion_average = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'])['answered_correctly'].mean()).rename(columns={'answered_correctly':'question_average'})\n#Dictionnary for questions count\nquestion_count = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id']).size(),columns=['question_count'])\n#Joining average and count\nquestion_df = question_average.join(question_count)\n#Computing sum as product of average and count\nquestion_df['question_sum'] = question_df['question_average'] * question_df['question_count']\n#Joining the new dataframe with questions data, getting more columns\nquestion_df = question_df.join(questions,how='outer')[['question_average','question_count','question_sum']]\n#Filling with default value\nquestion_df['question_average'].fillna(0,inplace=True)\nquestion_df['question_count'].fillna(0,inplace=True)\nquestion_df['question_sum'].fillna(0,inplace=True)\n#Cleaning for memory management\ndel question_average,question_count\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Cleaning full_train to keep only train and test set (as the full_train is too big)\ndel full_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Joining the average mark for the question to the train data\ntrain = train.join(question_df,on=['content_id'], rsuffix='_question')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Computing the mean by user and the mean by question to fill empty values\nmean_user = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].mean().mean()\nmean_question = train.loc[train['content_type_id'] == False].groupby(['content_id'])['answered_correctly'].mean().mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Computing the average for the questions that the user answered to in the past\ntrain['user_shift_question'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['question_average'].shift()\ncumulated_question = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift_question'].agg(['cumsum','cumcount'])\ntrain.loc[train['content_type_id'] == False,'average_past_questions'] = cumulated_question['cumsum'] / cumulated_question['cumcount']\ntrain['average_past_questions'].fillna(mean_question,inplace=True)\ntrain.drop(['user_shift_question','question_count','question_sum'],axis=1,inplace=True)\ndel cumulated_question","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Computing the average for each user he or she has until now\ntrain['user_shift'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].shift()\ncumulated = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift'].agg(['cumsum', 'cumcount'])\ntrain.loc[train['content_type_id'] == False,'answered_correctly_user_average'] = cumulated['cumsum'] / cumulated['cumcount']\ntrain['answered_correctly_user_average'].fillna(mean_user,inplace=True)\ntrain.drop(columns=['user_shift'], inplace=True)\ndel cumulated","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Data for user average\nuser_average = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'])['answered_correctly_user_average'].last()).rename(columns={'answered_correctly_user_average':'user_average'})\n#Data for user count\nuser_count = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id']).size() - 1,columns=['user_count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Here is what is working best at the moment (which is not base on the past events only)\n#To me, this is not suitable as it is a target leakage\n#To avoid this leakage, use the commented lines instead (which lead to a lower result until now)\ntmp = train.loc[train['content_type_id'] == False].groupby(['user_id']).mean()\nuser_performance = pd.DataFrame(tmp['answered_correctly'] - tmp['question_average'], columns=['performance'])\ndel tmp\n\n# train['performance'] = train['answered_correctly_user_average'] - train['average_past_questions']\n# user_performance = pd.DataFrame(train.loc[train['content_type_id'] == False].groupby(['user_id'])['performance'].last())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_df = user_performance.join(user_average).join(user_count)\nuser_df['user_sum'] = user_df['user_average'] * user_df['user_count']\ndel user_performance, user_count, user_average\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Utility function to get the sum of question's average that a user has in a new set\ndef question_average_sum_by_user(df,question_df):\n    my_dict = {}\n    group = df.groupby(['user_id'])\n    for user, val in group:\n        average_sum = 0.0\n        for row_index, row in val.iterrows():\n            if (row['content_type_id'] == False):\n                question_id = row['content_id']\n                question_average = question_df.at[question_id,'question_average']\n                average_sum += question_average\n    #         print(f'user = {user}, id = {question_id}, average = {question_average}, average_sum={average_sum}')\n        my_dict[user] = [average_sum]\n    return pd.DataFrame.from_dict(my_dict,orient='index',columns=['question_average_sum'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Utility function to add the answers of the prior_df into it to update performance and question average\ndef add_answers_to_prior_df(current_df,prior_df):\n    prior_df_ = prior_df.copy()\n    if (prior_df.shape[0] > 0):\n        val = eval(current_df.iloc[0]['prior_group_answers_correct'])\n        if (len(val) == prior_df.shape[0]):\n            prior_df_['answered_correctly_response'] = val\n    return prior_df_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Updating the question dataframe (especially for question average) for a new set of questions\ndef build_question_df(prior_df,question_df):\n    \n    if (prior_df.shape[0] == 0):\n        return question_df\n    \n    #Dictionnary for questions average\n    question_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\\\n                           .groupby(['content_id'])['answered_correctly_response'].sum())\\\n                           .rename(columns={'answered_correctly_response':'question_sum'})\n    \n    #Dictionnary for questions count\n    question_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\\\n                             .groupby(['content_id']).size(),columns=['question_count'])\n    \n    #Joining the two previous dataframes in one\n    question_df = question_df.join(question_sum_prior,rsuffix='_previous').join(question_count_prior,rsuffix='_previous')\n    \n    #Filling null values\n    question_df['question_average'].fillna(0,inplace=True)\n    question_df['question_count'].fillna(0,inplace=True)\n    question_df['question_sum'].fillna(0,inplace=True)\n    question_df['question_sum_previous'].fillna(0,inplace=True)\n    question_df['question_count_previous'].fillna(0,inplace=True)\n\n    #Updating values\n    question_df['question_sum'] = question_df['question_sum'] + question_df['question_sum_previous']\n    question_df['question_count'] = question_df['question_count'] + question_df['question_count_previous']\n    question_df['question_average'] = question_df['question_sum'] / question_df['question_count']\n    question_df.drop(['question_count_previous','question_sum_previous'],inplace=True,axis=1)\n    \n    return question_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Updating the user dataframe (especially for question average) for a new set of questions\ndef build_user_df(prior_df,user_df,question_df):\n    \n    if (prior_df.shape[0] == 0):\n        return user_df\n    \n    #Dictionnary for user average\n    user_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\\\n                       .groupby(['user_id'])['answered_correctly_response'].sum())\\\n                       .rename(columns={'answered_correctly_response':'user_sum'})\n    \n    #Dictionnary for user count\n    user_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\\\n                         .groupby(['user_id']).size(),columns=['user_count'])\n\n    #Joining the df with preexisting one\n    user_df = user_df.join(user_sum_prior,how='outer',rsuffix='_previous').join(user_count_prior,rsuffix='_previous')\n    \n    #Filling null values\n    user_df['performance'].fillna(0,inplace=True)\n    user_df['user_average'].fillna(0,inplace=True)\n    user_df['user_count'].fillna(0,inplace=True)\n    user_df['user_sum'].fillna(0,inplace=True)\n    user_df['user_count_previous'].fillna(0,inplace=True)\n    user_df['user_sum_previous'].fillna(0,inplace=True)\n    \n    #Computing the average of correct answers for the list of questions each user head in prior\n    user_df = user_df.join(question_average_sum_by_user(prior_df,question_df))\n    user_df['question_average_sum'].fillna(0,inplace=True)\n    \n    #Updating values\n    user_df['user_mean_performance'] = (user_df['user_sum'] - user_df['performance'] * user_df['user_count'] + user_df['question_average_sum']) / (user_df['user_count'] + user_df['user_count_previous'])\n    user_df['user_sum'] = user_df['user_sum'] + user_df['user_sum_previous']\n    user_df['user_count'] = user_df['user_count'] + user_df['user_count_previous']\n    user_df['user_average'] = user_df['user_sum'] / user_df['user_count']\n    user_df['performance'] = user_df['user_average'] - user_df['user_mean_performance']\n    user_df.drop(['user_sum_previous','user_count_previous','question_average_sum','user_mean_performance'],axis=1,inplace=True)\n    \n    return user_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Main code, initializing the dataframes\nprior_df = pd.DataFrame()\ncurrent_df = pd.DataFrame()\nprior_df = add_answers_to_prior_df(current_df,prior_df)\nquestion_df = build_question_df(prior_df,question_df)\nuser_df = build_user_df(prior_df,user_df,question_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prior_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"current_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET_COL = ['answered_correctly']\nFEATURE_COLS = ['row_id', 'performance', 'question_average']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[['row_id','user_id','content_id', 'content_type_id', 'answered_correctly']].reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_transform(df, is_training = True, is_validation = True): \n        \n    #Joining average marks for questions with the main dataframe\n    df = df.join(question_df['question_average'],on=['content_id'],rsuffix='_question_average')\n    \n    df = df.join(user_df[['performance','user_average', 'user_count']],on=['user_id'],rsuffix='_right')\n        \n    df = df.loc[df['content_type_id'] == False]\n    \n    if is_training or is_validation:\n        df = df[FEATURE_COLS + TARGET_COL]\n    else:\n        df = df[FEATURE_COLS]\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Transforming the train data\ntrain = data_transform(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Transforming the test data\ntest = data_transform(test,False,True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model\nWe use a lightgbm as proposed by many users\nI dit not dig into the parameters tuning until now"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Building the final train and test sets for lightgbm\nX_train = train[FEATURE_COLS]\ny_train = train[TARGET_COL]\nX_test = test[FEATURE_COLS]\ny_test = test[TARGET_COL]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nparams = {\n    'objective': 'binary',\n    'seed': 42,\n    'metric': 'auc',\n    'learning_rate': 0.025,\n    'max_bin': 1000,\n    'num_leaves': 80,\n    'num_iterations' : 100\n}\nlgb_train = lgb.Dataset(X_train.iloc[:,1:],y_train)\nlgb_val = lgb.Dataset(X_test.iloc[:,1:],y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X_train,y_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = lgb.train(\n    params,\n    lgb_train,\n    valid_sets=[lgb_train,lgb_val],\n    verbose_eval=1,\n    num_boost_round=100,\n    early_stopping_rounds=10\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npredictions = pd.DataFrame(model.predict(X_test.iloc[:,1:]),index=X_test.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nroc_auc_score(y_test,predictions[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nenv = riiideducation.make_env()\niter_test = env.iter_test()\niter_nb = 0\n\nfor (current_df, sample_prediction_df) in iter_test:\n    if (iter_nb != 0):\n        prior_df = add_answers_to_prior_df(current_df,prior_df)\n        question_df = build_question_df(prior_df,question_df)\n        user_df = build_user_df(prior_df,user_df,question_df)\n        \n    prior_df = current_df.copy()\n    current_df = data_transform(current_df,False,False)\n    current_df['answered_correctly'] = model.predict(current_df.iloc[:,1:])\n    iter_nb = 1\n    env.predict(current_df.loc[:, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}