{"cells":[{"metadata":{},"cell_type":"markdown","source":"In kaggle competitions,  As a beginner,  there is an urge to create a model and test quickly. Also very often when we attempt to write a code from scratch, we see that the results are not as good as what we get when we 'copy and edit' a popular notebook. \n\nOne of the learnings have been the importance of using some kind of 'unit test' at each stage of coding. Simple errors occur for some corner cases (or) though initially we would have printed and checked the code, a small change would have affected the data  and it would have been overlooked.  \n\nTo start with we could use a simple 'assert' to ensure data correctness at each step. So for each method/function, write the assert statements. Ensure we decide/identify the values that need to be checked ( with assert) even before we execute the method. \n\nWe saw an increase of .4 points (.7x to .74) by fixing the errors in the methods\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt \nimport seaborn as sns\nfrom scipy.stats import pearsonr, spearmanr\nimport gc\nimport pyarrow.parquet as pq\nimport pyarrow as pa\nfrom tqdm import tqdm\nimport pickle\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# Read the data and the question information. \n##\ntrain_df = pd.read_pickle(\"/kaggle/input/riid-pickle-file/train.pkl\")\nquestion_static_info = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/questions.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define constants\nRUN_UNIT_TEST = 1  \nMODEL_PICKLE_FILENAME = 'lgb_2.pkl'\nTRAIN_DATA_LIMIT = -25000000\nLAST_ENTRIES_PER_USER_CNT = 600","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# This method calculates the user performance statistics - user performance, total attempts, total correct, mean time, lecture count. \n##\ndef calculate_user_summary_data (data):\n    user_temp = pd.DataFrame()\n    user_temp['user_id'] = data['user_id'].unique()   \n    \n    user_group =  data.groupby('user_id').agg( {'content_type_id': 'sum'} )\n    user_group.columns = ['lecture_count']\n    lc_dict = user_group['lecture_count'].to_dict()\n    \n    data = data[data ['content_type_id']==False]\n    user_group =  data.groupby('user_id').agg( {'answered_correctly': ['sum', 'count'], 'prior_question_elapsed_time': 'median'} )\n    user_group.columns = ['user_correct_count', 'user_all_count', 'user_mean_time']\n    user_correct_count_dict = user_group['user_correct_count'].to_dict()\n    user_all_count_dict = user_group['user_all_count'].to_dict()\n    user_mean_time_dict = user_group['user_mean_time'].to_dict()\n    \n    user_temp['user_mean_time'] = user_temp['user_id'].apply( lambda x: user_mean_time_dict[x] if x in user_mean_time_dict else 0  )\n    user_temp['user_correct_count'] = user_temp['user_id'].apply( lambda x: user_correct_count_dict[x] if x in user_correct_count_dict else 0  )\n    user_temp['user_all_count'] = user_temp['user_id'].apply( lambda x: user_all_count_dict[x] if x in user_all_count_dict else 0  )\n    user_temp['lecture_count'] = user_temp['user_id'].apply( lambda x: lc_dict[x]  if x in lc_dict else 0 )\n    user_temp['user_performance'] = user_temp['user_correct_count']/user_temp['user_all_count']\n    user_temp = user_temp.set_index('user_id')\n    return user_temp \n\n\nif RUN_UNIT_TEST == 1 :\n    # Create simple 'assert' to ensure that the method is indeed returning the right data. \n    # Initially there was a mistake and the lecture count was returned as zero ( since we were checking for the content_type_id == 0 )\n    # When that was fixed, the 'assert' for 'user_correct_count' failed. (The answered_correctly is -1 for lectures and this impacted the result).\n    # Hence by using 'assert' we could catch the 'bug' easily instead of much later in the code\n    punit_data = train_df[train_df['user_id'].isin([115, 5382])]\n    punit_data = punit_data.join(question_static_info[['tags', 'part']], on=\"content_id\" )\n    punit_user_info = calculate_user_summary_data(punit_data)\n    assert punit_user_info.loc[115]['user_all_count'] == 46\n    assert punit_user_info.loc[5382]['lecture_count'] == 3\n    assert punit_user_info.loc[5382]['user_correct_count'] == 84\n    assert punit_user_info.loc[5382]['user_mean_time'] == 25000\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# Calculate the question statistics \n##\ndef calculate_question_summary_data (data ):    \n    question_temp = pd.DataFrame()\n    data = data[ data['content_type_id']== False ]\n    question_temp['question_id'] = data['content_id'].unique()\n    \n    question_group =  data.groupby('content_id').agg( {'answered_correctly': ['sum', 'count']} )\n    question_group.columns = ['question_correct_count', 'question_all_count']\n    question_correct_count_dict = question_group['question_correct_count'].to_dict()\n    question_all_count_dict = question_group['question_all_count'].to_dict()\n    \n    question_temp['question_correct_count'] = question_temp['question_id'].apply( lambda x: question_correct_count_dict[x] )\n    question_temp['question_all_count'] = question_temp['question_id'].apply( lambda x: question_all_count_dict[x] )\n    question_temp['question_performance']  = question_temp['question_correct_count']/question_temp['question_all_count']\n    question_temp = question_temp.set_index('question_id')\n    return question_temp\n\n\n\nif RUN_UNIT_TEST == 1:\n    # One of the important test is to ensure that there is no leakage of the lecture information into the question\n    # summary \n    punit_data = train_df[train_df['content_id'].isin([5000, 16736,6808 ])]\n    punit_question_info = calculate_question_summary_data(punit_data)\n    assert punit_question_info.loc[5000]['question_correct_count'] == 11091\n    assert punit_question_info.loc[6808]['question_all_count'] == 17252\n    assert (16736 in punit_question_info.index) == False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n##\n# Calculate the user part statistics. \n##    \ndef calculate_user_part_summary_data(data) : \n    user_part_temp = pd.DataFrame()\n    data = data[ data['content_type_id']== False ]\n    user_part_temp['user_part'] = data['user_part'].unique()\n\n    user_part_data = data.groupby('user_part').agg ({'answered_correctly' : ['sum', 'count']})\n    user_part_data.columns = ['user_part_correct_count', 'user_part_all_count']\n    user_part_correct_count_dict = user_part_data['user_part_correct_count'].to_dict()\n    user_part_all_count_dict = user_part_data['user_part_all_count'].to_dict()\n    \n    user_part_temp['user_part_correct_count'] = user_part_temp['user_part'].apply( lambda x: user_part_correct_count_dict[x] )\n    user_part_temp['user_part_all_count'] = user_part_temp['user_part'].apply( lambda x: user_part_all_count_dict[x] )\n    user_part_temp['user_part_performance'] = user_part_temp['user_part_correct_count']/user_part_temp['user_part_all_count']\n    user_part_temp = user_part_temp.set_index ('user_part')\n    return user_part_temp\n\n\n\nif RUN_UNIT_TEST == 1:\n    punit_data = train_df[train_df['user_id'].isin([115, 5382])]\n    punit_data = punit_data.join(question_static_info[['tags', 'part']], on=\"content_id\" )\n    punit_data['user_part'] = punit_data['user_id'].astype(str) + \" \" + punit_data['part'].astype(str)\n    punit_user_part_info = calculate_user_part_summary_data(punit_data)\n    assert punit_user_part_info.loc['5382 5.0']['user_part_correct_count'] == 51 \n    assert punit_user_part_info.loc['5382 2.0']['user_part_all_count'] == 32 \n    assert punit_user_part_info.loc['115 5.0']['user_part_performance'] == 1 \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# Calculate the summary data\n##\ndef calculate_summary_data (data):\n    user_info = calculate_user_summary_data ( data )\n    question_info = calculate_question_summary_data (data )\n    user_part_info = calculate_user_part_summary_data(data)\n    return user_info, question_info, user_part_info","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# For split into train and val , we are going to consider the last 500 entries for the data. We will be using the 'task container id'\n# - using the initial ones for calculating the statistics and the last few ones for train/\n# validation. Since we do not want a constant split, we generate a random \n# number between 500 and max_container_id for the split between initial data and ones for train/val.\n## \n\nimport random\ndef add_addln_info_to_data (train_df, question_static_info ):\n    train_df = train_df.join(question_static_info[['tags', 'part']], on=\"content_id\" )\n    train_df['tags'] =  train_df['tags'].fillna(-1)\n    train_df['part'] =  train_df['part'].fillna(-1)\n    #train_df['user_tag'] = train_df['user_id'].astype(str) + \" \" + train_df['tags'].astype(str)\n    train_df['user_part'] = train_df['user_id'].astype(str) + \" \" + train_df['part'].astype(str)\n    \n    task_group_max = train_df.groupby('user_id').agg({'task_container_id' : 'max'})\n    task_group_max = task_group_max.rename (columns={'task_container_id': 'max_task_container_id'})\n    task_group_max['cut_off'] = task_group_max['max_task_container_id'].apply(lambda x: LAST_ENTRIES_PER_USER_CNT if x>=LAST_ENTRIES_PER_USER_CNT else  x)\n    task_group_max['random_count'] = task_group_max[['cut_off', 'max_task_container_id']].apply(lambda x:random.randint(x['cut_off'], x['max_task_container_id']), axis =1)\n    train_df = train_df.join(task_group_max[['max_task_container_id', 'random_count', 'cut_off']], on='user_id')\n    return train_df\n\n\n\nif RUN_UNIT_TEST == 1:\n    # Dummy dataframes can also be created to test methods. \n    punit_df = pd.DataFrame ( \n\n                 [{'user_id':1, 'content_id': 1, 'answered_correctly':1, 'content_type_id':0 , 'task_container_id': 1 } ,\n                 {'user_id':1, 'content_id': 2, 'answered_correctly':0, 'content_type_id':0 , 'task_container_id': 1 },\n                  {'user_id':1, 'content_id': 2, 'answered_correctly':0, 'content_type_id':1, 'task_container_id': 2  },\n                  {'user_id':2, 'content_id': 1, 'answered_correctly':0, 'content_type_id':0 , 'task_container_id': 1 },\n                  {'user_id':3, 'content_id': 2, 'answered_correctly':1, 'content_type_id':0 , 'task_container_id': 5550 }\n                 ], index = [0, 1, 2, 3, 4 ]\n        )\n\n    punit_question_static_info= pd.DataFrame ( \n\n                 [{'question_id' : 1 , 'tags': \"23 24\" , 'part': 1 } ,\n                  {'question_id' : 12 , 'tags': \"25\" , 'part': 2 }\n                 ], index = [1,2]\n        )\n    punit_addln_data = add_addln_info_to_data(punit_df,punit_question_static_info )\n    assert punit_addln_data.loc[0]['tags'] == \"23 24\"\n    assert punit_addln_data.loc[0]['part'] == 1\n    assert punit_addln_data.loc[0]['user_part'] == \"1 1\"\n    assert punit_addln_data.loc[0]['cut_off'] == 2\n    assert punit_addln_data.loc[4]['cut_off'] == LAST_ENTRIES_PER_USER_CNT      \n    assert punit_addln_data.loc[4]['max_task_container_id'] == 5550      ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# Merge the summarized data with the actual data based on user_id/content_id. \n##\ndef merge_summary_info_with_data_joinmthd (train_data_val, user_info, question_info, user_part_info ):\n    train_data_val = train_data_val.join(user_part_info[['user_part_correct_count', 'user_part_performance']], on='user_part')\n    train_data_val = train_data_val.join(user_info[['user_correct_count', 'lecture_count', 'user_performance', 'user_mean_time']], on='user_id')\n    train_data_val = train_data_val.join(question_info[['question_correct_count', 'question_performance']], on='content_id')\n    return train_data_val\n\n\ndef create_dictionaries(user_info, question_info, user_part_info) :\n    user_part_correct_count_dict = user_part_info['user_part_correct_count'].to_dict()\n    user_part_performance_dict = user_part_info['user_part_performance'].to_dict()\n    user_correct_count_dict = user_info['user_correct_count'].to_dict()\n    user_performance_dict = user_info['user_performance'].to_dict()\n    lecture_count_dict = user_info['lecture_count'].to_dict()\n    user_mean_time_dict = user_info['user_mean_time'].to_dict()\n    question_correct_count_dict = question_info['question_correct_count'].to_dict()\n    question_performance_dict = question_info['question_performance'].to_dict()\n    \n    dict_dict = {\n        'user_part_correct_count' : user_part_correct_count_dict,\n        'user_part_performance' : user_part_performance_dict,\n        'user_correct_count' :  user_correct_count_dict,\n        'lecture_count' :  lecture_count_dict,\n        'user_performance': user_performance_dict,\n        'user_mean_time' :  user_mean_time_dict,\n        'question_correct_count' :  question_correct_count_dict,\n        'question_performance' : question_performance_dict\n        \n        }\n\n    return dict_dict\n    \ndef merge_summary_info_with_data(train_data_val, dict_dict ):\n    \n    user_part_correct_count_dict = dict_dict.get('user_part_correct_count')\n    user_part_performance_dict = dict_dict.get('user_part_performance')\n    user_correct_count_dict=dict_dict.get('user_correct_count') \n    lecture_count_dict=dict_dict.get('lecture_count')\n    user_performance_dict=dict_dict.get('user_performance')\n    user_mean_time_dict=dict_dict.get('user_mean_time')\n    question_correct_count_dict=dict_dict.get('question_correct_count')\n    question_performance_dict=dict_dict.get('question_performance')\n    \n    \n    train_data_val['user_part_correct_count'] = train_data_val['user_part'].apply( lambda x: user_part_correct_count_dict[x] \\\n                                                                                  if x in user_part_correct_count_dict else 0 )\n    train_data_val['user_part_performance'] = train_data_val['user_part'].apply( lambda x: user_part_performance_dict[x] \\\n                                                                              if x in user_part_performance_dict else np.NAN )\n    train_data_val['user_correct_count'] = train_data_val['user_id'].apply( lambda x: user_correct_count_dict[x] \\\n                                                                           if x in user_correct_count_dict else 0 )\n    train_data_val['lecture_count'] = train_data_val['user_id'].apply( lambda x: lecture_count_dict[x] \\\n                                                                      if x in lecture_count_dict else 0 )\n    train_data_val['user_performance'] = train_data_val['user_id'].apply( lambda x: user_performance_dict[x]  \\\n                                                                         if x in user_performance_dict else np. NAN )\n    train_data_val['user_mean_time'] = train_data_val['user_id'].apply( lambda x: user_mean_time_dict[x]  \\\n                                                                         if x in user_mean_time_dict else np. NAN )\n    train_data_val['question_correct_count'] = train_data_val['content_id'].apply( lambda x: question_correct_count_dict[x] \\\n                                                                                   if x in question_correct_count_dict else 0 )\n    train_data_val['question_performance']  = train_data_val['content_id'].apply( lambda x: question_performance_dict[x] \\\n                                                                                 if x in question_performance_dict else np.NAN )\n  \n    return train_data_val\n\nif RUN_UNIT_TEST == 1:\n    punit_df = pd.DataFrame ( \n\n                 [{'user_id':1, 'content_id': 1, 'answered_correctly':1, 'content_type_id':0 , 'task_container_id': 1, 'tags': \"23 24\", \"part\":1 , 'user_part': \"1 1\"} ,\n                 {'user_id':1, 'content_id': 2, 'answered_correctly':0, 'content_type_id':0 , 'task_container_id': 1, 'tags': '25', 'part': 2, 'user_part': \"1 2\"},\n                  {'user_id':1, 'content_id': 2, 'answered_correctly':0, 'content_type_id':1, 'task_container_id': 2 , 'tags': '25', 'part': 2, 'user_part': \"1 2\" },\n                  {'user_id':2, 'content_id': 1, 'answered_correctly':0, 'content_type_id':0 , 'task_container_id': 1, 'tags': \"23 24\", \"part\":1, 'user_part': \"2 1\" },\n                  {'user_id':3, 'content_id': 2, 'answered_correctly':1, 'content_type_id':0 , 'task_container_id': 5550 , 'tags': '25', 'part': 2 , 'user_part': \"3 2\"}\n                 ], index = [0, 1, 2, 3, 4 ]\n        )\n\n    punit_question_info= pd.DataFrame ( \n\n                 [{ 'question_all_count': 100 , 'question_correct_count': 55, 'question_performance':.55 } ,\n                  { 'question_all_count': 200 , 'question_correct_count': 50, 'question_performance':.25 } \n                 ], index = [1,2]\n        )\n    punit_user_info= pd.DataFrame ( \n\n                 [{ 'user_all_count': 100 , 'user_correct_count': 50, 'user_performance':.50, 'lecture_count': 12, 'user_mean_time':30000 } ,\n                  { 'user_all_count': 200 , 'user_correct_count': 50, 'user_performance':.25 , 'lecture_count': 19, 'user_mean_time':25000 } \n                 ], index = [1,2]\n        )\n    punit_user_part_info= pd.DataFrame ( \n\n                 [{ 'user_part_all_count': 10 , 'user_part_correct_count': 7, 'user_part_performance':.70 } ,\n                  { 'user_part_all_count': 20 , 'user_part_correct_count': 5, 'user_part_performance':.25  } ,\n                  { 'user_part_all_count': 10 , 'user_part_correct_count': 1, 'user_part_performance':.1  }\n                 ], index = [\"1 1\",\"2 1\", \"1 2\"]\n        ) \n    \n    dict_dict = create_dictionaries(punit_user_info,punit_question_info, punit_user_part_info)\n    punit_result = merge_summary_info_with_data(punit_df, dict_dict)\n\n    assert punit_result.loc[0]['user_performance'] == .50\n    assert punit_result.loc[0]['user_part_performance'] == .70\n    assert punit_result.loc[0]['question_performance'] == .55\n    assert punit_result.loc[0]['question_correct_count'] == 55   \n    assert punit_result.loc[0]['user_correct_count'] == 50\n    assert punit_result.loc[0]['lecture_count'] == 12\n    assert punit_result.loc[0]['user_part_correct_count'] == 7\n    \n    assert np.isnan(punit_result.loc[4]['user_performance']) == True\n    assert np.isnan(punit_result.loc[4]['user_part_performance']) == True\n    assert punit_result.loc[4]['question_performance'] == .25\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# Handle the missing values in the data before training \n##\ndef handle_missing_vals (train_data_val ) :\n    \n    \n    train_data_val['user_correct_count'] = train_data_val['user_correct_count'].fillna(0)\n    train_data_val['lecture_count'] = train_data_val['lecture_count'].fillna(0)\n    train_data_val['user_mean_time'] = train_data_val['user_mean_time'].fillna(26000)\n    \n    train_data_val['question_correct_count'] = train_data_val['question_correct_count'].fillna(0)\n    train_data_val['user_part_correct_count'] = train_data_val['user_part_correct_count'].fillna(0)\n    \n    train_data_val['user_performance'] = train_data_val['user_performance'].fillna(train_data_val['user_performance'].median())\n    train_data_val['question_performance'] = train_data_val['question_performance'].fillna(train_data_val['question_performance'].median())\n    \n    train_data_val['user_part_performance'] = train_data_val[['user_part_performance', 'user_performance']].apply ( \\\n        lambda x : x['user_part_performance'] if np.isnan(x['user_part_performance'])==False else x['user_performance'], axis=1)\n    \n    return train_data_val\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# Consider only the last TRAIN_DATA_LIMIT entries and add the additional info for spliting into train/val\n# and for feature engineering\n##\nFEATURE_COLUMNS = ['user_performance', 'user_mean_time', 'lecture_count', \\\n                'user_correct_count', 'question_performance', \\\n                'question_correct_count', 'user_part_performance', 'user_part_correct_count']\nTARGET_COLUMN = 'answered_correctly'\n\n\ntrain_df = train_df[TRAIN_DATA_LIMIT:-1]\ntrain_df = add_addln_info_to_data ( train_df, question_static_info  )\nunique_users = train_df['user_id'].unique()\ntrain_df = train_df.reset_index()\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n##\n# Use the initial set of data ( in the last 500 entries for a user) to calculate the statistics \n##\ninitial_data = train_df[train_df['task_container_id'] >= train_df['cut_off']]\ninitial_data = initial_data[initial_data['task_container_id'] < initial_data['random_count']]\ninitial_data = initial_data.reset_index()\nuser_info, question_info, user_part_info = calculate_summary_data(initial_data)\n\n##\n# Use the remaining of data ( in the last 500 entries for a  user) for train/val.\n##\ntrain_data_val = train_df[train_df['task_container_id'] >= train_df['cut_off']]\ntrain_data_val = train_data_val[train_data_val['task_container_id'] >= train_data_val['random_count']]\ntrain_data_val = train_data_val[train_data_val['content_type_id'] == 0 ]\ntrain_data_val = train_data_val.reset_index()\ntrain_data_val = merge_summary_info_with_data_joinmthd(train_data_val, user_info, question_info, user_part_info )\ntrain_data_val = handle_missing_vals (train_data_val )\n\ntrain_data_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# Take 90% of the users to train the data and the remaining for validation. Create a custom cv  \n##\n\nfrom random import sample \n\nlen_unique_user = len(unique_users)\ntrain_cnt = int(len_unique_user*.9)\nval_cnt = len_unique_user-train_cnt\n\ndef custom_cv():\n        for i in range (0, 3):\n            sampling_list = sample ( unique_users.tolist(), len_unique_user)\n            yield train_data_val[ train_data_val['user_id'].isin( sampling_list[:train_cnt]) ], \\\n                  train_data_val[ train_data_val['user_id'].isin( sampling_list[train_cnt:])]\n            \n# Check if the custom cross validation code is working as expected. \nccv = custom_cv()\nfor i in ccv:\n    print(i)\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.metrics import accuracy_score\n\ntrain_data, val_data = next(custom_cv())\n\n# This below code for creating/training the model has been taken from https://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering\n\nlgb_train = lgb.Dataset(train_data[FEATURE_COLUMNS], train_data['answered_correctly'])\nlgb_valid = lgb.Dataset(val_data[FEATURE_COLUMNS], val_data['answered_correctly'])\n\nmodel = lgb.train(\n                    {'objective': 'binary'}, \n                    lgb_train,\n                    valid_sets=[lgb_valid],\n                    verbose_eval=100,\n                    num_boost_round=10000,\n                    early_stopping_rounds=10\n                )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pickle.dump(model, open(MODEL_PICKLE_FILENAME, 'wb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# Check on a sample test data \n##\nfrom sklearn.metrics import roc_auc_score, accuracy_score\n\nmodel = pickle.load(open(MODEL_PICKLE_FILENAME, 'rb'))\ntemp_data, test_data = next(custom_cv())\nresult = model.predict ( test_data[FEATURE_COLUMNS] )\nprint ( roc_auc_score(test_data['answered_correctly'], result )) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df\ndel train_data\ndel val_data\ndel temp_data\ndel test_data \ndel user_info\ndel question_info\ndel user_part_info\ndel train_data_val\ndel initial_data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nlocal_vars = list(locals().items())\nfor var, obj in local_vars:\n    size = sys.getsizeof(obj)\n    if size > 1e7:\n        print(f'{var:<18}{size/1e6:>10,.1f} MB')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# Update the existing summary data based on the results obtained during testing in each batch \n##\ndef update_user_info(df, user_info):\n    for row in df[['user_id','answered_correctly','content_type_id']].values:\n        if row[2] == 0:\n            if int( row[0] ) in user_info.index :\n                user_info.loc[int(row[0]) ,'user_correct_count'] += row[1]\n                user_info.loc[int(row[0]) , 'user_all_count'] += 1\n                user_info.loc[int(row[0]) , 'lecture_count'] += 0\n            else :\n                user_info = user_info.append ( pd.DataFrame (\n                                        {\n                                            'user_correct_count' : row[1],\n                                            'user_all_count': 1 ,\n                                            'lecture_count': 0 ,\n                                            'user_mean_time': 26000\n                                        }, index = [int(row[0])]\n                    \n                                    ) \n                                 \n                                 \n                                 )\n    user_info['user_performance'] = user_info['user_correct_count']   / user_info ['user_all_count'] \n    return user_info \n\n\n\n\nif RUN_UNIT_TEST == 1:\n    punit_df = pd.DataFrame ( \n\n                 [{'user_id':1, 'content_id': 1, 'answered_correctly':1, 'content_type_id':0 } ,\n                 {'user_id':1, 'content_id': 2, 'answered_correctly':0, 'content_type_id':0 },\n                  {'user_id':1, 'content_id': 2, 'answered_correctly':0, 'content_type_id':1 },\n                  {'user_id':2, 'content_id': 1, 'answered_correctly':0, 'content_type_id':0 }\n                 ], index = [0, 1, 2, 3 ]\n        )\n\n    punit_user_info= pd.DataFrame ( \n\n                 [{'user_correct_count' : 11 , 'user_all_count': 20 , 'user_mean_time': 30000, 'user_performance': 0.5, 'lecture_count': 3 } ,\n                  {'user_correct_count' : 5 , 'user_all_count': 5 , 'user_mean_time': 20000, 'user_performance': 1.0 , 'lecture_count': 4} ,\n                 ], index = [1,3]\n        )\n    punit_new = update_user_info(punit_df , punit_user_info)\n    assert punit_new.loc[1]['user_all_count'] == 22\n    assert punit_new.loc[2]['user_correct_count'] == 0\n    assert punit_new.loc[2]['user_all_count'] == 1\n    assert punit_new.loc[2]['user_mean_time'] == 26000\n    assert punit_new.loc[3]['user_correct_count'] == 5 \n    assert punit_new.loc[3]['user_all_count'] == 5 \n    assert punit_new.loc[3]['user_mean_time'] == 20000 \n    assert punit_new.loc[3]['user_performance'] == 1.0 \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##\n# Precalculate the summary information for the data. This is done to save time. \n# Also it is observed that better results are obtained when the entire data is considered for \n# calculating the summary \n##\nuser_info = pq.read_table(\"/kaggle/input/user-info-with-mean-time/user_info_final-2.parquet\").to_pandas()\nquestion_info = pq.read_table(\"/kaggle/input/riidinferenceinput1/question_summary.parquet\").to_pandas()\nuser_part_info = pq.read_table(\"/kaggle/input/riidinferenceinput1/user_part_info_summary.parquet\").to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_performance_median = user_info['user_performance'].median()\nquestion_performance_median = question_info['question_performance'].median()\nuser_part_correct_count_dict = user_part_info['user_part_correct_count'].to_dict()\nuser_part_performance_dict = user_part_info['user_part_performance'].to_dict()\nuser_correct_count_dict = user_info['user_correct_count'].to_dict()\nuser_performance_dict = user_info['user_performance'].to_dict()\nuser_lecture_dict = user_info['lecture_count'].to_dict()\nuser_mean_time_dict = user_info['user_mean_time'].to_dict()\nquestion_correct_count_dict = question_info['question_correct_count'].to_dict()\nquestion_performance_dict = question_info['question_performance'].to_dict()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime\nimport random\nprint ( datetime.now() , \"Starting Inference\")\ndict_dict = create_dictionaries(user_info,question_info, user_part_info)\nfor (test_df, sample_prediction_df) in iter_test:\n    \n    test_df = test_df[test_df['content_type_id' ] == 0]\n    \n    print ( datetime.now(), test_df.shape[0] ) \n    test_users = test_df['user_id'].unique() \n\n    \n    X_val = test_df[test_df['content_type_id' ] == 0][['user_id', 'content_id']] \n    X_val = X_val.join(question_static_info['part'], on=\"content_id\" )\n    X_val['part'] =  X_val['part'].fillna(-1)\n    X_val['user_part'] = X_val['user_id'].astype(str) + \" \" + X_val['part'].astype(str)+\".0\"  \n     \n    X_val = merge_summary_info_with_data(X_val, dict_dict)\n    X_val = handle_missing_vals (X_val )\n\n    print ( datetime.now() , \"Merged the data \")\n    \n    X_val = X_val[FEATURE_COLUMNS]\n    result = model.predict(X_val)\n    \n    print( datetime.now(), \"Prediction complete for this batch\")\n    \n    test_df[TARGET_COLUMN] = result \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n    user_info = update_user_info(test_df, user_info)\n\n\n    \n    ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}