{"cells":[{"metadata":{},"cell_type":"markdown","source":"<h1>Riid AIEd Challenge 2020 - Part III - Feature Engineering</h1>\n\nDue to memory/time restrictions in this competition, work is divided into several parts (kernels):\n\n<lu>\n    <li>Part I - Memory optimization</li>\n    <li>Part II - Splitting data</li>\n    <li>Part III - Feature engineering</li>\n    <li>Part IV - Training and validation</li>\n    <li>Part V - Prediction and submission</li>\n</lu>\n\n\nThis is Part III. In this part I'll\n\nCreate user and content features from past data (saved in previous kernel) and a History class responsible for expanding a data frame with the format of the training or test sets (columns user_id, content_id, etc.) with user and content features, and of updating history with the new interactions information."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Imports\n\nimport os\nimport pandas as pd\nimport numpy as np\nimport pickle\nimport gc\nimport warnings\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"warnings.filterwarnings(action='ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2>Load data</h2>"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define directories used\n\nDATA_DIR = '/kaggle/input/riiid-test-answer-prediction'\nPART_II_OUTPUT_DIR = '/kaggle/input/riiid-aied-part-ii-splitting/'\nWORKING_DIR = '/kaggle/working'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Load the competition data \npast_data = pd.read_pickle(os.path.join(PART_II_OUTPUT_DIR, 'past_data.pkl'))\npast_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"past_data.memory_usage()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# As we need enough memory for feature engineering, let's get rid of unneccesary columns\ndrop_columns = ['timestamp', 'user_answer', \n                'prior_question_elapsed_time', 'prior_question_had_explanation', 'virtual_timestamp']\npast_data.drop(columns=drop_columns, inplace=True)\n\n_ = gc.collect()\n\npast_data.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2>Build history dataframes</h2>"},{"metadata":{},"cell_type":"markdown","source":"<h3>Question performance features</h3>"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read questions meta data\n\nquestion_types = {\n    'question_id': np.int16,\n    'bundle_id':np.int16,\n    'correct_answer':np.int8,\n    'part':np.int8,\n    'tags':'object'\n}\n\nquestions = pd.read_csv(os.path.join(DATA_DIR, 'questions.csv'), dtype=question_types)\nquestions.set_index('question_id', drop=True, inplace=True)\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nquestion_performance_columns = [\n    'q_mean', 'part'\n]\n\nquestion_performance = past_data[past_data.content_type_id == False].groupby('content_id')['answered_correctly'].agg(['mean']).astype(np.float32)\nquestion_performance = pd.merge(question_performance, questions['part'], left_index=True, right_index=True)\nquestion_performance.columns = question_performance_columns\n\n_ = gc.collect()\n\nquestion_performance.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_performance.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2>Task container features</h2>"},{"metadata":{"trusted":true},"cell_type":"code","source":"task_container_performance_columns = [\n    'tc_mean'\n]\n\ntask_container_performance = past_data.groupby('task_container_id')[['answered_correctly']].agg('mean').astype(np.float32)\ntask_container_performance.columns = task_container_performance_columns\n\n_ = gc.collect()\n\ntask_container_performance.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h3>User performance features</h3>"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_performance_columns = [\n    'u_count', 'u_correct', 'u_mean'\n]\n\nuser_performance = past_data.groupby('user_id')['answered_correctly'].agg(['count', 'sum']).astype(np.int32)\nuser_performance['u_mean'] = (user_performance['sum'] / user_performance['count']).astype(np.float32)\nuser_performance.columns = user_performance_columns\n\n_ = gc.collect()\n\nuser_performance.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_performance.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npast_data = pd.concat([past_data.reset_index(drop=True), questions['part'].reindex(past_data.content_id.values).reset_index(drop=True)], axis=1)\n_ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2>Build history class</h2>"},{"metadata":{"trusted":true},"cell_type":"code","source":"class History:\n    def __init__(self, user_performance, question_performance, task_container_performance):\n        self.user_performance = user_performance\n        self.question_performance = question_performance\n        self.task_container_performance = task_container_performance\n        \n    def expand_features(self, df):\n        '''\n        Expand dataframe df with features from history. \n        '''\n        \n        expanded_df = pd.concat([df.reset_index(drop=True), \n                            self.user_performance.reindex(df.user_id.values).reset_index(drop=True),\n                            self.question_performance.reindex(df.content_id.values).reset_index(drop=True),\n                            self.task_container_performance.reindex(df.task_container_id.values).reset_index(drop=True)], axis=1)\n         \n        expanded_df.fillna(0.5, inplace=True)\n        \n        expanded_df['uq_hmean'] = 2 * expanded_df['u_mean'] * expanded_df['q_mean'] / (expanded_df['u_mean'] + expanded_df['q_mean'])\n        expanded_df['utc_hmean'] = 2 * expanded_df['u_mean'] * expanded_df['tc_mean'] / (expanded_df['u_mean'] + expanded_df['tc_mean'])\n        \n        return expanded_df\n        \n    def update_features_df(self, df):\n        new_users_ids = set(df.user_id).difference(self.user_performance.index.values)\n        if new_users_ids:\n            new_users_df = pd.DataFrame(0, index=new_users_ids, columns=self.user_performance.columns)\n            self.user_performance = pd.concat([self.user_performance, new_users_df], axis='rows')\n    \n        user_update = df.groupby('user_id', sort=False)['answered_correctly'].agg(['count', 'sum'])  \n        self.user_performance.loc[user_update.index, ['u_count', 'u_correct']] += user_update.values\n        self.user_performance['u_mean'] = self.user_performance.uq_correct / self.user_performance.uq_count\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This class is tested and profiled using the competition example test set in another <a href='https://www.kaggle.com/jcesquiveld/riiid-aied-optimize-history-class'>kernel</a>"},{"metadata":{},"cell_type":"markdown","source":"<h2>Save everything for next Part</h2>"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save history object \n\nhistory = History(user_performance, question_performance, task_container_performance)\nfilehandler = open(os.path.join(WORKING_DIR, 'past_history.pkl'), 'wb') \npickle.dump(history, filehandler)\nfilehandler.close()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2>Proof of concept</h2>\n\nTest features with lightgbm model with default parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(data=[[115, 0, 0, 1], [2746, 25, 1, 3], [5382, 4, 2, 3]], columns=['user_id', 'content_id', 'task_container_id', 'part'])\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nhistory.expand_features(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"proof_of_concept = True\n\ngc.collect()\n\n# Set the hyper parameters for the booster\nparams = {\n    'objective': 'binary',\n    'seed': 42,\n    'metric': 'auc',\n}\n\n# Set the features used\nFEATURES = [\n    'prior_question_had_explanation', 'prior_question_elapsed_time',\n    'u_count', \n    'u_mean',\n    'uq_hmean',\n    'q_mean',\n    'tc_mean',\n    'utc_hmean',\n    'part'\n]\n\n# Mean for prior_question_elapsed_time (from memory optimization notebook)\nprior_question_elapsed_time_mean = 25423.810042960275\n\nif proof_of_concept:\n    # Read data for one of the folds\n    train = pd.read_pickle(os.path.join(PART_II_OUTPUT_DIR, 'train_0.pkl'))\n    val = pd.read_pickle(os.path.join(PART_II_OUTPUT_DIR, 'val_0.pkl'))\n    \n    # Preprocessing\n    train = train.loc[train.content_type_id == False]\n    train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype(np.int8)\n    train['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)\n    train = history.expand_features(train)\n    val = val.loc[val.content_type_id == False]\n    val['prior_question_had_explanation'] = val.prior_question_had_explanation.fillna(False).astype(np.int8)\n    val['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)\n    val = history.expand_features(val)\n    \n    # Datasets\n    lgb_train = lgb.Dataset(train[FEATURES], train['answered_correctly'])\n    lgb_val = lgb.Dataset(val[FEATURES], val['answered_correctly'])\n    \n    # Train\n    model = lgb.train(\n            params,\n            lgb_train,\n            valid_sets = [lgb_train, lgb_val],\n            verbose_eval = 100,\n            num_boost_round = 10000,\n            early_stopping_rounds = 50\n        )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_importance(model, importance_type='split', figsize=(6,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_importance(model, importance_type='gain', figsize=(6,10))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"That's all folks!!!"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}