{"cells":[{"metadata":{},"cell_type":"markdown","source":"# This notebook holds methods to prepare the training dataframe in small time for\n\n# https://www.kaggle.com/zyy2016/0-780-unoptimized-lgbm-interesting-features \n\n"},{"metadata":{},"cell_type":"markdown","source":"# Thanks Ordinary Student for your wonderful work\n\n## https://www.kaggle.com/zyy2016"},{"metadata":{},"cell_type":"markdown","source":"## The features are added into 1M training data records as efficiently as possiple"},{"metadata":{},"cell_type":"markdown","source":"\n\n"},{"metadata":{},"cell_type":"markdown","source":"# Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nRiiid Competition Submission Ver 1.0.1 Alpha\n(C) Copyright By Author 2020 - Now\nAll rights reserved\n'''\nimport sys\nsys.path.append('/kaggle/input/riiid-dataset/')\n#Dir\nquestion_metadata_dir = r'/kaggle/input/riiid-dataset/question_metadata.csv'\nlesson_metadata_dir = r'/kaggle/input/riiid-dataset/lesson_metadata.csv'\npickle_dir= r'/kaggle/input/riiid-dataset/stage.pickle'\nmodel_dir = r'/kaggle/input/riiid-dataset/classifier.model'\n\nimport datetime\n# Import some Stuff\n\nimport pandas as pd\nimport numpy as np\nimport pickle\nimport trueskill\nimport math\nimport lightgbm as lgb\nimport riiideducation\nimport time\nfrom sklearn.metrics import roc_auc_score\nfrom collections import Counter\nimport dask.dataframe as dd\nimport gc\nimport warnings \nwarnings.filterwarnings('ignore')\n\nprint(\"{} Import Completed\".format(str(datetime.datetime.now())))\nenv = trueskill.TrueSkill(mu=0.3, sigma=0.164486, beta=0.05, tau=0.00164, draw_probability=0)\nenv.make_as_global()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', low_memory=False,nrows=10**6 ,\n                       dtype={'row_id': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8',\n                              'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32', \n                             'prior_question_had_explanation': 'boolean'\n                             }\n                      )\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(props):\n    start_mem_usg = props.memory_usage().sum() / 1024**2 \n    print(\"Memory usage of properties dataframe is :\",start_mem_usg,\" MB\")\n    NAlist = [] # Keeps track of columns that have missing values filled in. \n    for col in props.columns:\n        if col in ['prior_question_elapsed_time','prior_question_had_explanation'] :\n            continue\n            \n        if props[col].dtype != object:  # Exclude strings\n            # make variables for Int, max and min\n            IsInt = False\n            mx = props[col].max()\n            mn = props[col].min()\n            \n            # Integer does not support NA, therefore, NA needs to be filled\n            if not np.isfinite(props[col]).all():\n                \n                NAlist.append(col)\n                props[col].fillna(mn-1,inplace=True)  \n                   \n            # test if column can be converted to an integer\n            asint = props[col].fillna(0).astype(np.int64)\n            result = (props[col] - asint)\n            result = result.sum()\n            if result > -0.01 and result < 0.01:\n                IsInt = True\n\n            \n\n            \n            # Make Integer/unsigned Integer datatypes\n            if IsInt:\n                if mn >= 0:\n                    if mx < 255:\n                        props[col] = props[col].astype(np.uint8)\n                    elif mx < 65535:\n                        props[col] = props[col].astype(np.uint16)\n                    elif mx < 4294967295:\n                        props[col] = props[col].astype(np.uint32)\n                    else:\n                        props[col] = props[col].astype(np.uint64)\n                else:\n                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:\n                        props[col] = props[col].astype(np.int8)\n                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:\n                        props[col] = props[col].astype(np.int16)\n                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:\n                        props[col] = props[col].astype(np.int32)\n                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:\n                        props[col] = props[col].astype(np.int64)    \n            \n            # Make float datatypes 32 bit\n            else:\n                props[col] = props[col].astype(np.float32)\n            \n            # Print new column type\n           # print(\"dtype after: \",props[col].dtype)\n           # print(\"******************************\")\n    \n    # Print final result\n    print(\"___MEMORY USAGE AFTER COMPLETION:___\")\n    mem_usg = props.memory_usage().sum() / 1024**2 \n    print(\"Memory usage is: \",mem_usg,\" MB\")\n    print(\"This is \",100*mem_usg/start_mem_usg,\"% of the initial size\")\n    return props","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def win_probability(team1, team2):\n    '''\n    Calculate the win possibility based on two Trueskill objects\n    :param team1:User TrueSkill Object\n    :param team2:Question Trueskill Object\n    :return: Winning Prob\n    '''\n    delta_mu = team1.mu - team2.mu\n    sum_sigma = sum([team1.sigma ** 2, team2.sigma ** 2])\n    size = 2\n    denom = math.sqrt(size * (0.05 * 0.05) + sum_sigma)\n    ts = trueskill.global_env()\n    \n    return [ts.cdf(elem / denom) for elem in delta_mu]\ndef win_probability_orig(team1, team2):\n    '''\n    Calculate the win possibility based on two Trueskill objects\n    :param team1:User TrueSkill Object\n    :param team2:Question Trueskill Object\n    :return: Winning Prob\n    '''\n    delta_mu = team1.mu - team2.mu\n    sum_sigma = sum([team1.sigma ** 2, team2.sigma ** 2])\n    size = 2\n    denom = math.sqrt(size * (0.05 * 0.05) + sum_sigma)\n    ts = trueskill.global_env()\n    \n    return ts.cdf(delta_mu / denom) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class user:\n    '''\n    User Class\n    '''\n#     __slots__ = ['question_answered_num', 'question_answered_num_agg_field', 'question_answered_mean_accuracy',\n#                 'question_answered_mean_accuracy_agg_field','question_answered_mean_difficulty_weighted_accuracy',\n#                 'question_answered_mean_difficulty_weighted_accuracy_agg_field','max_solved_difficulty',\n#                 'max_solved_difficulty_agg_field','min_wrong_difficulty','min_wrong_difficulty_agg_field',\n#                 'lessons_overall','lessons_overall_agg_field','session_time','since_last_session_time',\n#                 '_mmr_object','_mmr_object_agg_field','_most_liked_guess','_last_session_start_time','_first_action_time',\n#                 '_question_num_dict','_first_processed_flag']\n    def __init__(self):\n        '''\n        Init User Class\n        :param None\n        :return: None\n        '''\n        # Please Refer to Documentation above for meaning of features.\n        # Directly output features\n\n        # Counting \n        self.question_answered_num = 0  \n        self.question_answered_num_agg_field = [0] * 7 \n\n        # Correct Rate\n        self.question_answered_mean_accuracy = 0  \n        self.question_answered_mean_accuracy_agg_field = [0] * 7  \n        self.question_answered_mean_difficulty_weighted_accuracy = 0  \n        self.question_answered_mean_difficulty_weighted_accuracy_agg_field = [0] * 7  \n        # Min/Max stuff\n        self.max_solved_difficulty = 1\n        self.max_solved_difficulty_agg_field = [1] * 7\n        self.min_wrong_difficulty = 0 \n        self.min_wrong_difficulty_agg_field = [0] * 7  \n\n        # Lessons stuff\n        self.lessons_overall = 0 \n        self.lessons_overall_agg_field = [0] * 7  \n\n        # Session timing \n        self.session_time = 0  \n        self.since_last_session_time = 0  \n\n        # Features need some processing\n        self._mmr_object = trueskill.setup(mu=0.3, sigma=0.164486, beta=0.05, tau=0.00164,\n                                           draw_probability=0).Rating() \n        self._mmr_object_agg_field = [trueskill.setup(mu=0.3, sigma=0.164486, beta=0.05, tau=0.00164,\n                                                      draw_probability=0).Rating()] * 7\n        self._most_liked_guess = [0] * 4 \n        self._last_session_start_time = 0  \n        self._first_action_time = 0  \n        self._question_num_dict = {}  \n        self._first_processed_flag = False  \n\n    def update_user(self, data: pd.DataFrame):\n        '''\n        Update user with one row of DataFrame\n        :param data: pandas DataFrame\n        :return: None\n        '''\n        _temp = None\n\n        # Judging whether user are watching courses\n        if data['content_type_id'] == 0:\n            # Content Type = 0,means User are answering Questions.\n\n            # Counting Part\n            self.question_answered_num = self.question_answered_num + 1\n            question_field = int(data['content_field'])\n            self.question_answered_num_agg_field[question_field - 1] = int(self.question_answered_num_agg_field[\n                                                                               question_field - 1]) + 1\n\n            # Average Correct Rate\n            if data['answered_correctly'] == 1:\n                self.question_answered_mean_accuracy = \\\n                    (self.question_answered_mean_accuracy * (\n                            self.question_answered_num - 1) + 1) / self.question_answered_num\n\n                self.question_answered_mean_accuracy_agg_field[question_field - 1] = \\\n                    (self.question_answered_mean_accuracy_agg_field[question_field - 1] * (\n                            self.question_answered_num_agg_field[question_field - 1] - 1) + 1) \\\n                    / self.question_answered_num_agg_field[question_field - 1]\n\n                self.question_answered_mean_difficulty_weighted_accuracy = \\\n                    (self.question_answered_mean_difficulty_weighted_accuracy * (self.question_answered_num - 1) + (\n                            1 - data['mean_question_accuracy']) * 3) \\\n                    / self.question_answered_num\n\n                self.question_answered_mean_difficulty_weighted_accuracy_agg_field[question_field - 1] = \\\n                    (self.question_answered_mean_difficulty_weighted_accuracy_agg_field[question_field - 1] * (\n                            self.question_answered_num_agg_field[question_field - 1] - 1) + (\n                             1 - data['mean_question_accuracy']) * 3) \\\n                    / self.question_answered_num_agg_field[question_field - 1]\n\n\n            else:\n                self.question_answered_mean_accuracy = \\\n                    (self.question_answered_mean_accuracy * (\n                            self.question_answered_num - 1)) / self.question_answered_num\n\n                self.question_answered_mean_accuracy_agg_field[question_field - 1] = \\\n                    (self.question_answered_mean_accuracy_agg_field[question_field - 1] * (\n                            self.question_answered_num_agg_field[question_field - 1] - 1)) / \\\n                    self.question_answered_num_agg_field[question_field - 1]\n\n                self.question_answered_mean_difficulty_weighted_accuracy = \\\n                    (self.question_answered_mean_difficulty_weighted_accuracy * (self.question_answered_num - 1)) \\\n                    / self.question_answered_num\n\n                self.question_answered_mean_difficulty_weighted_accuracy_agg_field[question_field - 1] = \\\n                    (self.question_answered_mean_difficulty_weighted_accuracy_agg_field[question_field - 1] * (\n                            self.question_answered_num_agg_field[question_field - 1] - 1)) \\\n                    / self.question_answered_num_agg_field[question_field - 1]\n\n            # Min/Max Part\n\n            if data['answered_correctly'] == 1:\n                if data['mean_question_accuracy'] < self.max_solved_difficulty:\n                    self.max_solved_difficulty = data['mean_question_accuracy']\n                if data['mean_question_accuracy'] < self.max_solved_difficulty_agg_field[question_field - 1]:\n                    self.max_solved_difficulty_agg_field[question_field - 1] = data['mean_question_accuracy']\n            else:\n                if data['mean_question_accuracy'] > self.min_wrong_difficulty:\n                    self.min_wrong_difficulty = data['mean_question_accuracy']\n                if data['mean_question_accuracy'] > self.min_wrong_difficulty_agg_field[question_field - 1]:\n                    self.min_wrong_difficulty_agg_field[question_field - 1] = data['mean_question_accuracy']\n\n            # Guessing Part\n            if data['answered_correctly'] == 0:\n                self._most_liked_guess[int(data['user_answer'])] = self._most_liked_guess[\n                                                                       int(data['user_answer'])] + 1\n\n            # Session Timing part\n            if self._first_action_time == 0:\n                self._first_action_time = data['timestamp']\n                self._last_session_start_time = data['timestamp']\n            else:\n                if data['timestamp'] - self._last_session_start_time >= 7200 * 1000:\n                    self.since_last_session_time = (data[\n                                                        'timestamp'] - self._last_session_start_time) / 1000 / 3600\n                    self._last_session_start_time = data['timestamp']\n                    self.session_time = 0\n                else:\n                    self.session_time = (data['timestamp'] - self._last_session_start_time) / 1000 / 60\n\n            # Answer history part\n            if str(data['content_id']) in self._question_num_dict:\n                self._question_num_dict[str(data['content_id'])] = self._question_num_dict[str(data['content_id'])] + 1\n            else:\n                self._question_num_dict[str(data['content_id'])] = 1\n\n            # Trueskill part\n            if data['answered_correctly'] == 1:\n                self._mmr_object, _temp = \\\n                    trueskill.rate_1vs1(self._mmr_object,\n                                        trueskill.setup(mu=1 - data['mean_question_accuracy'], sigma=0.164486,\n                                                        beta=0.05, tau=0.00164, draw_probability=0).Rating())\n                self._mmr_object_agg_field[question_field - 1], _temp = \\\n                    trueskill.rate_1vs1(self._mmr_object_agg_field[question_field - 1],\n                                        trueskill.setup(mu=1 - data['mean_question_accuracy'], sigma=0.164486,\n                                                        beta=0.05,\n                                                        tau=0.00164, draw_probability=0).Rating())\n            else:\n                _temp, self._mmr_object = \\\n                    trueskill.rate_1vs1(trueskill.setup(mu=1 - data['mean_question_accuracy'], sigma=0.164486,\n                                                        beta=0.05, tau=0.00164, draw_probability=0).Rating(),\n                                        self._mmr_object)\n\n                _temp, self._mmr_object_agg_field[question_field - 1] = \\\n                    trueskill.rate_1vs1(trueskill.setup(mu=1 - data['mean_question_accuracy'], sigma=0.164486,\n                                                        beta=0.05,\n                                                        tau=0.00164, draw_probability=0).Rating(),\n                                        self._mmr_object_agg_field[question_field - 1])\n\n\n\n        else:\n            # Content Type !=0,User are watching a  lecture\n\n            self.lessons_overall = self.lessons_overall + 1\n            lesson_field = int(data['content_field'])\n            self.lessons_overall_agg_field[lesson_field - 1] = self.lessons_overall_agg_field[lesson_field - 1] + 1\n    def get_user_dict(self):\n        return {'question_answered_num':self.question_answered_num,\n               'question_answered_mean_accuracy':self.question_answered_mean_accuracy,\n               'max_solved_difficulty':self.max_solved_difficulty,\n                'min_wrong_difficulty':self.min_wrong_difficulty,\n                'lessons_overall':self.lessons_overall,\n                'session_time':self.session_time,\n                'time_to_last_session':self.since_last_session_time,\n                'mmr_overall':self._mmr_object.mu,\n                'mmr_confidence':self._mmr_object.sigma,\n                'question_answered_mean_difficulty_weighted_accuracy':self.question_answered_mean_difficulty_weighted_accuracy,\n                \n               }, {'question_answered_num_agg_field':self.question_answered_num_agg_field,\n                'question_answered_mean_accuracy_agg_field':self.question_answered_mean_accuracy_agg_field,\n                 'question_answered_mean_difficulty_weighted_accuracy_agg_field':self.question_answered_mean_difficulty_weighted_accuracy_agg_field,\n                 'max_solved_difficulty_agg_field':self.max_solved_difficulty_agg_field,\n                 'min_wrong_difficulty_agg_field':self.min_wrong_difficulty_agg_field,\n                 'lessons_overall_agg_field':self.lessons_overall_agg_field,\n                 'mmr_overall_agg_field':self._mmr_object_agg_field\n\n                }\n    def get_mmr_obj(self):\n        return self._mmr_object\n    def get_mmr_object_agg_field(self):\n        return self._mmr_object_agg_field\n    def get_question_num_dict(self):\n        return self._question_num_dict\n    def get_most_liked_guess(self):\n        return self._most_liked_guess\n    def process_output(self, data):\n        '''\n        \n         Output data according to user's existing attributes\n        :param data: One row of dataset\n        :return: output_dict dict data for training/predicting\n        '''\n        output_dict = {}\n\n        # Counting Part\n        output_dict['question_answered_num'] = self.question_answered_num\n        output_dict['question_answered_num_agg_field'] = self.question_answered_num_agg_field[\n            int(data['content_field']) - 1]\n\n        # Average Correct Rate\n        output_dict['question_answered_mean_accuracy'] = self.question_answered_mean_accuracy\n\n        output_dict['question_answered_mean_accuracy_agg_field'] = self.question_answered_mean_accuracy_agg_field[\n            int(data['content_field']) - 1]\n        output_dict[\n            'question_answered_mean_difficulty_weighted_accuracy'] = self.question_answered_mean_difficulty_weighted_accuracy\n        output_dict['question_answered_mean_difficulty_weighted_accuracy_agg_field'] = \\\n            self.question_answered_mean_difficulty_weighted_accuracy_agg_field[int(data['content_field']) - 1]\n\n        #  Min/Max Part\n\n        output_dict['max_solved_difficulty'] = self.max_solved_difficulty\n        output_dict['max_solved_difficulty_agg_field'] = self.max_solved_difficulty_agg_field[\n            int(data['content_field']) - 1]\n        output_dict['min_wrong_difficulty'] = self.min_wrong_difficulty\n        output_dict['min_wrong_difficulty_agg_field'] = self.min_wrong_difficulty_agg_field[\n            int(data['content_field']) - 1]\n\n        # Lesson Learning part\n        output_dict['lessons_overall'] = self.lessons_overall\n        output_dict['lessons_overall_agg_field'] = self.lessons_overall_agg_field[int(data['content_field']) - 1]\n        if output_dict['lessons_overall_agg_field'] > 0:\n            output_dict['field_learnt'] = 1\n        else:\n            output_dict['field_learnt'] = 0\n        # Session Timing part\n        output_dict['session_time'] = self.session_time\n        output_dict['time_to_last_session'] = self.since_last_session_time\n\n        output_dict['task_id'] = data['task_container_id']\n        output_dict['prior_time'] = data['prior_question_elapsed_time']\n        # Question Statics part\n        output_dict['mean_question_accuracy'] = data['mean_question_accuracy']\n        output_dict['std_question_accuracy'] = data['std_accuracy']\n        output_dict['question_id'] = data['content_id']\n        # TrueSkill part\n        output_dict['mmr_overall'] = self._mmr_object.mu\n        output_dict['mmr_overall_agg_field'] = self._mmr_object_agg_field[int(data['content_field']) - 1].mu\n        output_dict['mmr_confidence'] = self._mmr_object.sigma\n\n        output_dict['mmr_overall_agg_field'] = self._mmr_object_agg_field[int(data['content_field']) - 1].sigma\n        output_dict['mmr_win_prob'] = win_probability(self._mmr_object,\n                                                      trueskill.setup(mu=1 - data['mean_question_accuracy'],\n                                                                      sigma=0.164486,\n                                                                      beta=0.05, tau=0.00164,\n                                                                      draw_probability=0).Rating())\n        output_dict['mmr_win_prob_agg_field'] = win_probability(\n            self._mmr_object_agg_field[int(data['content_field']) - 1],\n            trueskill.setup(mu=1 - data['mean_question_accuracy'], sigma=0.164486, beta=0.05,\n                            tau=0.00164, draw_probability=0).Rating())\n        output_dict['user_id'] = data['user_id']\n        output_dict['tag_1'] = data['tag_1']\n        output_dict['tag_2'] = data['tag_2']\n\n        output_dict['tags_encoded'] = data['tags_encoded']\n        # Other features\n\n        if not pd.isna(['prior_question_had_explanation']):\n            output_dict['previous_explained'] = data['prior_question_had_explanation']\n        else:\n            output_dict['previous_explained'] = False\n\n        if str(data['content_id']) in self._question_num_dict:\n            output_dict['question_seen'] = 1\n        else:\n            output_dict['question_seen'] = 0\n\n        # Guessing part\n        max_choice = 0\n        max_choice_num = 0\n        i = 0\n        for item in self._most_liked_guess:\n            if item > max_choice_num:\n                max_choice_num = item\n                max_choice = i\n            i = i + 1\n\n        if output_dict['mmr_win_prob'] <= 0.4:\n            if max_choice == data['correct_answer']:\n                output_dict['most_liked_guess_correct'] = True\n            else:\n                output_dict['most_liked_guess_correct'] = False\n        else:\n            output_dict['most_liked_guess_correct'] = True\n\n        # Target\n        #output_dict['answered_correctly'] = data['answered_correctly']\n\n        return output_dict\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import Metadata\nquestion_metadata = pd.read_csv(question_metadata_dir)\nlesson_metadata = pd.read_csv(lesson_metadata_dir)\nprint(\"{} Metadata Imported\".format(str(datetime.datetime.now())))\n#Indexing Metadata\nquestion_metadata = question_metadata.set_index(keys=['content_id'])\nlesson_metadata = lesson_metadata.set_index(keys=['content_id'])\nprint(\"{} Metadata Indexed\".format(str(datetime.datetime.now())))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Import pickle Object\nwith open(pickle_dir, 'rb') as fo:\n    user_pickle = pickle.load(fo)\n\nprint(\"{} Pickle Object Imported\".format(str(datetime.datetime.now())))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#Rebuilding Trueskill \nfor user_id,user_info in user_pickle.items():\n    user_pickle[user_id]._mmr_object = trueskill.setup(mu=user_pickle[user_id]._mmr_object[0],\n                                                       sigma=user_pickle[user_id]._mmr_object[1],\n                                                       beta=0.05, tau=0.00164,\n                                                       draw_probability=0).Rating()\n    for i in range(0, 7):\n        # 1+1\n        user_pickle[user_id]._mmr_object_agg_field[i] =  trueskill.setup(mu=user_pickle[user_id]._mmr_object_agg_field[i][0],\n                                                       sigma=user_pickle[user_id]._mmr_object_agg_field[i][1],\n                                                       beta=0.05, tau=0.00164,\n                                                       draw_probability=0).Rating()\n\nprint(\"{} Pickle Trueskill Rebuilt\".format(str(datetime.datetime.now())))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Init Others\nrows_accum = 0 #Row Counter\nfirst_submission = True \nmodel_prd = [0]\ntrue_value = []\nlast_df = pd.DataFrame()\nprint(\"{} Init Done!\".format(str(datetime.datetime.now())))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_df = reduce_mem_usage(train_df) \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_data(df):    \n    result_df = pd.DataFrame()\n    df['answered_correctly'] = 0.6524\n    st = float(time.time())\n    # Merging and Concating\n#     try:\n    sub_1 = df[df['content_type_id'] == False]\n    sub_2 = df[df['content_type_id'] == True]\n    del df\n    sub_1 = sub_1.merge(question_metadata, on=\"content_id\", how=\"left\")\n    sub_2 = sub_2.merge(lesson_metadata, on=\"content_id\", how=\"left\")\n    df = pd.DataFrame()\n    df = pd.concat([sub_1,sub_2])\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = process_data(train_df.copy())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get the users information and Aggregation information"},{"metadata":{"trusted":true},"cell_type":"code","source":"users_dict = {}\nfor user in user_pickle:\n    users_dict[user] = user_pickle[user].get_user_dict()[0]\n\nusers_df = pd.DataFrame(users_dict)\nusers_df = users_df.T.reset_index()\ndf = df.merge(users_df,left_on='user_id',right_on='index')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"agg_dict = {}\nfor user in user_pickle:\n    agg_dict[user] = user_pickle[user].get_user_dict()[1]\n    \nagg_df = pd.DataFrame(agg_dict)\nagg_df = agg_df.T.reset_index()\ndf = df.merge(agg_df,left_on='user_id',right_on='index')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in agg_df.columns:\n    if col != 'index':\n        df_np = df[['content_field',col]].values \n        df[col] = [elem [ind-1]for ind,elem in zip(df_np[:,0],df_np[:,1])]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['field_learnt'] = (df['lessons_overall_agg_field'] >0).astype('int')\ndf['mmr_overall_agg_field']=[elem.sigma for elem in df['mmr_overall_agg_field'].values]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Features needs looping on users"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndf['mmr_win_prob'] = 0\ndf['mmr_win_prob_agg_field'] = 0\ndf['most_liked_guess_correct'] = 1\n\nk=0\n\nusers = df.user_id.unique()\nl = len(users)\nfor user in users:\n    k+=1\n    mmr_object = user_pickle[user].get_mmr_obj()\n    user_df = df[df['user_id']==user]\n    # mmr_win_prob\n    df.loc[user_df.index,'mmr_win_prob'] = win_probability(mmr_object,\n                                                      trueskill.setup(mu=1 - np.array(user_df['mean_question_accuracy']),\n                                                                      sigma=0.164486,\n                                                                      beta=0.05, tau=0.00164,draw_probability=0).Rating())\n    # most_liked_guess_correct\n    most_liked_guess = user_pickle[user].get_most_liked_guess()\n    i = np.argmax(most_liked_guess)\n    \n    user_df_filtered = user_df[user_df['mmr_win_prob'] <= 0.4]\n    user_df_filtered = user_df_filtered[user_df_filtered['correct_answer']==i]\n    df.loc[user_df_filtered.index,'most_liked_guess_correct'] = 0\n                                                                      \n    # mmr_win_prob_agg_field and question_seen\n    mmr_object_agg_field = user_pickle[user].get_mmr_object_agg_field()\n    user_df_content = user_df['content_field']\n    a = user_df_content.values\n    mmr_object_agg_field_arr = [mmr_object_agg_field[i-1] for i in a]\n\n    user_df = df[df['user_id']==user]\n    df.loc[user_df.index,'mmr_win_prob_agg_field'] =  [win_probability_orig(b,trueskill.setup(mu=1 - a,\n                                      sigma=0.164486,\n                                      beta=0.05, tau=0.00164,\n                                    draw_probability=0).Rating())for a,b in zip(user_df['mean_question_accuracy'],mmr_object_agg_field_arr)]\n    \n    question_num_dict = user_pickle[user].get_question_num_dict()\n    df.loc[user_df.index,'question_seen'] = (user_df['content_id'].isin(question_num_dict) ).astype('int')\n    \n    \n    if k%500==0:\n        print(l-k)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['answered_correctly']=train_df['answered_correctly']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_pickle('preprocessed_df.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}