{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-10-11T03:57:11.52569Z","iopub.status.busy":"2020-10-11T03:57:11.524866Z","iopub.status.idle":"2020-10-11T03:57:12.680486Z","shell.execute_reply":"2020-10-11T03:57:12.679584Z"},"papermill":{"duration":1.184369,"end_time":"2020-10-11T03:57:12.680634","exception":false,"start_time":"2020-10-11T03:57:11.496265","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n        \nfrom sklearn.metrics import roc_auc_score\n\nimport lightgbm\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import GroupShuffleSplit, KFold","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.020999,"end_time":"2020-10-11T03:57:12.72393","exception":false,"start_time":"2020-10-11T03:57:12.702931","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Description"},{"metadata":{"papermill":{"duration":0.020564,"end_time":"2020-10-11T03:57:12.765289","exception":false,"start_time":"2020-10-11T03:57:12.744725","status":"completed"},"tags":[]},"cell_type":"markdown","source":"In my previous notebook, I have done a brief analysis of train data and used the results to train a baseline model. You can find it here: https://www.kaggle.com/ilialar/simple-eda-and-baseline\n\nIn this notebook, I will try to build reliable 5-folds cross-validation."},{"metadata":{"papermill":{"duration":0.020356,"end_time":"2020-10-11T03:57:12.808875","exception":false,"start_time":"2020-10-11T03:57:12.788519","status":"completed"},"tags":[]},"cell_type":"markdown","source":"It is hard to do proper validation in this competition due to the following reasons:\n- The data has timely nature - so ideally, we can't use future data (at least about one user) in training and historical data in validation.\n- Classical time series-validation doesn't let us use all data for training. \n- User-stratified validation also doesn't work because test data can probalby have both old and new users.\n- The main features we have are `user_id` and `conten_id`, so probably target encoding features are among the most useful ones. That means we should use double-validation."},{"metadata":{"papermill":{"duration":0.020328,"end_time":"2020-10-11T03:57:12.851072","exception":false,"start_time":"2020-10-11T03:57:12.830744","status":"completed"},"tags":[]},"cell_type":"markdown","source":"So my idea is to generate random valid sets of ~20% of data that contains either a new user or the latest data for old users.\nThis notebook is a little bit messy, but it solves the task and can help to increase the score."},{"metadata":{"papermill":{"duration":0.020271,"end_time":"2020-10-11T03:57:12.933373","exception":false,"start_time":"2020-10-11T03:57:12.913102","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Double validation requires a lot of memory, so I will demonstrate how it works on the subset of the 1M rows.\nI have also trained several folds using the whole data and the same notebook on a machine with 128GB (probably 64 will be enough to rut it as is) of RAM and uploaded the results to this dataset: https://www.kaggle.com/ilialar/riiid-models"},{"metadata":{"papermill":{"duration":0.021014,"end_time":"2020-10-11T03:57:12.976635","exception":false,"start_time":"2020-10-11T03:57:12.955621","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Data loading"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-11T03:57:13.031425Z","iopub.status.busy":"2020-10-11T03:57:13.029176Z","iopub.status.idle":"2020-10-11T03:57:15.895018Z","shell.execute_reply":"2020-10-11T03:57:15.894171Z"},"papermill":{"duration":2.897555,"end_time":"2020-10-11T03:57:15.895157","exception":false,"start_time":"2020-10-11T03:57:12.997602","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"data_types_dict = {\n#     'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n#     'content_type_id': 'int8',\n#     'task_container_id': 'int16',\n#     'user_answer': 'int8',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float16',\n    'prior_question_had_explanation': 'boolean'\n}\n\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                       nrows = 10**6,\n                       usecols = data_types_dict.keys(),\n                       dtype=data_types_dict, \n#                        index_col = 0\n                      )","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-11T03:57:15.944206Z","iopub.status.busy":"2020-10-11T03:57:15.943263Z","iopub.status.idle":"2020-10-11T03:57:15.963191Z","shell.execute_reply":"2020-10-11T03:57:15.96233Z"},"papermill":{"duration":0.047313,"end_time":"2020-10-11T03:57:15.963333","exception":false,"start_time":"2020-10-11T03:57:15.91602","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nlectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.021682,"end_time":"2020-10-11T03:57:16.006834","exception":false,"start_time":"2020-10-11T03:57:15.985152","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Useful functions"},{"metadata":{"papermill":{"duration":0.020704,"end_time":"2020-10-11T03:57:16.049212","exception":false,"start_time":"2020-10-11T03:57:16.028508","status":"completed"},"tags":[]},"cell_type":"markdown","source":"FeatureGenerator is used for aggregated feature generation. It simply stores 4 data frames with different aggregated features. `enrich` method adds these features to the provided data frame. `combine` is used to merge two objects of this class into the one."},{"metadata":{"execution":{"iopub.execute_input":"2020-10-11T03:57:16.133901Z","iopub.status.busy":"2020-10-11T03:57:16.100524Z","iopub.status.idle":"2020-10-11T03:57:16.137368Z","shell.execute_reply":"2020-10-11T03:57:16.136547Z"},"papermill":{"duration":0.0674,"end_time":"2020-10-11T03:57:16.137505","exception":false,"start_time":"2020-10-11T03:57:16.070105","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class FeatureGenerator:\n    def get_questions_feaures(self, questions_df,train_questions_only_df):\n        grouped_by_content_df = train_questions_only_df.groupby('content_id')\n        content_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count'] })\n        content_answers_df.columns = ['mean_accuracy', 'question_asked']\n        questions_df = questions_df.merge(content_answers_df, left_on = 'question_id', right_on = 'content_id', how = 'left')\n        bundle_dict = questions_df['bundle_id'].value_counts().to_dict()\n        questions_df['right_answers'] = questions_df['mean_accuracy'] * questions_df['question_asked']\n        questions_df['bundle_size'] =questions_df['bundle_id'].apply(lambda x: bundle_dict[x])\n        questions_df.set_index('question_id', inplace = True)\n        return questions_df\n    \n    def get_users_features(self, train_questions_only_df):\n        grouped_by_user_df = train_questions_only_df.groupby('user_id')\n        user_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count','sum']}).copy()\n        user_answers_df.columns = ['mean_user_accuracy', 'questions_answered', 'questions_asked_user']\n        return user_answers_df\n    \n    def get_bundle_features(self, questions_df):\n        grouped_by_bundle_df = questions_df.groupby('bundle_id')\n        bundle_answers_df = grouped_by_bundle_df.agg({'right_answers': 'sum', 'question_asked': 'sum'}).copy()\n        bundle_answers_df.columns = ['bundle_right_answers', 'bundle_questions_asked']\n        bundle_answers_df['bundle_accuracy'] = bundle_answers_df['bundle_right_answers'] / bundle_answers_df['bundle_questions_asked']\n        return bundle_answers_df\n    \n    def get_part_features(self, questions_df):    \n        grouped_by_part_df = questions_df.groupby('part')\n        part_answers_df = grouped_by_part_df.agg({'right_answers': 'sum', 'question_asked': 'sum'}).copy()\n        part_answers_df.columns = ['part_right_answers', 'part_questions_asked']\n        part_answers_df['part_accuracy'] = part_answers_df['part_right_answers'] / part_answers_df['part_questions_asked']\n        return part_answers_df\n        \n    def from_df(self, df, questions_df):\n        # computes aggregated target features for a given dataset\n        self.questions_df = self.get_questions_feaures(questions_df.copy(), df)\n        self.user_answers_df = self.get_users_features(df)\n        self.bundle_answers_df = self.get_bundle_features(self.questions_df)\n        self.part_answers_df = self.get_part_features(self.questions_df)\n        return self\n        \n    def enrich(self,df):\n        # adds aggregated featurea to a given dataset\n        df = df.merge(self.user_answers_df, how = 'left', on = 'user_id')\n        df = df.merge(self.questions_df, how = 'left', left_on = 'content_id', right_on = 'question_id')\n        df = df.merge(self.bundle_answers_df, how = 'left', on = 'bundle_id')\n        df = df.merge(self.part_answers_df, how = 'left', on = 'part')\n        return df\n    \n    def combine(self, fg):\n        # combines two FeatureGenerators into one using all the data\n        for df1,df2, question_asked_c, right_answer_c, accuracy_c, index_c in [\n            (self.questions_df, fg.questions_df, 'question_asked', 'right_answers', 'mean_accuracy', 'question_id'),\n            (self.user_answers_df, fg.user_answers_df, 'questions_asked_user', 'questions_answered', 'mean_user_accuracy', 'user_id'),\n            (self.bundle_answers_df, fg.bundle_answers_df, 'bundle_questions_asked', 'bundle_right_answers', 'bundle_accuracy', 'bundle_id'),\n            (self.part_answers_df, fg.part_answers_df, 'part_questions_asked', 'part_right_answers', 'part_accuracy', 'part'),\n        ]:\n            df1 = df1.merge(df2[[question_asked_c, right_answer_c]], how = 'outer', on = index_c)\n            df1[question_asked_c] = df1[[f'{question_asked_c}_x', f'{question_asked_c}_y']].sum(1)\n            df1[right_answer_c] = df1[[f'{right_answer_c}_x', f'{right_answer_c}_y']].sum(1)\n            df1.drop([f'{question_asked_c}_x', f'{question_asked_c}_y',f'{right_answer_c}_x', f'{right_answer_c}_y'], 1)\n            df1[accuracy_c] = df1[right_answer_c] / df1[question_asked_c]\n        \n    def normalize(self, factor):\n        # normalizes additive features\n        for df, features in [\n            (self.questions_df, ['question_asked', 'right_answers']),\n            (self.user_answers_df, ['questions_asked_user', 'questions_answered']),\n            (self.bundle_answers_df, ['bundle_questions_asked', 'bundle_right_answers']),\n            (self.part_answers_df, ['part_questions_asked', 'part_right_answers']),            \n        ]:\n            for c in features:\n                df[c] /= factor\n    \n    def save(self, n, path):\n        self.questions_df.to_csv(f'{path}/questions_{n}.csv')\n        self.user_answers_df.to_csv(f'{path}/user_answers_{n}.csv')\n        self.bundle_answers_df.to_csv(f'{path}/bundle_answers_{n}.csv')\n        self.part_answers_df.to_csv(f'{path}/part_answers_{n}.csv')\n        \n    def load(self, n, path):\n        self.questions_df = pd.read_csv(f'{path}/questions_{n}.csv', index_col = 0)\n        self.user_answers_df = pd.read_csv(f'{path}/user_answers_{n}.csv', index_col = 0)\n        self.bundle_answers_df = pd.read_csv(f'{path}/bundle_answers_{n}.csv', index_col = 0)\n        self.part_answers_df = pd.read_csv(f'{path}/part_answers_{n}.csv', index_col = 0)\n        return self","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-11T03:57:16.187517Z","iopub.status.busy":"2020-10-11T03:57:16.186416Z","iopub.status.idle":"2020-10-11T03:57:16.190513Z","shell.execute_reply":"2020-10-11T03:57:16.189773Z"},"papermill":{"duration":0.031718,"end_time":"2020-10-11T03:57:16.19064","exception":false,"start_time":"2020-10-11T03:57:16.158922","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# features to use in the model\nfeatures = [\n    'timestamp', 'prior_question_elapsed_time', 'prior_question_had_explanation', # original data\n    'mean_user_accuracy', 'questions_answered', # user data\n    'mean_accuracy', 'question_asked','right_answers',# questions answers\n    'bundle_size', 'bundle_accuracy', # bundle features\n    'part_accuracy', 'part' # part features\n           ]\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-11T03:57:16.239616Z","iopub.status.busy":"2020-10-11T03:57:16.238773Z","iopub.status.idle":"2020-10-11T03:57:16.242516Z","shell.execute_reply":"2020-10-11T03:57:16.241773Z"},"papermill":{"duration":0.030645,"end_time":"2020-10-11T03:57:16.242646","exception":false,"start_time":"2020-10-11T03:57:16.212001","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# we will save trained models and fitted feature generators here\nos.mkdir('models')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.020978,"end_time":"2020-10-11T03:57:16.285167","exception":false,"start_time":"2020-10-11T03:57:16.264189","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Let's create the function that computes aggregated functions "},{"metadata":{"execution":{"iopub.execute_input":"2020-10-11T03:57:16.342708Z","iopub.status.busy":"2020-10-11T03:57:16.341883Z","iopub.status.idle":"2020-10-11T03:57:16.345376Z","shell.execute_reply":"2020-10-11T03:57:16.345993Z"},"papermill":{"duration":0.039628,"end_time":"2020-10-11T03:57:16.346153","exception":false,"start_time":"2020-10-11T03:57:16.306525","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def kfold_enreach(df, n = 5, random_state = 0):\n    \"\"\"Inner cycle of double validation\n    For each fold computes the aggregated target-encodign features based on (n-1)/n part of data\n    and applies it to the rest 1/n.\n    Returns FeatureGenerator effectively trained on whole dataset \n    and the dataset with leak-free target-encoded features\n    \"\"\"\n    data_list = []\n    \n    splitter = KFold(n, shuffle = True)\n    # simplified KFold validation for the target encoding\n    # can be improved by using the same technique as in 1st level splitting\n    for j, (train_idx, valid_idx) in enumerate(splitter.split(df)):\n        fg = FeatureGenerator().from_df(df.iloc[train_idx], questions_df)\n        valid_df = df.iloc[valid_idx]\n        valid_df = fg.enrich(valid_df)\n        valid_df = valid_df[[c for c in valid_df.columns if c not in df.columns and c in features]]\n        valid_df.index = valid_idx\n        data_list.append(valid_df)\n        \n        if j == 0:\n            final_fg = fg\n        else:\n            final_fg.combine(fg)\n        \n    # normalize additive columns that were used several times\n    final_fg.normalize(n-1)\n    \n    new_faetures_df =  pd.concat(data_list).sort_index()\n\n    return final_fg, pd.concat([df[[c for c in df.columns if c in features + [target]]], new_faetures_df[[c for c in new_faetures_df.columns if c in features]]], 1)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.021095,"end_time":"2020-10-11T03:57:16.389918","exception":false,"start_time":"2020-10-11T03:57:16.368823","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Training"},{"metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2020-10-11T03:57:16.46335Z","iopub.status.busy":"2020-10-11T03:57:16.447274Z","iopub.status.idle":"2020-10-11T03:57:46.029434Z","shell.execute_reply":"2020-10-11T03:57:46.028233Z"},"papermill":{"duration":29.618244,"end_time":"2020-10-11T03:57:46.029633","exception":false,"start_time":"2020-10-11T03:57:16.411389","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"trained_models = []\nscores = []\n\n# we don't know this parameters for the test set so there should be better combination\ntest_size_final = 0.20\nuser_percent_having_history = 0.9\naverage_history_precent = 0.5\n\ntest_size = test_size_final / (user_percent_having_history * (1 - average_history_precent))\nassert test_size < 1.0\n\ntrain_df = train_df[train_df[target] != -1]\ntrain_df.reset_index(inplace=True, drop = True)\nsplitter = GroupShuffleSplit(5, test_size = test_size, random_state = 0)\n\nfor j, (train_idx, test_idx) in enumerate(splitter.split(train_df,groups = train_df['user_id'])):\n    user_count_dict = train_df['user_id'].iloc[test_idx].value_counts().to_dict()\n    user_indices = train_df.groupby('user_id').indices\n    # adding some of the early information of test users to train set\n    new_train_id = []\n    \n    for i,user in enumerate(user_count_dict.keys()):\n        if i % 10000 == 0: print(i)\n        if np.random.rand() < user_percent_having_history:\n            samples_to_add = np.random.binomial(user_count_dict[user], average_history_precent)\n            if samples_to_add > 0:\n                new_train_id.append(user_indices[user][:samples_to_add])\n    train_idx = np.hstack(new_train_id + [train_idx])\n    test_idx = np.setdiff1d(test_idx,train_idx)\n    \n    train_fold_df = train_df.iloc[train_idx]\n    valid_fold_df = train_df.iloc[test_idx]\n    train_fold_df.reset_index(inplace = True, drop = True)\n    \n    # adding target-encodign features usign double validation\n    final_fg, train_fold_df = kfold_enreach(train_fold_df, n = 5, random_state = j + 1)\n    valid_fold_df = final_fg.enrich(valid_fold_df)\n    \n    # I didn't do any params optimisation yet, the current ones are similar to: https://www.kaggle.com/dwit392/expanding-on-simple-lgbm\n    params = {\n    'objective': 'binary',\n    'max_bin': 700,\n    'learning_rate': 0.1,\n    'num_leaves': 31,\n    'num_boost_round': 10000\n}\n    \n    lgbm = LGBMClassifier(\n        **params,\n    )\n    \n    fill_dict = {x:0.6 for x in ['mean_user_accuracy','mean_accuracy','bundle_accuracy', 'part_accuracy'] if x in train_fold_df.columns}\n    print(fill_dict)\n    \n    # filling NaNs\n    for df in [train_fold_df, valid_fold_df]:\n        if 'prior_question_had_explanation' in features:\n            df['prior_question_had_explanation'] = df['prior_question_had_explanation'].fillna(value = False).astype(bool)\n        df.fillna(fill_dict, inplace = True)\n        df.fillna(value = 0, inplace = True)\n\n    lgbm.fit(train_fold_df[features], train_fold_df[target],\n            eval_set = [\n                (valid_fold_df[features], valid_fold_df[target]),\n                (train_fold_df[features], train_fold_df[target]),\n            ],\n            categorical_feature = ['part'],\n            early_stopping_rounds = 5,\n            eval_metric='auc',\n            )\n    \n    # saving the trained model\n    lgbm.booster_.save_model(f'models/model_{j}.txt')\n    final_fg.save(j, 'models')\n    trained_models.append({'model': lgbm, 'feature_extractor': final_fg})\n    scores.append(lgbm.best_score_['valid_0']['auc'])\n    \n    \nprint(np.mean(scores))\nprint(scores)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.043881,"end_time":"2020-10-11T03:57:46.125112","exception":false,"start_time":"2020-10-11T03:57:46.081231","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Submission"},{"metadata":{"papermill":{"duration":0.04064,"end_time":"2020-10-11T03:57:46.214279","exception":false,"start_time":"2020-10-11T03:57:46.173639","status":"completed"},"tags":[]},"cell_type":"markdown","source":"We will load previously trained models and feature generators (from https://www.kaggle.com/ilialar/riiid-models) for submission."},{"metadata":{"execution":{"iopub.execute_input":"2020-10-11T03:57:46.310737Z","iopub.status.busy":"2020-10-11T03:57:46.308194Z","iopub.status.idle":"2020-10-11T03:57:46.657286Z","shell.execute_reply":"2020-10-11T03:57:46.658041Z"},"papermill":{"duration":0.401414,"end_time":"2020-10-11T03:57:46.658228","exception":false,"start_time":"2020-10-11T03:57:46.256814","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"load_pretrained_models = True\nmax_models_num = 10\n\npath = '/kaggle/input/riiid-models/'\nif load_pretrained_models:\n    trained_models = []\n    i = 0\n    while os.path.exists(f\"{path}/questions_{i}.csv\"):\n        fg = FeatureGenerator().load(i, f'{path}')\n        model = lightgbm.Booster(model_file=f'{path}/model_{i}.txt')\n        trained_models.append({'model': model, 'feature_extractor': fg})\n        i += 1\n        if i == max_models_num:\n            break\nelse:\n    for data in trained_models:\n        data['model'] = data['model'].booster_","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-11T03:57:46.811369Z","iopub.status.busy":"2020-10-11T03:57:46.810368Z","iopub.status.idle":"2020-10-11T03:57:46.823585Z","shell.execute_reply":"2020-10-11T03:57:46.822717Z"},"papermill":{"duration":0.081823,"end_time":"2020-10-11T03:57:46.823756","exception":false,"start_time":"2020-10-11T03:57:46.741933","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-11T03:57:46.916765Z","iopub.status.busy":"2020-10-11T03:57:46.915859Z","iopub.status.idle":"2020-10-11T03:57:46.919453Z","shell.execute_reply":"2020-10-11T03:57:46.9187Z"},"papermill":{"duration":0.049742,"end_time":"2020-10-11T03:57:46.919578","exception":false,"start_time":"2020-10-11T03:57:46.869836","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-11T03:57:47.021895Z","iopub.status.busy":"2020-10-11T03:57:47.021019Z","iopub.status.idle":"2020-10-11T03:57:47.376798Z","shell.execute_reply":"2020-10-11T03:57:47.377543Z"},"papermill":{"duration":0.416174,"end_time":"2020-10-11T03:57:47.377753","exception":false,"start_time":"2020-10-11T03:57:46.961579","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"for j, (test_df, sample_prediction_df) in enumerate(iter_test):\n    for i, pipeline in enumerate(trained_models):\n        # making predictions\n        local_test_df = pipeline['feature_extractor'].enrich(test_df.copy())\n        local_test_df['prior_question_had_explanation'] = local_test_df['prior_question_had_explanation'].fillna(value = False).astype(bool)\n        local_test_df.fillna(fill_dict, inplace = True)\n        local_test_df.fillna(value = 0, inplace = True)\n        \n        if i == 0:\n            predicition = pipeline['model'].predict(local_test_df[features])\n        else:\n            predicition += pipeline['model'].predict(local_test_df[features])\n        \n    predicition /= len(trained_models)\n\n    test_df['answered_correctly'] = predicition\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}