{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#basic\nimport numpy as np\nimport pandas as pd\n\n#Model imports\nfrom sklearn.model_selection import train_test_split\nfrom lightgbm import LGBMClassifier\n\n#optuna\nimport optuna\nfrom optuna.samplers import TPESampler\nfrom sklearn.metrics import roc_auc_score\n\n# You can only call make_env() once, so don't lose it!\nimport riiideducation\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"used_data_types_dict = {\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float16',\n    'prior_question_had_explanation': 'boolean'\n}\n\ntrain_df = pd.read_csv(\n    '/kaggle/input/riiid-test-answer-prediction/train.csv',\n    usecols = used_data_types_dict.keys(),\n    dtype=used_data_types_dict,\n    #nrows=10**7,\n)\n\nused_data_types_dict = {\n    'question_id': 'int16',\n    'bundle_id': 'int16',\n    'correct_answer': 'int8',\n    'part': 'int8',\n    'tags': 'str',\n}\n\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',\n                       usecols = used_data_types_dict.keys(), dtype=used_data_types_dict)\n\nlectures = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')\nex = pd.read_csv('../input/riiid-test-answer-prediction/example_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#90% of the train_df\nfeatures_df = train_df.iloc[:int(9 /10 * len(train_df))]\n#10% of the train_df\ntrain_df = train_df.iloc[int(9 /10 * len(train_df)):]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#removes rows that are lectures and adds tags and part to each interaction\ntrain_questions_only_df = features_df[features_df['answered_correctly']!=-1]\n\ntrain_questions_only_df = pd.merge(train_questions_only_df, questions[['part','tags']], \n                                   left_on='content_id', right_index=True, how = 'left')\n\n#getting the mean accuracy, question count of each user and other math stuff\ngrouped_by_user_df = train_questions_only_df.groupby('user_id')\n\nuser_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count']}).copy()\nuser_answers_df.columns = [\n    'user_mean_accuracy', \n    'user_questions_answered', \n]\n\nuser_answers_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#grouping by content_id\ngrouped_by_content_df = train_questions_only_df.groupby('content_id')\n\n#getting mean count and other stuff for each content_id\ncontent_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count', 'std', 'median', 'skew']}).copy()\ncontent_answers_df.columns = [\n    'q_mean_accuracy', \n    'q_question_asked', \n    'q_std_accuracy', \n    'q_median_accuracy', \n    'q_skew_accuracy'\n]\n\ncontent_answers_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#grouping by content_id\ngrouped_by_tags_df = train_questions_only_df.groupby('tags')\n\ntags_answers_df = grouped_by_tags_df.agg({'answered_correctly': ['mean', 'count', 'std', 'median', 'skew']}).copy()\n\ntags_answers_df.columns = [\n    'tags_mean_accuracy', \n    'tags_question_asked', \n    'tags_std_accuracy', \n    'tags_median_accuracy', \n    'tags_skew_accuracy'\n]\n\ntags_answers_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_part_df = train_questions_only_df.groupby('part')\n\npart_answers_df = grouped_by_part_df.agg({'answered_correctly': ['mean', 'count', 'std', 'skew']}).copy()\npart_answers_df.columns = [\n    'part_mean_accuracy', \n    'part_questions_answered', \n    'part_std_user_accuracy',  \n    'part_skew_user_accuracy',\n]\n\npart_answers_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Filling missing accuracy for each question_id"},{"metadata":{"trusted":true},"cell_type":"code","source":"#missing questions in training data\nmissing_q= questions.index.difference(content_answers_df.index)\n\n#filled the one missing tag with most freq in part\nquestions['tags'] = questions.tags.fillna('27')\n#creating dataframe with missing q's\ndf_copy = content_answers_df.iloc[0:0,:].copy()\n\n#creating rows with each missing_q\nfor i in missing_q:\n    df_copy = df_copy.append({'content_id': i}, ignore_index=True)\n\ndf_copy.content_id = df_copy.content_id.astype('int64')\ndf_copy = df_copy.set_index('content_id')\n\n#fill in the df_copy dataset with values\nfor i in missing_q:\n    i_tags = questions.loc[i].tags\n    df_copy.loc[i] = tags_answers_df.loc[i_tags]\n\n#making the datatypes between the two dataframes the same (if error make sure at least nrows=10**7)\ndf_copy.q_median_accuracy = df_copy.q_median_accuracy.round(decimals=0)\ndf_copy['q_question_asked'] = 1\ndf_copy.q_question_asked = df_copy.q_question_asked.astype('int64')\n\n#adding questions that havent been seen to the dataframe\ncontent_answers_df=content_answers_df.append(df_copy).sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del features_df\ndel grouped_by_user_df\ndel grouped_by_content_df\ndel grouped_by_tags_df\ndel grouped_by_part_df\ndel df_copy\ndel missing_q","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n    'user_mean_accuracy', \n    'user_questions_answered',\n    'q_mean_accuracy', \n    'q_question_asked', \n    'q_std_accuracy', \n    'q_median_accuracy', \n    'q_skew_accuracy',\n    'tags_mean_accuracy', \n    'tags_question_asked', \n    'tags_std_accuracy', \n    'tags_median_accuracy', \n    'tags_skew_accuracy',\n    'part_mean_accuracy', \n    'part_questions_answered', \n    'part_std_user_accuracy',  \n    'part_skew_user_accuracy',\n    'prior_question_elapsed_time', \n    'prior_question_had_explanation',\n    'part'\n]\n\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[train_df[target] != -1]\n\ntrain_df = pd.merge(train_df, questions[['part','tags']], \n                    left_on='content_id', right_index=True, how = 'left')\n\ntrain_df = train_df.merge(user_answers_df, how='left', on='user_id')\ntrain_df = train_df.merge(content_answers_df, how='left', on='content_id')\ntrain_df = train_df.merge(part_answers_df, how='left', left_on='part', right_index=True)\ntrain_df = train_df.merge(tags_answers_df, how='left', left_on='tags', right_index=True)\n\ntrain_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].fillna(value=False).astype(bool)\ntrain_df = train_df.fillna(value=0.5)\n\ntrain_df = train_df[features + [target]]\ntrain_df = train_df.replace([np.inf, -np.inf], np.nan)\ntrain_df = train_df.fillna(0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, test_df = train_test_split(train_df, random_state=314, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sampler = TPESampler(seed=314)\n\ndef create_model(trial):\n    num_leaves = trial.suggest_int(\"num_leaves\", 2, 31)\n    n_estimators = trial.suggest_int(\"n_estimators\", 50, 300)\n    max_depth = trial.suggest_int('max_depth', 3, 8)\n    min_child_samples = trial.suggest_int('min_child_samples', 100, 1200)\n    learning_rate = trial.suggest_uniform('learning_rate', 0.0001, 0.30)\n    min_data_in_leaf = trial.suggest_int('min_data_in_leaf', 5, 90)\n    bagging_fraction = trial.suggest_uniform('bagging_fraction', 0.0001, 1.0)\n    feature_fraction = trial.suggest_uniform('feature_fraction', 0.0001, 1.0)\n    \n    model = LGBMClassifier(\n        num_leaves=num_leaves,\n        n_estimators=n_estimators, \n        max_depth=max_depth, \n        min_child_samples=min_child_samples, \n        min_data_in_leaf=min_data_in_leaf,\n        learning_rate=learning_rate,\n        feature_fraction=feature_fraction,\n        random_state=314\n    )\n    return model\n\ndef objective(trial):\n    model = create_model(trial)\n    model.fit(train_df[features], train_df[target])\n    score = roc_auc_score(test_df[target].values, model.predict_proba(test_df[features])[:,1])\n    return score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#study = optuna.create_study(direction=\"maximize\", sampler=sampler)\n#study.optimize(objective, n_trials=15)\n#params = study.best_params\n#params['random_state'] = 314","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 24,\n          'n_estimators': 104,\n          'max_depth': 7,\n          'min_child_samples': 689,\n          'learning_rate': 0.2221239593291603,\n          'min_data_in_leaf': 28,\n          'bagging_fraction': 0.2273386395906522,\n          'feature_fraction': 0.7763591512041167}\n\nmodel = LGBMClassifier(**params)\nmodel.fit(train_df[features], train_df[target])\nprint('LGB score: ', roc_auc_score(test_df[target].values, model.predict_proba(test_df[features])[:,1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"feature importance"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(model.feature_importances_)\nprint(train_df.columns[:-1])\n\n#we can use train_df.columns[:-1] only because the target column is at the end of the dataframe\n\npd.DataFrame({'col_name': model.feature_importances_},\n                index=train_df.columns[:-1]).sort_values(by='col_name', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Final Preds"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\nprior_test_df = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df = pd.merge(test_df, questions[['part','tags']], \n                    left_on='content_id', right_index=True, how = 'left')\n    test_df = test_df.merge(user_answers_df, how='left', on='user_id')\n    test_df = test_df.merge(content_answers_df, how='left', on='content_id')\n    test_df = test_df.merge(part_answers_df, how='left', left_on='part', right_index=True)\n    test_df = test_df.merge(tags_answers_df, how='left', left_on='tags', right_index=True)\n    \n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value=False).astype(bool)\n    test_df.fillna(value = 0.6, inplace = True)\n\n    test_df['answered_correctly'] = model.predict_proba(test_df[features])[:,1]\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}