{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import riiideducation\nfrom sklearn.metrics import roc_auc_score\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nfrom collections import Counter\n\nfrom sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.model_selection import RandomizedSearchCV, cross_val_score, StratifiedKFold, learning_curve\n\nimport xgboost as xgb\nenv = riiideducation.make_env()\n\n\n                 \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\ntrain= pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n               usecols=[1, 2, 3,7,8,9], dtype={'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','answered_correctly':'int8','prior_question_elapsed_time': 'float32','prior_question_had_explanation': 'boolean'})\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.sort_values(['timestamp'], ascending=True)\n\ntrain.drop(['timestamp'], axis=1,   inplace=True)\n\nresults_c = train.iloc[0:9000000,:][['content_id','answered_correctly']].groupby(['content_id']).agg(['mean'])\nresults_c.columns = [\"answered_correctly_content\"]\n\nresults_u = train.iloc[0:9000000,:][['user_id','answered_correctly']].groupby(['user_id']).agg(['mean', 'sum'])\nresults_u.columns = [\"answered_correctly_user\", 'sum']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.iloc[900000:990000,:]\nX = pd.merge(X, results_u, on=['user_id'], how=\"left\")\nX = pd.merge(X, results_c, on=['content_id'], how=\"left\")\nX=X[X.answered_correctly!= -1 ]\nX=X.sort_values(['user_id'])\nY = X[[\"answered_correctly\"]]\nX = X.drop([\"answered_correctly\"], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nlb_make = LabelEncoder()\nX['prior_question_had_explanation_enc'] = lb_make.fit_transform(X['prior_question_had_explanation'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = X[['answered_correctly_user', 'answered_correctly_content', 'sum','prior_question_elapsed_time','prior_question_had_explanation_enc']] \nX.fillna(0.5,  inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from  sklearn.model_selection import train_test_split\nXt, Xv, Yt, Yv = train_test_split(X, Y, test_size =0.2, shuffle=False)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Cross validate model with Kfold stratified cross val\nkfold = StratifiedKFold(n_splits=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Hyperparameter tuning"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n\nYt = np.ravel(Yt)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### META MODELLING  WITH ADABOOST, RF, EXTRATREES and GRADIENTBOOSTING\n\n# Adaboost\nDTC = DecisionTreeClassifier()\n\nadaDTC = AdaBoostClassifier(DTC, random_state=7)\n\nada_param_grid = {\"base_estimator__criterion\" : [\"gini\", \"entropy\"],\n              \"base_estimator__splitter\" :   [\"best\", \"random\"],\n              \"algorithm\" : [\"SAMME\",\"SAMME.R\"],\n              \"n_estimators\" :[1,2,5,10],\n              \"learning_rate\":  [0.0001, 0.001,0.01]}\n\ngsadaDTC = RandomizedSearchCV(adaDTC,param_distributions = ada_param_grid, cv=kfold,n_jobs = 4, scoring=\"accuracy\", verbose = 1)\n\ngsadaDTC.fit(Xt,Yt)\n\nada_best = gsadaDTC.best_estimator_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gsadaDTC.best_score_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#ExtraTrees \nExtC = ExtraTreesClassifier()\n\n\n## Search grid for optimal parameters\nex_param_grid = {\"max_depth\": [None],\n              \"max_features\": [1, 3, 5],\n              \"min_samples_split\": [2, 3,5],\n              \"min_samples_leaf\": [1, 3,5],\n              \"bootstrap\": [False],\n              \"n_estimators\" :[100,150,200],\n              \"criterion\": [\"gini\"]}\n\n\ngsExtC = RandomizedSearchCV(ExtC,param_distributions = ex_param_grid, cv=kfold, scoring=\"accuracy\", n_jobs= 4, verbose = 1)\n\ngsExtC.fit(Xt,Yt)\n\nExtC_best = gsExtC.best_estimator_\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Best score\ngsExtC.best_score_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# RFC Parameters tunning \nRFC = RandomForestClassifier()\n\n\n## Search grid for optimal parameters\nrf_param_grid = {\"max_depth\": [None],\n              \"max_features\": [1, 3, 5],\n              \"min_samples_split\": [2, 3, 5],\n              \"min_samples_leaf\": [1, 3, 5],\n              \"bootstrap\": [True],\n              \"n_estimators\" :[100, 200, 300],\n              \"criterion\": [\"gini\"]}\n\n\ngsRFC = RandomizedSearchCV(RFC,param_distributions = rf_param_grid, cv=kfold, scoring=\"accuracy\", n_jobs= 4, verbose = 1)\n\ngsRFC.fit(Xt,Yt)\n\nRFC_best = gsRFC.best_estimator_\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Best score\ngsRFC.best_score_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Gradient boosting tuning\n\nGBC = GradientBoostingClassifier()\ngb_param_grid = {'loss' : [\"deviance\"],\n              'n_estimators' : [100,200],\n              'learning_rate': [0.1, 0.05, 0.01],\n              'max_depth': [4, 8],\n              'min_samples_leaf': [100,150],\n              'max_features': [0.3, 0.1] \n              }\n\ngsGBC = RandomizedSearchCV(GBC,param_distributions = gb_param_grid, cv=kfold, scoring=\"accuracy\", n_jobs= 4, verbose = 1)\n\ngsGBC.fit(Xt,Yt)\n\nGBC_best = gsGBC.best_estimator_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Best score\ngsGBC.best_score_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"votingC = VotingClassifier(estimators=[('rfc', RFC_best), ('extc', ExtC_best), ('adac',ada_best),('gbc',GBC_best)], voting='soft', n_jobs=4)\n\nvotingC = votingC.fit(Xt, Yt)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = votingC.predict(Xv)\ny_true = np.array(Yv)\nroc_auc_score(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test =  pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_test.csv')\ntest[\"prior_question_had_explanation_enc\"] = lb_make.fit_transform(test[\"prior_question_had_explanation\"])\ntest = pd.merge(test, results_u, on=['user_id'],  how=\"left\")\ntest = pd.merge(test, results_c, on=['content_id'],  how=\"left\")\ntest[['answered_correctly_user', 'answered_correctly_content', 'sum','prior_question_elapsed_time','prior_question_had_explanation_enc']]\ntest.fillna(0.5, inplace=True)\n\ny_pred = votingC.predict(test[['answered_correctly_user', 'answered_correctly_content', 'sum','prior_question_elapsed_time','prior_question_had_explanation_enc']])\n\ntest['answered_correctly'] = y_pred\n\nresults_c = train[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean'])\nresults_c.columns = [\"answered_correctly_content\"]\n\nresults_u = train[['user_id','answered_correctly']].groupby(['user_id']).agg(['mean', 'sum'])\nresults_u.columns = [\"answered_correctly_user\", 'sum']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df = pd.merge(test_df, results_u, on=['user_id'],  how=\"left\")\n    test_df = pd.merge(test_df, results_c, on=['content_id'],  how=\"left\")\n    test_df['answered_correctly_user'].fillna(0.5, inplace=True)\n    test_df['answered_correctly_content'].fillna(0.5, inplace=True)\n    test_df['sum'].fillna(0, inplace=True)\n    test_df['prior_group_answers_correct'].fillna(0.5,inplace=True)\n    test_df['prior_group_responses'].fillna(0.5,inplace=True)\n    test_df['prior_question_elapsed_time'].fillna(0,inplace=True)   \n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df[\"prior_question_had_explanation_enc\"] = lb_make.fit_transform(test_df[\"prior_question_had_explanation\"])\n    test_df['answered_correctly'] =  votingC.predict(test_df[['answered_correctly_user', 'answered_correctly_content', 'sum','prior_question_elapsed_time','prior_question_had_explanation_enc']])\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}