{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input/riiid-test-answer-prediction'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import riiideducation\n\nfrom pandas.plotting import scatter_matrix\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import roc_auc_score, confusion_matrix\nfrom sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, learning_curve\nimport lightgbm as lgb\nfrom lightgbm import LGBMClassifier\nimport eli5\n\n\n\n%matplotlib inline\n# for heatmap and other plots\ncolorMap1 = sns.color_palette(\"RdBu_r\")\n# for countplot and others plots\ncolorMap2 = 'Blues_r'\n\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = \"/kaggle/input/riiid-test-answer-prediction/train.csv\"\nquestions_path = \"../input/riiid-test-answer-prediction/questions.csv\"\nlectures_path = \"../input/riiid-test-answer-prediction/lectures.csv\"\ntest = \"../input/riiid-test-answer-prediction/example_test.csv\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **DATA EXPLORATION & EDA**","metadata":{}},{"cell_type":"markdown","source":"We used the 5% of the data","metadata":{}},{"cell_type":"code","source":"%%time\n\ntrain = pd.read_csv(train_path, low_memory=False,  nrows=10123033,\n                       dtype={'row_id': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8',\n                              'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32', \n                             'prior_question_had_explanation': 'boolean',\n                             }\n                      )\ntrain","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train shape: {train.shape}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.isnull().sum())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=train.dropna()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.isnull().sum())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.memory_usage()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(['row_id', 'timestamp'], axis=1, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe().style.background_gradient(cmap='Blues')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe().transpose()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['answered_correctly'].describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(train['answered_correctly']);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['answered_correctly'].hist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature = ['answered_correctly','user_id', 'content_id', 'task_container_id',\n       'user_answer', 'prior_question_elapsed_time','prior_question_had_explanation']\ndf = train[feature]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_with_target = df.corr()[\"answered_correctly\"].sort_values(ascending=False)\nplt.figure(figsize=(14,7))\ncorr_with_target.drop(\"answered_correctly\").plot.bar()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(25,20))\nsns.set(font_scale=1)\nplt.figure(figsize=(16,8))\nsns.heatmap(df.corr(),annot=True,cmap=colorMap1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correlation_matrix  = df.corr()\ncorrelation_matrix[\"answered_correctly\"].sort_values(ascending=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Features","metadata":{}},{"cell_type":"markdown","source":"I'll give just some part from our data bacause of the RAM limit on Kaggle kernel","metadata":{}},{"cell_type":"code","source":"n = int(train.shape[0] * 1)\ntrain_n = train.sample(n=n, random_state=42)\nn","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_characteristics = train.groupby('user_id').agg({'answered_correctly':\n                                                  ['mean', 'median', 'std', 'skew', 'count']})\nuser_characteristics.columns = [\n    'mean_user_acc',\n    'median_user_acc',\n    'std_user_acc',\n    'skew_user_acc',\n    'number_of_answered_q'\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_characteristics.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"task_container_characteristics = train.groupby('task_container_id').agg({'answered_correctly':\n                                                                      ['mean', 'median', 'std', 'skew', 'count']})\ntask_container_characteristics.columns = [\n    'mean_task_acc',\n    'median_task_acc',\n    'std_task_acc',\n    'skew_task_acc',\n    'number_of_asked_task_containers'\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"task_container_characteristics.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"content_characteristics = train.groupby('content_id').agg({'answered_correctly':\n                                                        ['mean', 'median', 'std', 'skew', 'count']})\ncontent_characteristics.columns = [\n    'mean_acc',\n    'median_acc',\n    'std_acc',\n    'skew_acc',\n    'number_of_asked_q'\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"content_characteristics.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_n.copy()\ndel train_n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.merge(user_characteristics, how='left', on='user_id')\ntrain = train.merge(task_container_characteristics, how='left', on='task_container_id')\ntrain = train.merge(content_characteristics, how='left', on='content_id')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The target: answered_correctly","metadata":{}},{"cell_type":"markdown","source":"Answered_correctly is our target, and we have to predict to probability for an answer to be correct. ","metadata":{}},{"cell_type":"code","source":"features = [\n    'prior_question_elapsed_time', \n    'prior_question_had_explanation',\n    'mean_user_acc',\n    'median_user_acc',\n    'std_user_acc',\n    'skew_user_acc',\n    'number_of_answered_q',\n    'mean_task_acc',\n    'median_task_acc',\n    'std_task_acc',\n    'skew_task_acc',\n    'number_of_asked_task_containers',\n    'mean_acc',\n    'median_acc',\n    'std_acc',\n    'skew_acc',\n    'number_of_asked_q'\n]\n\ntarget = 'answered_correctly'\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Drop features that we are not going to use in our model","metadata":{}},{"cell_type":"code","source":"col_to_drop = set(train.columns.values.tolist()).difference(features + [target])\nfor col in col_to_drop:\n    del train[col]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['prior_question_had_explanation'] = train['prior_question_had_explanation'].fillna(value=False).astype(bool)\ntrain = train.fillna(value=0.5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.replace([np.inf, -np.inf], np.nan)\ntrain = train.fillna(0.5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"train_df, test_df, y_train, y_test = train_test_split(train[features], train[target], random_state=777, test_size=0.3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. **LGBMClassifier**","metadata":{}},{"cell_type":"code","source":"# clf = LGBMClassifier(random_state=777)\n\n#params = {\n#   'n_estimators': [50, 150, 300],'max_depth': [3, 5, 10],'num_leaves': [5, 15, 30],'min_data_in_leaf': [5, 50, 100],\n#    'feature_fraction': [0.1, 0.5, 1.],'lambda': [0., 0.5, 1.],\n#}\n#cv = RandomizedSearchCV(clf, param_distributions=params, cv=5, n_iter=50, verbose=2)\n#cv.fit(train_df, y_train)\n\n#print(cv.best_params_)\n#print(cv.best_score_)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'num_leaves': 30, \n    'n_estimators': 300, \n    'min_data_in_leaf': 100, \n    'max_depth': 5, \n    'lambda': 0.0, \n    'feature_fraction': 1.0\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LGBMClassifier(**params)\nmodel.fit(train_df, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('LGB ROC-AUC score: ', roc_auc_score(y_test.values, model.predict_proba(test_df)[:, 1]))\naccuracy1=roc_auc_score(y_test.values, model.predict_proba(test_df)[:, 1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eli5.show_weights(model, top=20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb.plot_importance(model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2. **LogisticRegression**","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logreg = LogisticRegression()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logreg.fit(train_df, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predicting the test set results and calculating the accuracy","metadata":{}},{"cell_type":"code","source":"y_pred = logreg.predict(test_df)\nprint('Accuracy of logistic regression classifier on test set: {:.2f}'.format(logreg.score(test_df, y_test)))\naccuracy2= logreg.score(test_df, y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nconfusion_matrix = confusion_matrix(y_test, y_pred)\nprint(confusion_matrix)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The result is telling us that we have 208239+1844304 = 2052543 correct predictions and 804476+108660 = 913136 incorrect predictions.**","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\ncm = confusion_matrix(y_test, y_pred, labels=logreg.classes_)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm,display_labels=logreg.classes_)\ndisp.plot() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\nprint(classification_report(y_test, y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The support is the number of occurrences of each class in y_test.**\n\nROC Curve","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import roc_curve\nlogit_roc_auc = roc_auc_score(y_test, logreg.predict(test_df))\nfpr, tpr, thresholds = roc_curve(y_test, logreg.predict_proba(test_df)[:,1])\nplt.figure()\nplt.plot(fpr, tpr, label='Logistic Regression (area = %0.2f)' % logit_roc_auc)\nplt.plot([0, 1], [0, 1],'r--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver operating characteristic')\nplt.legend(loc=\"lower right\")\nplt.savefig('Log_ROC')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The receiver operating characteristic (ROC) curve is another common tool used with binary classifiers. The dotted line represents the ROC curve of a purely random classifier; a good classifier stays as far away from that line as possible (toward the top-left corner).","metadata":{}},{"cell_type":"markdown","source":"3. **RandomForestClassifier**","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = RandomForestClassifier(max_depth=2, random_state=0)\nclf.fit(train_df, y_train)\nclf_pred = clf.predict(test_df)\nprint('Accuracy of Random Forest Classifier on test set: {:.2f}'.format(clf.score(test_df, y_test)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy3= clf.score(test_df, y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\ncm = confusion_matrix(y_test, clf_pred, labels=clf.classes_)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm,display_labels=clf.classes_)\ndisp.plot() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix = confusion_matrix(y_test, clf_pred)\nprint(confusion_matrix)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2106652 correct et 859027 fauses","metadata":{}},{"cell_type":"code","source":"clf_roc_auc = roc_auc_score(y_test, clf.predict(test_df))\nfpr, tpr, thresholds = roc_curve(y_test, clf.predict_proba(test_df)[:,1])\nplt.figure()\nplt.plot(fpr, tpr, label='Random Forest Classifier(area = %0.2f)' % clf_roc_auc)\nplt.plot([0, 1], [0, 1],'r--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver operating characteristic')\nplt.legend(loc=\"lower right\")\nplt.savefig('clf_ROC')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"leaderboard  = pd.DataFrame({'Accuracy':[accuracy1,accuracy2, accuracy3]},index = ['LGBMClassifier', ' Logistic Regression', 'Random Forest'])\nfig_dims = (15, 8)\n\nfig, a = plt.subplots(figsize=fig_dims)\nax = sns.barplot(x=\"Accuracy\", y=leaderboard.index, data=leaderboard,ax=a)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SUBMISSION ","metadata":{}},{"cell_type":"code","source":"env = riiideducation.make_env()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    # merge\n    test_df = test_df.merge(user_characteristics, on = \"user_id\", how = \"left\")\n    test_df = test_df.merge(task_container_characteristics, on = \"task_container_id\", how = \"left\")\n    test_df = test_df.merge(content_characteristics, on = \"content_id\", how = \"left\")\n    \n    # type transformation\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value=False).astype(bool)\n    test_df.fillna(value = 0.5, inplace = True)\n    test_df = test_df.replace([np.inf, -np.inf], np.nan)\n    test_df = test_df.fillna(0.5)\n    \n    # preds\n    test_df['answered_correctly'] = model.predict_proba(test_df[features])[:, 1]\n    cols_to_submission = ['row_id', 'answered_correctly', 'group_num']\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}