{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.style as style\n# style.use('fivethirtyeight')\nimport seaborn as sns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load first 10*5 rows of train data\nfolder = '../input/riiid-test-answer-prediction/'\ntrain = pd.read_csv(folder + 'train.csv', low_memory=False, nrows=10**5,\n                       dtype={'row_id': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8',\n                              'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32',\n                             'prior_question_had_explanation': 'boolean'})\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop the lecture and first question bundle (Typically the first several questions a user sees were part of an onboarding diagnostic test where they did not get any feedback.)\ntrain = train.dropna()\n\n# covert prior_question_had_explanation data type False,True to 0,1\n# train['prior_question_had_explanation'] = train.prior_question_had_explanation.astype('int8')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature engineering","metadata":{}},{"cell_type":"markdown","source":"## Select features\n\n1. collect all possible features in a dataframe.\n2. normalize feature values.\n3. try and decide feature importance.\n\nFirst, make a dataframe to hold all the values of possible features.","metadata":{}},{"cell_type":"markdown","source":"### By user\n1. mean (answered_correctly)\n2. number of questions answered\n3. total time spent on APP","metadata":{}},{"cell_type":"code","source":"# user accuracy and # of question answered\nuser_correct = train.groupby('user_id')['answered_correctly'].agg(user_sum = 'sum', \n                user_mean = 'mean').reset_index()\n# total time spent on riiid APP of each user\nuser_time_total = train.groupby('user_id')['timestamp'].agg(user_time_total = 'max')\n\n# merge\nuser_df = user_correct.merge(user_time_total, on = 'user_id')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Other features from train data\n1. answered_correctly\n2. prior_question_elapsed_time\n3. prior_question_had_explanation","metadata":{}},{"cell_type":"code","source":"# only select the needed columns\ntrain = train[['user_id', 'content_id', 'answered_correctly','prior_question_elapsed_time', 'prior_question_had_explanation']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### By question\n1. mean accuracy of each question\n2. number of questions answered\n3. mean accuracy of question part(section of the TOEIC test)\n","metadata":{}},{"cell_type":"code","source":"# load question data\nquestion = pd.read_csv(folder + 'questions.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge question to train\ntrain = train.merge(question[['question_id', 'part']],\n            left_on ='content_id', right_on = 'question_id', how = 'left')\\\n            .drop('question_id', axis = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get question mean and part mean\nquestion_mean = train.groupby('content_id')['answered_correctly']\\\n                .agg( question_mean = 'mean', question_sum = 'sum').reset_index()\npart_mean = train.groupby('part')['answered_correctly'].agg( part_mean = 'mean').reset_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge question mean, part mean to train\ntrain_df = train.merge(question_mean,on ='content_id', how = 'left')\\\n            .drop('content_id', axis = 1)\ntrain_df = train_df.merge(part_mean, on = 'part', how ='left')\\\n            .drop('part', axis = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge user_df to train\ntrain_df = train_df.merge(user_df, on = 'user_id', how ='left')\\\n            .drop('user_id', axis = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape # same rows as original train data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normalize inputs(features) ？","metadata":{}},{"cell_type":"code","source":"# feature_names = list(train_df.columns)[1:]\n# for i in feature_names:\n#     print(i)\n#     train_df[i] = train_df[i] / train_df[i].std()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature distributions","metadata":{}},{"cell_type":"code","source":"# plot histogram to show distribution of features by answered_correctly categories\ndef plot_histogram_answered_correctly(x,y):\n    plt.hist(list(x[y==0]), alpha = 0.5, label='answered_correctly = 0')\n    plt.hist(list(x[y==1]), alpha = 0.5, label='answered_correctly = 1')\n    plt.title(\"Histogram of '{var_name}'\".format(var_name = x.name))\n    plt.xlabel('Value')\n    plt.ylabel('Frequency')\n    plt.legend()\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_answered_correctly(train_df['question_mean'], train_df['answered_correctly'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature outliers detection\n## ****Detect outliers\nFirst, let's define a function to find all the possible outliers.","metadata":{}},{"cell_type":"markdown","source":"### IQR (inter quartile range)\nEquation: InterQuartile Range = Q3 - Q1\n\nThe first quartile, denoted Q1, is the value in the data set that holds 25% of the values below it.\n\nThe third quartile, denoted Q3, is the value in the data set that holds 25% of the values above it.\n\nOutliers and Tukey Fences: Tukey is one of th emethods fro determing outliers in a sample. It is very popular method is based on the following:\n\noutliers = below (Q1 - 1.5(Q3-Q1)) or above (Q3 + 1.5(Q3-Q1))","metadata":{}},{"cell_type":"code","source":"# define a func to find outliers\ndef find_outliers(x):\n    q1 =  data.quantile(q=.25)\n    q3 =  data.quantile(q=.75)\n    iqr = q3-q1\n    floor = q1 - 1.5*iqr\n    ceiling = q3 + 1.5*iqr\n    outlier_indices = list(x.index[(x < floor)|(x > ceiling)])\n    outlier_values = list(x[outlier_indices])\n    \n    return outlier_indices, outlier_values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find outliers in each column\n# try different features\noutlier_index = []\n\nfor c in train_df.columns:\n    data = train_df[c]\n    outlier_indices, outlier_values = find_outliers(data) \n    outlier_index.extend(outlier_indices)\n    print('Ther are ', len(outlier_values), ' outliers in', c)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# interpret the outliers\nprint('Ther are a total of {} outliers in train_df, accounting for {}% of the total train_df data ({}).'\\\n      .format(len(outlier_index),round(100*(len(outlier_index)/len(train_df)),2), len(train_df)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Remove outliers\nNow, let's remove all the outliers and get a clean data frame ready for next step.","metadata":{}},{"cell_type":"code","source":"# remove outliers from feature dataframe\ncleaned_train_df = train_df.drop(train_df.index[outlier_index])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the shape \ncleaned_train_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Machine Learning\n## Feature selection and model building\n### Split data to train and test data","metadata":{}},{"cell_type":"code","source":"# split data into train and test sets\nfrom sklearn.model_selection import train_test_split\n\n# get X, y data\ndata = cleaned_train_df\nX = data.drop('answered_correctly', axis=1)\ny = data.answered_correctly\n\n# Split the dataset\nX_train, X_test, y_train, y_test = train_test_split(\n        X, y, test_size=.3, random_state = 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature selection","metadata":{}},{"cell_type":"code","source":"# with such a large of features(9 in total) an cause overfitting and also slow computing\n# use feature selection to select the most important features\nimport sklearn.feature_selection","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def select_features(X, X_train, y_train, k):\n    if k > len(X):\n        print('K must less than the lence of input X')\n    select = sklearn.feature_selection.SelectKBest(k=k)\n    selected_features = select.fit(X_train, y_train)\n    indices_selected = selected_features.get_support(indices = True)\n    colnames_selected = [X.columns[i] for i in indices_selected]\n    return colnames_selected","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colnames_selected = select_features(X, X_train, y_train, 3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colnames_selected","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use the selected feature to fit model\nX_train_selected = X_train[colnames_selected]\nX_test_selected = X_test[colnames_selected]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Build model using cleaned data (without outliers)","metadata":{}},{"cell_type":"code","source":"# Perform Logistic Regression\nfrom sklearn.linear_model import LogisticRegression\n\nlog_reg = LogisticRegression()\nlog_reg.fit(X_train_selected, y_train)\n\n# Make prediction using the model\ny_pred = log_reg.predict(X_test_selected) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate model\n#### Evaluate Model --- ROC Curve\nNow, we can plot the ROC (Receiver Operating Characteristic) Curve which displays the percentage of true positives predicted by the model as the prediction probability cutoff is lowered from 1 to 0.\n\nThe higher the AUC (area under the curve), the more accurately our model is able to predict outcomes:","metadata":{}},{"cell_type":"code","source":"# quickly visualize results\nfrom sklearn import metrics\ndef ROC_Curve(log_model, X_test, y_test):\n    #define metrics\n    y_pred_proba = log_model.predict_proba(X_test)[::,1]\n    fpr, tpr, _ = metrics.roc_curve(y_test,  y_pred_proba)\n    auc = metrics.roc_auc_score(y_test, y_pred_proba)\n\n    #create ROC curve\n    plt.plot(fpr,tpr,label=\"AUC=\"+str(auc))\n    plt.legend(loc=4)\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROC_Curve(log_reg, X_test_selected, y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Evaluate Model --- ROC & AUC Score","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import confusion_matrix, roc_auc_score\n\ndef find_model_perf(X_train, X_test, y_train, y_test):\n    # Perform Logistic Regression\n    log_reg = LogisticRegression()\n    log_reg.fit(X_train, y_train)\n    y_pred = [x[1] for x in log_reg.predict_proba(X_test)]\n    auc = roc_auc_score(y_test, y_pred)  ## ?? roc_auc_score vs accuraccy\n    return auc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"auc_processed = find_model_perf(X_train_selected, X_test_selected, y_train, y_test)\nprint(auc_processed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Evaluate Model --- Model Scores","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, recall_score, precision_score\n\ndef estimate_logreg_scores(y_test, y_pred):\n\n    print('\\nLogistic Regression Report')\n    print('\\nUsing 0.3 as test size:')\n    print('Accuracy = {:.5f}'.format(accuracy_score(y_test, y_pred)))\n    print('Precision = {:.5f}'.format(precision_score(y_test, y_pred)))\n    print('Recall = {:.5f}'.format(recall_score(y_test, y_pred)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"estimate_logreg_scores(y_test, y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Evaluate Model --- Confusion Matrix","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import plot_confusion_matrix\n\ndef confusion_matrix_plot(log_model,  X_test, y_test):\n    plot_confusion_matrix(log_model, X_test, y_test,\n                             cmap=plt.cm.Blues);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the Confusion Matrix\nconfusion_matrix(y_test, y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix_plot(log_reg, X_test_selected, y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Feature coefficients","metadata":{}},{"cell_type":"code","source":"# interpret feature coefficients \ncoefficients = np.hstack((log_reg.intercept_, log_reg.coef_[0]))\ninterpret_result = pd.DataFrame(data={'variable': ['intercept'] + list(X_train_selected.columns), \n                   'coefficient': coefficients}).sort_values('coefficient', ascending = False)\ninterpret_result","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note: How to interpret the coefficient? need scale or standardize the numerical values?","metadata":{}},{"cell_type":"markdown","source":"## Make logistic regression model with outliers","metadata":{}},{"cell_type":"code","source":"# get X, y data\ndata = train_df\nX_outlier = data.drop('answered_correctly', axis=1)\ny_outlier = data.answered_correctly\n\n# Split the dataset\nX_train_outlier, X_test_outlier, y_train_outlier, y_test_outlier = train_test_split(\n       X_outlier, y_outlier, test_size=.3, random_state = 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_outlier.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colnames_selected_outlier = select_features(X_outlier, X_train_outlier, y_train_outlier, 2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# colnames_selected_outlier","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colnames_selected_outlier = ['user_mean','question_mean','question_sum','part_mean',\n       'prior_question_had_explanation']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use the selected feature to fit model\nX_train_selected_outlier = X_train_outlier[colnames_selected_outlier]\nX_test_selected_outlier = X_test_outlier[colnames_selected_outlier]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Perform Logistic Regression\nfrom sklearn.linear_model import LogisticRegression\n\nlog_reg_outlier = LogisticRegression()\nlog_reg_outlier.fit(X_train_selected_outlier, y_train_outlier)\n\n# Make prediction using the model\ny_pred_outlier = log_reg_outlier.predict(X_test_selected_outlier) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nROC_Curve(log_reg_outlier, X_test_selected_outlier, y_test_outlier)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"find_model_perf(X_train_selected_outlier, X_test_selected_outlier, y_train_outlier, y_test_outlier)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"estimate_logreg_scores(y_test_outlier, y_pred_outlier)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the Confusion Matrix\nconfusion_matrix(y_test_outlier, y_pred_outlier)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Three features with outliers')\nconfusion_matrix_plot(log_reg_outlier, X_test_selected_outlier, y_test_outlier)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}