{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import riiideducation\nimport numpy as np \nimport pandas as pd \n\nfrom sklearn.metrics import roc_auc_score\nfrom  sklearn.tree import DecisionTreeClassifier\nfrom  sklearn.model_selection import train_test_split\n\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/riiid-test-answer-prediction/train.csv', \n                      usecols=[1,2,3,4,5,7,8,9],nrows=50000000)\nlectures = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')\nquestions = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')\nexample_test = pd.read_csv('../input/riiid-test-answer-prediction/example_test.csv')\nexample_sample_submission = pd.read_csv('../input/riiid-test-answer-prediction/example_sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## PREPROCESSING"},{"metadata":{},"cell_type":"markdown","source":"#### SELECTS ONLY QUESTIONS "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[train_df['content_type_id'] == 0]\n#keeping just the questions \n\ntrain_df= train_df.drop(columns=['content_type_id'])\n#dropping the column content_type_id and the answer of the users column","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_c = train_df[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean'])\nresults_c.columns = [\"answered_correctly_content\"]\n\nresults_u = train_df[['user_id','answered_correctly']].groupby(['user_id']).agg(['mean', 'sum', 'count'])\nresults_u.columns = ['answered_correctly_user', 'sum', 'count']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df =train_df.sort_values(['timestamp'], ascending=True)\ntrain_df = train_df.iloc[10000000:,:]\n# sort the dataset by timestamp and than take only the last N observation. In this way all the values with timestamp = 0 are removed, and the db is \n# more easy to treat    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.merge(train_df, results_u, on=['user_id'], how=\"left\")\ntrain_df = pd.merge(train_df, results_c, on=['content_id'], how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### MERGING TRAIN DATASET WITH QUESTION DATASET"},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = questions.rename(columns={'question_id':'content_id'})\ntrain_df = train_df.merge(questions)\ntrain_df= train_df.drop(columns=['correct_answer'])\ntrain_df= train_df.drop(columns=['bundle_id'])\n#merging together the 2 db and dropping the column correct_answer and bundle_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['task_container_id'] = (\n    train_df\n    .groupby('user_id')['task_container_id']\n    .transform(lambda x : pd.factorize(x)[0])\n    .astype('int16')\n)\n#this is a function that assure the monotonicity of task container id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### ENCODING VARIABLE \"PRIOR_QUESTION_HAD_EXPLANATION\""},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nlb_make = LabelEncoder()\ntrain_df[\"prior_question_had_explanation_enc\"] = lb_make.fit_transform(train_df[\"prior_question_had_explanation\"])\ntrain_df = train_df.drop(columns=['prior_question_had_explanation'])\n#this is just for encoding 0-1 the variable prior question had explanation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.sort_values(by=['user_id'])\n# sorting by user_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=train_df.drop(columns=['part'])\ntrain_df=train_df.drop(columns=['tags'])\n\n#I remove part and tag, I put these command here because in a second moment I would like to integrete these 2 variables, it could be useful.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime\ntrain_df['timestamp'] = pd.to_datetime(train_df['timestamp'], unit='ms',origin='2017-1-1')\ntrain_df['month']=(train_df.timestamp.dt.month)\ntrain_df['day']=(train_df.timestamp.dt.day)\n#i trasform timestamp in date format, then I extrapolte month and day to generate 2 columns\n\naveg = train_df[['user_id','month','day','prior_question_elapsed_time']].groupby(['user_id','month','day']).mean()/1000\naveg.columns=['mean']\n#with the 2 columns generated before it is now possible \n#to calculate the average elapsed time for each user for each month for each day. \n\ntrain_df = pd.merge(train_df, aveg, on=['user_id','month','day'], how='left')\n# merge the 2 db","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train_df[[\"answered_correctly\"]]\n# extrapolate the dependent variable ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum(axis = 0)\n#checking for any missing value","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"keep = ['prior_question_had_explanation_enc',\n        'mean', \n        'answered_correctly_user',\n        'sum', \n        'count',\n        'answered_correctly_content']\nx=train_df[keep]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xt, Xv, Yt, Yv = train_test_split(x, y, test_size =0.2, shuffle=False)\n# split train in train and validation \n\nimport lightgbm as lgb\n\n'''\nhttps://lightgbm.readthedocs.io/en/latest/Parameters.html\n'''\n\nparams = {\n    'objective': 'binary', #specify how is the dependet variable, binary can be used for logistic regression or log loss classification\n    'max_bin': 600, #max number of bins that features values will be bucketed in. Small number may reduce training accuracy but may increase general power\n    'learning_rate': 0.02, #learning_rate refers to the step size at each interation while moving toward an optimal point\n    'num_leaves': 80 # maximum number of leaves in a tree, where a leave is a final termination of a tree\n}\n\n\nlgb_train = lgb.Dataset(Xt, Yt)\nlgb_eval = lgb.Dataset(Xv, Yv, reference=lgb_train)\n#lightgbm need to take as argument lightgbm dataset, it is required to make this trasformation\n\nmodel = lgb.train(\n    params, lgb_train, #it is required to insert the parameters, then the train set\n    valid_sets=[lgb_train, lgb_eval],\n    verbose_eval=10,\n    num_boost_round=1000, # number of boosting iterations \n    early_stopping_rounds=10 # will stop training if one metric of one validation data doesn’t improve in last early_stopping_round rounds, so if \n    #  for ten 'epochs' the model will stop, in this way the num_boost_round is a maximum value.  \n)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict(Xv)\ny_true = np.array(Yv)\nroc_auc_score(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### CHECK ON EXAMPLE TEST"},{"metadata":{},"cell_type":"markdown","source":"Before submitting of the real test sets, let's try out if evrything it's working on the example test. Let's repeat all the operation computed before."},{"metadata":{"trusted":true},"cell_type":"code","source":"example_test = pd.read_csv('../input/riiid-test-answer-prediction/example_test.csv')\nexample_test[\"prior_question_had_explanation_enc\"] = lb_make.fit_transform(example_test[\"prior_question_had_explanation\"])\nexample_test.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example_test['timestamp'] = pd.to_datetime(example_test['timestamp'], unit='ms',origin='2017-1-1')\nexample_test['month']=(example_test.timestamp.dt.month)\nexample_test['day']=(example_test.timestamp.dt.day)\naveg = example_test[['user_id','month','day','prior_question_elapsed_time']].groupby(['user_id','month','day']).mean()/1000\naveg.columns=['mean']\n\nexample_test = pd.merge(example_test, results_u, on=['user_id'], how=\"left\")\nexample_test = pd.merge(example_test, results_c, on=['content_id'], how=\"left\")\nexample_test = pd.merge(example_test, aveg, on=['user_id','month','day'], how='left')\nexample_test = example_test[keep]\nexample_test.head(10)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nimp = IterativeImputer(max_iter=10, random_state=0)\nimp.fit(example_test)\nexample_test = pd.DataFrame(imp.transform(example_test), columns = example_test.columns)\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example_test['answered_correctly_user'].fillna(example_test['answered_correctly_user'].mean(), inplace=True)\nexample_test['sum'].fillna(example_test['sum'].mean(), inplace=True)\nexample_test['count'].fillna(example_test['count'].mean(), inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example_test.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict(example_test[keep])\nexample_test['answered_correctly'] = y_pred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### SUBMIT MODEL PREDICTIONS "},{"metadata":{},"cell_type":"markdown","source":"It's now possible to submit the model prediction using the method explained by the host of the competition."},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\n    imp = IterativeImputer(max_iter=10, random_state=0)\n    imp.fit(test_df)\n    test_df = pd.DataFrame(imp.transform(test_df), columns = test_df.columns)\n'''\n\niter_test = env.iter_test()\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df[\"prior_question_had_explanation_enc\"] = lb_make.fit_transform(test_df[\"prior_question_had_explanation\"])\n    \n    test_df['timestamp'] = pd.to_datetime(test_df['timestamp'], unit='ms',origin='2017-1-1')\n    test_df['month']=(test_df.timestamp.dt.month)\n    test_df['day']=(test_df.timestamp.dt.day)\n    avegm = test_df[['user_id','month','day','prior_question_elapsed_time']].groupby(['user_id','month','day']).mean()/1000\n    avegm.columns=['mean']\n    \n    test_df = pd.merge(test_df, results_u, on=['user_id'],  how=\"left\")\n    test_df = pd.merge(test_df, results_c, on=['content_id'],  how=\"left\")\n    test_df = pd.merge(test_df, avegm, on=['user_id','month','day'], how='left')\n    \n    test_df['answered_correctly_user'].fillna(test_df['answered_correctly_user'].mean(), inplace=True)\n    test_df['sum'].fillna(test_df['sum'].mean(), inplace=True)\n    test_df['count'].fillna(test_df['count'].mean(), inplace=True)\n    \n    test_df['answered_correctly'] =  model.predict(test_df[keep])\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('finish')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}