{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Version history\n- V10: data sorted by timestamp, accuracy metrics, stratifiedkfold (score: 0.744)\n- V11: tried TimeSeriesSplit with 500 iterations and predict_proba (score: error)\n- V12: removed sort by timestamp, reducing iterations to 350, keeping TimeSeriesSplit (running for more than 5 hours)\n- V13: increased data batch size, trying BlockingTimeSeriesSplit and AUC metrics (score: 0.672)\n- V14: reverting back to timestamp and Accuracy metrics, BlockingTimeSeriesSplit retained (score: 0.742)\n- V15: adding features from the questions table. Forgot to update the prediction part. D***!!!\n- V16: trying [this](https://www.kaggle.com/shoheiazuma/riiid-lgbm-starter) out.\n- V17: persisting with [this](https://www.kaggle.com/shoheiazuma/riiid-lgbm-starter) and TimeSeriesSplit (score: 0.747)\n- V18: removed batch processing, turned on GPU (error in predict part)\n- V19: corrected key error in predict cycle (score: 0.717)\n- V20: trying LGB on the same data to compare results (no submission file)\n- V21: LGB retry (score: 0.729)\n- V22: retrying with catboost (score: 0.758)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Suppressing warnings because of skopt verbosity\nimport sys, warnings\nwarnings.filterwarnings(\"ignore\")\nfrom catboost import CatBoostClassifier, Pool\n\nfrom sklearn.model_selection import StratifiedKFold, TimeSeriesSplit\nfrom sklearn.metrics import roc_auc_score, average_precision_score\nfrom sklearn.metrics import make_scorer\nfrom sklearn.model_selection import cross_val_score, cross_val_predict\nfrom sklearn.metrics import accuracy_score,  precision_score, recall_score,f1_score\nfrom sklearn.preprocessing import LabelEncoder\n\n_ = np.seterr(divide='ignore', invalid='ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = \"../input/riiid-test-answer-prediction/train.csv\"\nquestions_path = \"../input/riiid-test-answer-prediction/questions.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndtype = {\n    \"row_id\": \"int64\",\n    \"timestamp\": \"int64\",\n    \"user_id\": \"int32\",\n    \"content_id\": \"int16\",\n    \"content_type_id\": \"boolean\",\n    \"task_container_id\": \"int16\",\n    \"user_answer\": \"int8\",\n    \"answered_correctly\": \"int8\",\n    \"prior_question_elapsed_time\": \"float32\", \n    \"prior_question_had_explanation\": 'boolean'\n}\n\ntrain = pd.read_parquet(\"../input/riiid-parquet-files/train.parquet\")\ntrain = train[dtype.keys()]\ntrain = train.astype(dtype)\ntrain = train[train['answered_correctly']!=-1]\ntrain['prior_question_had_explanation'].fillna(False, inplace=True)\ntrain['prior_question_had_explanation'] = train['prior_question_had_explanation'].astype(bool)\ntrain = train[['user_id','content_id','answered_correctly',\n               'prior_question_elapsed_time', 'prior_question_had_explanation']]\ntrain.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# with open(data_path) as f:\n#     first_line = f.readline()\n# first_line","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cols = len(first_line.split(','))\n# cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# with open(data_path) as fp:\n#     for (rows, _) in enumerate(fp, 1):\n#        pass\n# rows","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# OTHER CONSTANTS\nTARGET = \"answered_correctly\"\nTIME_MEAN = 21000.0\nTIME_MIN = 0.0\nTIME_MAX = 300000.0\n#map_prior = {True:1, False:0}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dtype={'question_id':'int16','part':'int8','bundle_id':'int8', 'tags':'str'}\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',\n                        usecols=dtype.keys(),\n                        dtype=dtype)\nquestions['tags'].fillna('None', inplace=True)\nquestions['num_tags'] = questions['tags'].apply(lambda x:len(x.split()) if pd.notna(x) else 0) \n#questions = questions.rename(columns={'part':'qpart'})\n\nquestions['tags'].fillna('None', inplace=True)\nle = LabelEncoder()\nquestions['tags_label'] = le.fit_transform(questions['tags'].values)\n\nquestions = questions[['question_id','part','tags_label']]\nquestions.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef preprocess(df):\n    df = df[df[TARGET] != -1].reset_index(drop=True)\n    df = df.sort_values(['timestamp'], ascending=True).reset_index(drop = True)\n    df = df.merge(questions, left_on='content_id', right_on='question_id', how='left')\n    df.drop(columns=['question_id'], inplace=True)\n    df[\"prior_question_had_explanation\"].fillna(False, inplace=True)\n    df[\"prior_question_elapsed_time\"] = df[\"prior_question_elapsed_time\"].fillna(TIME_MEAN)\n    #df[\"duration\"] = (df[\"prior_question_elapsed_time\"] - TIME_MIN) / (TIME_MAX - TIME_MIN)\n    \n    df['lag'] = df.groupby('user_id')[TARGET].shift()\n    cum = df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])\n    df['user_correctness'] = cum['cumsum'] / cum['cumcount']\n    df.drop(columns=['lag'], inplace=True)\n    \n    user_agg = df.groupby('user_id')[TARGET].agg(['sum', 'count'])\n    content_agg = df.groupby('content_id')[TARGET].agg(['sum', 'count'])\n    df['content_count'] = df['content_id'].map(content_agg['count']).astype('int32')\n    df['content_mean'] = df['content_id'].map(content_agg['sum'] / content_agg['count'])\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain['lag'] = train.groupby('user_id')[TARGET].shift()\ncum = train.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])\ntrain['user_mean'] = cum['cumsum'] / cum['cumcount']\ntrain.drop(columns=['lag'], inplace=True)\n\nuser_agg = train.groupby('user_id')[TARGET].agg(['sum', 'count'])\ncontent_agg = train.groupby('content_id')[TARGET].agg(['sum', 'count'])\n\n#----------------------------\ntrain_df = train.groupby('user_id').tail(24).reset_index(drop=True)\ntrain_df = pd.merge(train_df, questions, left_on='content_id', right_on='question_id', how='left')\ntrain_df.drop(columns=['question_id'], inplace=True)\n#----------------------------\n\ntrain_df['content_count'] = train_df['content_id'].map(content_agg['count']).astype('int32')\n\ntrain_df['content_mean'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])\n\nvalid_df = train_df.groupby('user_id').tail(6)\ntrain_df.drop(valid_df.index, inplace=True)\ndel train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FE = ['content_mean','content_count','user_mean',\n      'prior_question_elapsed_time','prior_question_had_explanation',\n      'part','tags_label']\nCF = ['prior_question_had_explanation','part']  ##'bundle_id', 'num_tags']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, y_train = train_df[FE], train_df[TARGET]\nX_test, y_test = valid_df[FE], valid_df[TARGET]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initializing a CatBoostClassifier with best parameters\nbest_params = {#'bagging_temperature': 0.6,\n               #'border_count': 128,\n               #'depth': 8,\n               'iterations': 20000,\n               #'l2_leaf_reg': 30,\n               #'learning_rate': 0.5,\n               #'random_strength': 0.01,\n               #'scale_pos_weight': 0.48\n            }\n\ncatb = CatBoostClassifier(**best_params,\n                          loss_function='CrossEntropy', #loss_function='Logloss',\n                          eval_metric = 'AUC',\n                          #nan_mode='Min',\n                          thread_count=2,\n                          use_best_model=True,\n                          task_type = \"GPU\",\n                          verbose = False)\n\nroc_auc = list()\naverage_precision = list()\nbest_iteration = list()\n    \nX_train, y_train = train_df[FE], train_df[TARGET]\nX_test, y_test = valid_df[FE], valid_df[TARGET]\n    \ntrain = Pool(data=X_train, \n             label=y_train,            \n             feature_names=FE,\n             cat_features=[])\n\ntest = Pool(data=X_test, \n            label=y_test,\n            feature_names=FE,\n            cat_features=[])\n\ncatb.fit(train,\n         verbose_eval=50, \n         early_stopping_rounds=100,\n         eval_set=test,\n         use_best_model=True,\n         plot=False)\n\nbest_iteration.append(catb.best_iteration_)\npreds = catb.predict_proba(X_test)\nroc_auc.append(roc_auc_score(y_true=y_test, y_score=preds[:,1]))\naverage_precision.append(average_precision_score(y_true=y_test, y_score=preds[:,1]))\nprint(\"Average cv roc auc score %0.3f ± %0.3f\" % (np.mean(roc_auc), np.std(roc_auc)))\nprint(\"Average cv roc average precision %0.3f ± %0.3f\" % (np.mean(average_precision), np.std(average_precision)))\n\ncatb.save_model('catb_model.cbm')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#catb = catb.load_model('../input/riiid-catboost-attempt/catb_model.cbm')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\n\nuser_sum_dict = user_agg['sum'].astype('int16').to_dict(defaultdict(int))\nuser_count_dict = user_agg['count'].astype('int16').to_dict(defaultdict(int))\ncontent_sum_dict = content_agg['sum'].astype('int32').to_dict(defaultdict(int))\ncontent_count_dict = content_agg['count'].astype('int32').to_dict(defaultdict(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nprior_test_df = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfor test_df, sample_prediction_df in iter_test:\n    #------------------------------------------\n    if prior_test_df is not None:\n        prior_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prior_test_df = prior_test_df[prior_test_df[TARGET] != -1].reset_index(drop=True)\n        \n        user_ids = prior_test_df['user_id'].values\n        content_ids = prior_test_df['content_id'].values\n        targets = prior_test_df[TARGET].values\n        \n        for user_id, content_id, target in zip(user_ids, content_ids, targets):\n            user_sum_dict[user_id] += target\n            user_count_dict[user_id] += 1\n            content_sum_dict[content_id] += target\n            content_count_dict[content_id] += 1\n    prior_test_df = test_df.copy()\n    #------------------------------------------\n    #test_df = preprocess(test_df)\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df = test_df.merge(questions, left_on='content_id', right_on='question_id', how='left')\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation']\\\n                                                               .fillna(False).astype(bool)\n    user_sum = np.zeros(len(test_df), dtype=np.int16)\n    user_count = np.zeros(len(test_df), dtype=np.int16)\n    content_sum = np.zeros(len(test_df), dtype=np.int32)\n    content_count = np.zeros(len(test_df), dtype=np.int32)\n    \n    for i, (user_id, content_id) in enumerate(zip(test_df['user_id'].values, \\\n                                                  test_df['content_id'].values)):\n        user_sum[i] = user_sum_dict[user_id]\n        user_count[i] = user_count_dict[user_id]\n        content_sum[i] = content_sum_dict[content_id]\n        content_count[i] = content_count_dict[content_id]\n        \n    test_df['user_mean'] = user_sum / user_count\n    test_df['content_count'] = content_count\n    test_df['content_mean'] = content_sum / content_count\n    \n    #------------------------------------------\n    Xtest = test_df[FE].values\n    Xtest = Pool(data=Xtest,\n                 feature_names=FE,\n                 cat_features=[])\n    test_df['answered_correctly'] = catb.predict_proba(Xtest)[:,1]\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n    #------------------------------------------\n    #test_df[TARGET] = model.predict(test_df[FE])\n    #env.predict(test_df[['row_id', TARGET]])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}