{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reading the dataset from raw csv file\nimport datatable as dt\n\ntrain = dt.fread(\"../input/riiid-test-answer-prediction/train.csv\").to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# saving the dataset in .jay (binary format)\n# dt.fread(\"../input/riiid-test-answer-prediction/train.csv\").to_jay(\"train.jay\")\n# train = dt.fread(\"train.jay\").to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_types_dict = {\n    'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    'content_type_id': 'int8',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float32',\n    'prior_question_had_explanation': 'boolean'\n}\ntrain = train.astype(data_types_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print('Part of missing values for every column')\n# print(len(train))\n# print(train.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_part_len = int(9 /10 * len(train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convertBoolean(x):\n    if str(x) == \"False\":\n        return 0\n    elif str(x) == \"True\":\n        return 1\n    else:\n        return 0\n\n# TOEIC Section\ndef TOEICSection(part):\n    if part >= 1 and part <= 4:\n        return \"Listening\"\n    elif part >= 5 and part <= 7:\n        return \"Reading\"\n    else:\n        return \"Missing\"\n    \ndef tagsCount(tags):\n    arr = str(tags).split(\" \")\n    return len(arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag = questions[\"tags\"].str.split(\" \", expand = True)\ntag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions =  pd.concat([questions,tag],axis=1)\nquestions['tags1'] = pd.to_numeric(questions['tags1'], errors='coerce')\nquestions['tags2'] = pd.to_numeric(questions['tags2'], errors='coerce')\nquestions['tags3'] = pd.to_numeric(questions['tags3'], errors='coerce')\nquestions['tags4'] = pd.to_numeric(questions['tags4'], errors='coerce')\nquestions['tags5'] = pd.to_numeric(questions['tags5'], errors='coerce')\nquestions['tags6'] = pd.to_numeric(questions['tags6'], errors='coerce')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')\nlectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])\npart_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]\ntypes_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_lectures = train[train.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_lectures.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lecture_stats_part.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add boolean features\nfor column in user_lecture_stats_part.columns:\n    bool_column = column + '_boolean'\n    user_lecture_stats_part[bool_column] = (user_lecture_stats_part[column] > 0).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lecture_stats_part.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#clearing memory\ntrain_lectures = None\ndel(train_lectures)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_X_Y(_train):\n    _train = _train[_train.content_type_id == False]\n    #arrange by timestamp\n    _train = _train.sort_values(['timestamp'], ascending=True)\n\n    _train.drop(['timestamp','content_type_id'], axis=1,   inplace=True)\n    \n    X = _train.iloc[train_part_len:,:]\n\n    X = X[X.answered_correctly!= -1 ]\n    X = X.sort_values(['user_id'])\n    Y = X[[\"answered_correctly\"]]\n    X = X.drop([\"answered_correctly\"], axis=1)\n\n    return X, Y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import category_encoders as ce\ncount_en = ce.CountEncoder()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def addFeatures(_train, _questions):\n    train_questions_only_df = _train[_train['answered_correctly']!=-1]\n    grouped_by_user_df = train_questions_only_df.groupby('user_id')\n    user_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count', 'sum']}).copy()\n    user_answers_df.columns = ['mean_answered_correctly_user', 'questions_answered', 'sum_answered_correctly_user']\n    \n    grouped_by_content_df = train_questions_only_df.groupby('content_id')\n    content_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count'] }).copy()\n    content_answers_df.columns = ['mean_answered_correctly_content', 'question_asked']\n    \n    questions_df = _questions.merge(content_answers_df, left_on = 'question_id', right_on = 'content_id', how = 'left')\n    bundle_dict = questions_df['bundle_id'].value_counts().to_dict()\n    questions_df['right_answers'] = questions_df['mean_answered_correctly_content'] * questions_df['question_asked']\n    questions_df['bundle_size'] = questions_df['bundle_id'].apply(lambda x: bundle_dict[x])\n\n    grouped_by_bundle_df = questions_df.groupby('bundle_id')\n    bundle_answers_df = grouped_by_bundle_df.agg({'right_answers': 'sum', 'question_asked': 'sum'}).copy()\n    bundle_answers_df.columns = ['bundle_rignt_answers', 'bundle_questions_asked']\n    bundle_answers_df['bundle_accuracy'] = bundle_answers_df['bundle_rignt_answers'] / bundle_answers_df['bundle_questions_asked']\n\n    grouped_by_part_df = questions_df.groupby('part')\n    part_answers_df = grouped_by_part_df.agg({'right_answers': 'sum', 'question_asked': 'sum'}).copy()\n    part_answers_df.columns = ['part_rignt_answers', 'part_questions_asked']\n    part_answers_df['part_accuracy'] = part_answers_df['part_rignt_answers'] / part_answers_df['part_questions_asked']\n\n    return user_answers_df, questions_df, bundle_answers_df, part_answers_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def handleMissing(X, Y, _user_answers, _questions, _bundle_answers, _part_answers, _lectures):\n    X['prior_question_had_explanation'].fillna(False, inplace=True)\n    X[\"prior_question_had_explanation_enc\"] = X['prior_question_had_explanation'].apply(convertBoolean)\n    \n    X = X[['user_id', 'content_id', 'prior_question_elapsed_time','prior_question_had_explanation_enc']] \n    \n    X = X.merge(_user_answers, how = 'left', on = 'user_id')\n    X = X.merge(_questions, how = 'left', left_on = 'content_id', right_on = 'question_id')\n    X = X.merge(_bundle_answers, how = 'left', on = 'bundle_id')\n    X = X.merge(_part_answers, how = 'left', on = 'part')\n    X = X.merge(_lectures, how = 'left', on=['user_id'])\n    \n    X[\"toeic_section\"] = X['part'].apply(TOEICSection)\n\n    cat_ce = count_en.fit_transform(X[\"toeic_section\"])\n    X = X.join(cat_ce.add_suffix(\"_ce\"))\n\n    X[\"tags_count\"] = X['tags'].apply(tagsCount)\n    X = X.drop(columns=['user_id', 'content_id', 'bundle_id', 'correct_answer', 'tags', 'toeic_section'])\n    X['sum_answered_correctly_user'].fillna(0, inplace=True)\n    \n    X['part_1'].fillna(0, inplace = True)\n    X['part_2'].fillna(0, inplace = True)\n    X['part_3'].fillna(0, inplace = True)\n    X['part_4'].fillna(0, inplace = True)\n    X['part_5'].fillna(0, inplace = True)\n    X['part_6'].fillna(0, inplace = True)\n    X['part_7'].fillna(0, inplace = True)\n    X['type_of_concept'].fillna(0, inplace = True)\n    X['type_of_intention'].fillna(0, inplace = True)\n    X['type_of_solving_question'].fillna(0, inplace = True)\n    X['type_of_starter'].fillna(0, inplace = True)\n    X['part_1_boolean'].fillna(0, inplace = True)\n    X['part_2_boolean'].fillna(0, inplace = True)\n    X['part_3_boolean'].fillna(0, inplace = True)\n    X['part_4_boolean'].fillna(0, inplace = True)\n    X['part_5_boolean'].fillna(0, inplace = True)\n    X['part_6_boolean'].fillna(0, inplace = True)\n    X['part_7_boolean'].fillna(0, inplace = True)\n    X['type_of_concept_boolean'].fillna(0, inplace = True)\n    X['type_of_intention_boolean'].fillna(0, inplace = True)\n    X['type_of_solving_question_boolean'].fillna(0, inplace = True)\n    X['type_of_starter_boolean'].fillna(0, inplace = True)\n    \n    X.fillna(0.5, inplace=True)\n    \n    target_columns = ['question_id', 'part']\n    target_enc = ce.TargetEncoder(cols=target_columns)\n    target_enc.fit(X[target_columns], Y)\n    X = X.join(target_enc.transform(X[target_columns]).add_suffix('_target'))\n    \n    catboost_columns = ['prior_question_elapsed_time', 'prior_question_had_explanation_enc']\n    catboost_enc = ce.CatBoostEncoder(cols=catboost_columns)\n    catboost_enc.fit(X[catboost_columns], Y)\n    X = X.join(catboost_enc.transform(X[catboost_columns]).add_suffix('_catboost'))\n\n    return X, target_enc, catboost_enc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_full, Y_full = split_X_Y(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_full.info()\n# print('Part of missing values for every column')\n# print(X_full.isnull().sum() / len(X_full))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_answers_df, questions_df, bundle_answers_df, part_answers_df = addFeatures(train, questions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = None\ndel(train)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_full, target_enc, catboost_enc = handleMissing(X_full, Y_full, user_answers_df, questions_df, bundle_answers_df, part_answers_df, user_lecture_stats_part)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_full.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nfrom xgboost import XGBClassifier, plot_importance\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = XGBClassifier(\n    tree_method=\"hist\",\n    learning_rate=0.1,\n    gamma=0.2,\n    n_estimators=200,\n    max_depth=8,\n    min_child_weight=40,\n    subsample=0.87,\n    colsample_bytree=0.95,\n    reg_alpha=0.04,\n    reg_lambda=0.073,\n    objective='binary:logistic',\n    nthread=4,\n    scale_pos_weight=1,\n    seed=27\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X_full, Y_full.values.ravel())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc_score(Y_full.values, model.predict_proba(X_full)[:,1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10,8))\nplot_importance(model, ax=ax)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df[\"prior_question_had_explanation_enc\"] = test_df['prior_question_had_explanation'].apply(convertBoolean)\n\n    test_df = test_df.merge(user_answers_df, how = 'left', on = 'user_id')\n    test_df = test_df.merge(questions_df, how = 'left', left_on = 'content_id', right_on = 'question_id')\n    test_df = test_df.merge(bundle_answers_df, how = 'left', on = 'bundle_id')\n    test_df = test_df.merge(part_answers_df, how = 'left', on = 'part')\n    test_df = test_df.merge(user_lecture_stats_part, how = 'left', on=['user_id'])\n    test_df['part_1'].fillna(0, inplace = True)\n    test_df['part_2'].fillna(0, inplace = True)\n    test_df['part_3'].fillna(0, inplace = True)\n    test_df['part_4'].fillna(0, inplace = True)\n    test_df['part_5'].fillna(0, inplace = True)\n    test_df['part_6'].fillna(0, inplace = True)\n    test_df['part_7'].fillna(0, inplace = True)\n    test_df['type_of_concept'].fillna(0, inplace = True)\n    test_df['type_of_intention'].fillna(0, inplace = True)\n    test_df['type_of_solving_question'].fillna(0, inplace = True)\n    test_df['type_of_starter'].fillna(0, inplace = True)\n    test_df['part_1_boolean'].fillna(0, inplace = True)\n    test_df['part_2_boolean'].fillna(0, inplace = True)\n    test_df['part_3_boolean'].fillna(0, inplace = True)\n    test_df['part_4_boolean'].fillna(0, inplace = True)\n    test_df['part_5_boolean'].fillna(0, inplace = True)\n    test_df['part_6_boolean'].fillna(0, inplace = True)\n    test_df['part_7_boolean'].fillna(0, inplace = True)\n    test_df['type_of_concept_boolean'].fillna(0, inplace = True)\n    test_df['type_of_intention_boolean'].fillna(0, inplace = True)\n    test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)\n    test_df['type_of_starter_boolean'].fillna(0, inplace = True)\n    \n    test_df[\"toeic_section\"] = test_df['part'].apply(TOEICSection)\n    \n    cat_ce = count_en.fit_transform(test_df[\"toeic_section\"])\n    test_df = test_df.join(cat_ce.add_suffix(\"_ce\"))\n    \n    test_df[\"tags_count\"] = test_df['tags'].apply(tagsCount)\n    test_df['sum_answered_correctly_user'].fillna(0, inplace=True)\n    test_df.fillna(0.5, inplace=True)\n    \n    target_columns = ['question_id', 'part']\n    test_df = test_df.join(target_enc.transform(test_df[target_columns]).add_suffix('_target'))\n    catboost_columns = ['prior_question_elapsed_time', 'prior_question_had_explanation_enc']\n    test_df = test_df.join(catboost_enc.transform(test_df[catboost_columns]).add_suffix('_catboost'))\n\n    test_df['answered_correctly'] =  model.predict_proba(test_df[X_full.columns])[:,1]\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}