{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from pathlib import Path\nfrom sklearn.metrics import roc_auc_score\n        \nfrom fastai.tabular.all import *\nfrom sklearn.model_selection import KFold\nfrom tqdm.notebook import tqdm as tqdm\nimport datatable as dt\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check fast ai version\nimport fastai\nfastai.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input/riiid-test-answer-prediction')\nassert path.exists()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Initial Load"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_df = dt.fread(\"/kaggle/input/riidtrainjay/train.jay\").to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for f in ['timestamp', 'prior_question_elapsed_time']:\n    train_df[f] = pd.to_numeric(train_df[f], downcast='float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Feature generation"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_df.drop(train_df.columns.difference(['timestamp', 'user_id', 'content_id', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']), 1, inplace=True)\ntrain_df = train_df.loc[train_df['answered_correctly'] != -1].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.sort_values(['timestamp'], ascending=True)\nquestions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\ndel train_df['timestamp']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a unique tag combination representation\n\nunique_tags_combos_keys = {v:i for i,v in enumerate(questions_df['tags'].unique())}\nquestions_df['tags_encoded'] = questions_df['tags'].apply(lambda x : unique_tags_combos_keys[x])\nquestion_tags_df = questions_df[['question_id', 'tags_encoded']].copy()\nquestion_tags_df.set_index('question_id', inplace=True)\nquestion_tags_df['tags_encoded'] = pd.to_numeric(question_tags_df['tags_encoded'], downcast='integer')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_tag_factory(tag_pos):\n    def extract_tag(x):\n        if isinstance(x, str) and tag_pos < len(x.split()):\n            splits = x.split()\n            splits.sort()\n            return int(splits[tag_pos])\n        else:\n            return 255\n    return extract_tag\n        \nfor i in range(0, 2):\n    questions_df[f'tag_{i + 1}'] = questions_df['tags'].apply(extract_tag_factory(i))\n    questions_df[f'tag_{i + 1}'] = questions_df[f'tag_{i + 1}'].astype('uint8')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_tags_df = question_tags_df.merge(questions_df[['question_id', 'bundle_id', 'tag_1', 'tag_2']], how='left', on='question_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_tags_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.merge(question_tags_df, how='left', left_on='content_id', right_index=True)\ndel train_df['question_id']\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].fillna(value = False).astype(bool)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"thresh = int(len(train_df) * 0.1)\nX = train_df[:thresh]\nfeatures_df = train_df[thresh:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Create a field to see how far a user is from the mean of content id answered_correctly\n\ncontent_id_answered_correctly_mean = pd.DataFrame(features_df.groupby('content_id')['answered_correctly'].agg('mean'))\ncontent_id_answered_correctly_mean.columns = ['content_id_answered_correctly_mean']\nfeatures_df = features_df.merge(content_id_answered_correctly_mean, how='left', left_on='content_id', right_index=True)\nfeatures_df['resid'] = features_df['answered_correctly'] - features_df['content_id_answered_correctly_mean']\ndel features_df['content_id_answered_correctly_mean']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del content_id_answered_correctly_mean\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stats_funcs = ['mean', 'count', 'std', 'median', 'skew']\n\nfeature_user_cols = ['mean_user_accuracy', 'questions_answered', 'std_user_accuracy', 'median_user_accuracy', 'skew_user_accuracy', 'residual_content_mean']\nfeature_content_cols = ['mean_accuracy', 'question_asked', 'std_accuracy', 'median_accuracy', 'skew_accuracy']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# User based stats on answered_correctly\n\nuser_answers_df = features_df.groupby('user_id').agg({'answered_correctly': stats_funcs, 'resid': ['mean']}).copy()\nuser_answers_df.columns = feature_user_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"percentage of common users: {float(len(user_answers_df)) / float(len(X['user_id'].unique()))}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Question based stats on answered_correctly\n\ncontent_answers_df = features_df.groupby('content_id').agg({'answered_correctly': stats_funcs}).copy()\ncontent_answers_df.columns = feature_content_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Tag based stats on answered_correctly\n\ncontent_tags_df = features_df.groupby('tags_encoded').agg({'answered_correctly': ['mean']}).copy()\ncontent_tags_df.columns = ['tags_encoded_answered_mean']\ncontent_tags_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Bundle based stats on answered_correctly\n\n# bundle_df = features_df.groupby('bundle_id').agg({'answered_correctly': ['mean']}).copy()\n# bundle_df.columns = ['bundle_answered_mean']\n# bundle_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del features_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def merge_features(df):\n    df = df.merge(user_answers_df, how='left', on='user_id')\n    df = df.merge(content_answers_df, how='left', on='content_id')\n    df = df.merge(content_tags_df, how='left', left_on='tags_encoded', right_index=True)\n#     df = df.merge(bundle_df, how='left', left_on='bundle_id', right_index=True)\n    return df\n    \nX = merge_features(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = feature_user_cols + feature_content_cols + ['prior_question_elapsed_time', 'prior_question_had_explanation', 'tags_encoded', 'tag_1', 'tag_2', 'tags_encoded_answered_mean']\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Simple na, overriden by another option below\ndef replace_na(x):\n    return x.replace([np.inf, -np.inf], np.nan).fillna(0)\n\nmean_user_accuracy_default = user_answers_df['mean_user_accuracy'].median()\nstd_user_accuracy_default = user_answers_df['std_user_accuracy'].mean()\nmedian_user_accuracy_default = user_answers_df['median_user_accuracy'].mean()\nskew_user_accuracy_default = user_answers_df['skew_user_accuracy'].mean()\nresidual_content_mean_default = user_answers_df['residual_content_mean'].median()\nquestions_answered_default = user_answers_df['questions_answered'].median()\n\n## Alternative replace na method, which relies on median values\ndef replace_na(x):\n    x = x.replace([np.inf, -np.inf], np.nan)\n    x['mean_user_accuracy'] = x['mean_user_accuracy'].fillna(mean_user_accuracy_default)\n    x['std_user_accuracy'] = x['std_user_accuracy'].fillna(std_user_accuracy_default)\n    x['median_user_accuracy'] = x['median_user_accuracy'].fillna(median_user_accuracy_default)\n    x['skew_user_accuracy'] = x['skew_user_accuracy'].fillna(skew_user_accuracy_default)\n    x['residual_content_mean'] = x['residual_content_mean'].fillna(residual_content_mean_default)\n    x['questions_answered'] = x['questions_answered'].fillna(questions_answered_default)\n    return x.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = X[features + [target]]\nX = replace_na(X)\nX['prior_question_had_explanation'] = X['prior_question_had_explanation'].astype(np.int8)\nX['answered_correctly'] = X['answered_correctly'].astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for f in ['mean_user_accuracy', 'questions_answered', 'std_user_accuracy', 'median_user_accuracy', 'skew_user_accuracy', 'residual_content_mean', \n          'mean_accuracy', 'std_accuracy', 'median_accuracy', 'skew_accuracy', 'tags_encoded_answered_mean']:\n    X[f] = pd.to_numeric(X[f], downcast='float')\n\nX.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# valid_range = list(X.sample(frac=0.1, random_state=42).index)\nvalid_range = list(range(len(X) - int(len(X) / 10), len(X)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS=5\nBATCH_SIZE=2048","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndls = TabularDataLoaders.from_df(X, \n    procs=[Categorify, FillMissing, Normalize],\n    cat_names=['prior_question_had_explanation', 'tags_encoded', 'tag_1', 'tag_2'], \n    cont_names=['mean_user_accuracy', 'questions_answered', 'std_user_accuracy', 'median_user_accuracy', 'skew_user_accuracy', \n                'mean_accuracy', 'question_asked', 'std_accuracy', 'median_accuracy', 'skew_accuracy', 'prior_question_elapsed_time', 'residual_content_mean', 'tags_encoded_answered_mean'],\n    y_names='answered_correctly', valid_idx=valid_range, bs=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def my_auc(inp, targ):\n    \"Simple wrapper around scikit's roc_auc_score function for regression problems\"\n    inp,targ = flatten_check(inp,targ)\n    return roc_auc_score(targ.cpu().numpy(), inp.cpu().numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.nn import functional as F\n\ndef huber(inp,targ):\n    \"Huber loss between `inp` and `targ`.\"\n    inp,targ = flatten_check(inp,targ)\n    loss = F.smooth_l1_loss(inp, targ)\n    return loss\n\ndef bce(inp,targ):\n    \"Huber loss between `inp` and `targ`.\"\n    inp,targ = flatten_check(inp,targ)\n    loss = F.binary_cross_entropy(inp, targ)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = tabular_learner(dls, layers=[200,100], metrics=my_auc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.model.layers.add_module('sigmoid', nn.Sigmoid())\nlearn.loss_func = bce","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_find_res = learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_find_res.lr_min","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nlearn.fit_one_cycle(5, lr=lr_find_res.lr_min)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_loss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_sched()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_batch(self, df):\n    dl = self.dls.test_dl(df)\n    dl.dataset.conts = dl.dataset.conts.astype(np.float32)\n    inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)\n    return preds.numpy()\n\nsetattr(learn, 'predict_batch', predict_batch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nsample_size = 2_000_000\npreds = learn.predict_batch(learn, X[features].iloc[:sample_size])\ny_pred_sample = X[target][:sample_size].values\nroc_auc_score(y_pred_sample, preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    y_preds = []\n    test_df = test_df.merge(question_tags_df, how='left', left_on='content_id', right_index=True)\n    test_df = merge_features(test_df)\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value = False).astype(bool)\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype(np.int8)\n    test_df = replace_na(test_df)\n    test_df.fillna(value=0, inplace = True)\n    \n    y_preds = learn.predict_batch(learn, test_df)\n\n    test_df['answered_correctly'] = y_preds\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Used for checking the users that are new on the test set\ntest_df.loc[~test_df['user_id'].isin(user_answers_df.index)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}