{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Tabnet Starter\nSimple starter notebook, which uses for prediction a simple ensemble with tabnet and a linear model using the statsmodel library and another fast ai neural network."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.1-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\nimport datatable as dt\n\nimport statsmodels.api as sm\nfrom sklearn.metrics import roc_auc_score\n\nfrom matplotlib import pyplot as plt\nimport riiideducation\nfrom pathlib import Path\nimport seaborn as sns\n\nfrom pytorch_tabnet.tab_model import TabNetClassifier","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input')\nassert path.exists()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndata_types_dict = {\n    'user_id': 'int32', \n    'content_id': 'int16', \n    'answered_correctly': 'int8', \n    'prior_question_elapsed_time': 'float32', \n    'prior_question_had_explanation': 'bool'\n}\ntarget = 'answered_correctly'\ntrain_df = dt.fread(path/\"riidtrainjay/train.jay\", columns=set(data_types_dict.keys())).to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_df = train_df[train_df[target] != -1].reset_index(drop=True)\ntrain_df.drop(columns=['timestamp'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Feature generation"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['prior_question_had_explanation'].fillna(False, inplace=True)\ntrain_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].astype('uint8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['lag'] = train_df.groupby('user_id')[target].shift()\ntrain_df['lag'].fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ncum = train_df.groupby(['user_id'])['lag'].agg(['cumsum', 'cumcount'])\ntrain_df['user_correctness'] = cum['cumsum'] / cum['cumcount']\ntrain_df.drop(columns=['lag'], inplace=True)\ndel cum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])\ncontent_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for f in ['prior_question_elapsed_time']:\n    train_df[f] = pd.to_numeric(train_df[f], downcast='float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_df['residual'] =  train_df[target] - train_df['content_id'].map(content_agg['sum'] / content_agg['count'])\nresidual_agg = train_df.groupby('user_id')['residual'].agg(['sum'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prior_question_elapsed_time_agg = train_df.groupby('user_id').agg({'prior_question_elapsed_time': ['sum', lambda x: len(x)]})\nprior_question_elapsed_time_agg.columns = ['sum', 'count']\nprior_question_elapsed_time_agg['count'] = prior_question_elapsed_time_agg['count'].astype('int32')\nprior_question_elapsed_time_agg.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Covariance between time and user mean\n\ntrain_df['user_prior_question_elapsed_time_diff'] = (train_df[target] - train_df['user_id'].map(user_agg['sum'] / user_agg['count'])) * (train_df['prior_question_elapsed_time'] - train_df['user_id'].map(prior_question_elapsed_time_agg['sum'] / prior_question_elapsed_time_agg['count']))\nuser_prior_question_elapsed_time_diff_agg = train_df.groupby('user_id')['user_prior_question_elapsed_time_diff'].agg(['sum'])\ntrain_df['user_prior_question_elapsed_time_diff_mean'] = train_df['user_id'].map(user_prior_question_elapsed_time_diff_agg['sum'] / user_agg['count'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"USER_TRIES = 70\n\nimport math\nVALID_TRIES = math.ceil(USER_TRIES / 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.groupby('user_id').tail(USER_TRIES).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Question related"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_types_dict = {'question_id': 'int16', 'part': 'int8', 'bundle_id': 'int16', 'tags': 'string'}\n\nquestions_df = pd.read_csv(\n    path/'riiid-test-answer-prediction/questions.csv', \n    usecols=data_types_dict.keys(),\n    dtype=data_types_dict\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_tags_combos_keys = {v:i for i,v in enumerate(questions_df['tags'].unique())}\nquestions_df['tags_encoded'] = questions_df['tags'].apply(lambda x : unique_tags_combos_keys[x])\nquestions_df['tags_encoded'] = pd.to_numeric(questions_df['tags_encoded'], downcast='integer')\nquestions_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_tag_factory(tag_pos):\n    def extract_tag(x):\n        if isinstance(x, str) and tag_pos < len(x.split()):\n            splits = x.split()\n            splits.sort()\n            return int(splits[tag_pos])\n        else:\n            return 255\n    return extract_tag\n        \nfor i in range(0, 3):\n    questions_df[f'tag_{i + 1}'] = questions_df['tags'].apply(extract_tag_factory(i))\n    questions_df[f'tag_{i + 1}'] = questions_df[f'tag_{i + 1}'].astype('uint8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')\ntrain_df.drop(columns=['question_id'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['content_count'] = train_df['content_id'].map(content_agg['count']).astype('int32')\ntrain_df['content_id'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['prior_question_elapsed_time_mean'] = train_df['user_id'].map(prior_question_elapsed_time_agg['sum'] / prior_question_elapsed_time_agg['count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['residual_user_mean'] = train_df['user_id'].map(residual_agg['sum'] / user_agg['count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['prior_question_elapsed_time'].fillna(train_df['prior_question_elapsed_time'].mean(), inplace=True)\ntrain_df['user_correctness'].fillna(train_df['user_correctness'].mean(), inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for f in ['user_correctness', 'content_id']:\n    train_df[f] = pd.to_numeric(train_df[f], downcast='float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df = train_df.groupby('user_id').tail(VALID_TRIES)\n# train_df.drop(valid_df.index, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['user_correctness'] = train_df['user_correctness'].replace(train_df['user_correctness'].mean(), 0.0)\nvalid_df['user_correctness'] = valid_df['user_correctness'].replace(valid_df['user_correctness'].mean(), 0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n    'prior_question_elapsed_time',\n    'prior_question_had_explanation',\n    'user_correctness',\n    'part',\n    'content_id',\n    'content_count',\n    'tags_encoded',\n    'tag_1',\n    'tag_2',\n    'prior_question_elapsed_time_mean',\n    'residual_user_mean'\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Linear model"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nmodel = sm.OLS(train_df[target], train_df[features])\nlin_model = model.fit()\nroc_auc_score(valid_df[target], lin_model.predict(valid_df[features]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lin_model.predict(valid_df[features].values[:10])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Fast AI"},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS=5\nBATCH_SIZE=4096","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_features = ['tags_encoded', 'tag_1', 'tag_2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cont_features = [x for x in features if x not in cat_features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cont_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check fast ai version\nimport fastai\nfrom fastai.tabular.all import *\n\nfastai.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[target] = train_df[target].astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndls = TabularDataLoaders.from_df(train_df, \n    procs=[Categorify, FillMissing, Normalize],\n    cat_names=cat_features, \n    cont_names=cont_features,\n    y_names=target, valid_idx=valid_df.index, bs=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def my_auc(inp, targ):\n    \"Simple wrapper around scikit's roc_auc_score function for regression problems\"\n    inp,targ = flatten_check(inp,targ)\n    return roc_auc_score(targ.cpu().numpy(), inp.cpu().numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bce(inp,targ):\n    \"Binary cross entropy\"\n    inp,targ = flatten_check(inp,targ)\n    loss = F.binary_cross_entropy(inp, targ)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = tabular_learner(dls, layers=[200,100], metrics=my_auc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.model.layers.add_module('sigmoid', nn.Sigmoid())\nlearn.loss_func = bce","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_find_res = learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nlearn.fit_one_cycle(3, lr=lr_find_res.lr_min)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_batch(self, df):\n    dl = self.dls.test_dl(df)\n    dl.dataset.conts = dl.dataset.conts.astype(np.float32)\n    inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)\n    return preds.numpy()\n\nsetattr(learn, 'predict_batch', predict_batch)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### LightGBM"},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_features = ['tags_encoded', 'tag_1', 'tag_2']\n\nlgb_train = lgb.Dataset(train_df[features], train_df[target], categorical_feature = cat_features, free_raw_data=False)\nlgb_eval = lgb.Dataset(valid_df[features], valid_df[target], categorical_feature = cat_features, reference=lgb_train, free_raw_data=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"METRICS = ['auc']\n\nparams = {\n    'objective': 'binary',\n    'seed': 42,\n    'metric': METRICS,\n    'learning_rate': 0.05,\n    'max_bin': 800,\n    'num_leaves': 80\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nNUM_BOOST_ROUNDS = 300\n\nevals_result = {}\n\nlgb_model = lgb.train (\n    params, \n    lgb_train, \n    valid_sets=[lgb_train, lgb_eval], \n    verbose_eval=20, \n    num_boost_round=NUM_BOOST_ROUNDS, \n    early_stopping_rounds=20,\n    evals_result=evals_result\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_importance(lgb_model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Tabnet"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\n\n# Tabnet object\nclf_tabnet = TabNetClassifier(cat_idxs=[list(train_df[features].columns).index(x) for x in cat_features], \n                              scheduler_params={\"step_size\": 50, \"gamma\": 0.9},\n                                scheduler_fn=torch.optim.lr_scheduler.StepLR)\nclf_tabnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Fit TabNet model\nclf_tabnet.fit(\n    X_train=train_df[features].values, y_train=train_df[target].values,\n    eval_set=[(valid_df[features].values, valid_df[target].values)],\n    max_epochs=2,\n    batch_size=8192 * 4\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = clf_tabnet.predict_proba(valid_df[features].values[:1000])\npreds[:,1].shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_sum_dict = user_agg['sum'].astype('int16').to_dict(defaultdict(int))\nuser_count_dict = user_agg['count'].astype('int16').to_dict(defaultdict(int))\ncontent_sum_dict = content_agg['sum'].astype('int32').to_dict(defaultdict(int))\ncontent_count_dict = content_agg['count'].astype('int32').to_dict(defaultdict(int))\nresidual_sum_dict = residual_agg['sum'].astype('float32').to_dict(defaultdict(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prior_question_elapsed_time_sum_dict = prior_question_elapsed_time_agg['sum'].astype('int64').to_dict(defaultdict(int))\nprior_question_elapsed_time_count_dict = prior_question_elapsed_time_agg['count'].astype('int32').to_dict(defaultdict(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\nprior_test_df = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clip(count): return np.clip(count, 1e-8, np.inf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    if prior_test_df is not None:\n        prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)\n        \n        user_ids = prior_test_df['user_id'].values\n        content_ids = prior_test_df['content_id'].values\n        prior_question_elapsed_times = prior_test_df['prior_question_elapsed_time'].values\n        targets = prior_test_df[target].values\n        \n        for user_id, content_id, prior_question_elapsed_time, answered_correctly in zip(user_ids, content_ids, prior_question_elapsed_times, targets):\n            user_sum_dict[user_id] += answered_correctly\n            user_count_dict[user_id] += 1\n            content_sum_dict[content_id] += answered_correctly\n            content_count_dict[content_id] += 1\n            mean_accuracy = content_sum_dict[content_id] / clip(content_count_dict[content_id])\n            residual_sum_dict[user_id] += answered_correctly - mean_accuracy\n            \n            prior_question_elapsed_time_sum_dict[user_id] += 0 if np.isnan(prior_question_elapsed_time) else prior_question_elapsed_time\n            prior_question_elapsed_time_count_dict[user_id] += 0 if np.isnan(prior_question_elapsed_time) else 1\n    \n    prior_test_df = test_df.copy()\n    \n    test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    \n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False).astype('uint8')\n    \n    user_sum = np.zeros(len(test_df), dtype=np.int16)\n    user_count = np.zeros(len(test_df), dtype=np.int16)\n    res_sum = np.zeros(len(test_df), dtype=np.float32)\n    content_sum = np.zeros(len(test_df), dtype=np.int32)\n    content_count = np.zeros(len(test_df), dtype=np.int32)\n    prior_question_elapsed_time_sum = np.zeros(len(test_df), dtype=np.int32)\n    prior_question_elapsed_time_count = np.zeros(len(test_df), dtype=np.int32)\n    \n    for i, (user_id, content_id) in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values)):\n        user_sum[i] = user_sum_dict[user_id]\n        user_count[i] = user_count_dict[user_id]\n        res_sum[i] = residual_sum_dict[user_id]\n        content_sum[i] = content_sum_dict[content_id]\n        content_count[i] = content_count_dict[content_id]\n        prior_question_elapsed_time_sum[i] = prior_question_elapsed_time_sum_dict[user_id]\n        prior_question_elapsed_time_count[i] = prior_question_elapsed_time_count_dict[user_id]\n\n    content_count = clip(content_count)\n    user_count = clip(user_count)\n    prior_question_elapsed_time_count = clip(prior_question_elapsed_time_count)\n    test_df['user_correctness'] = user_sum / user_count\n    test_df['residual_user_mean'] = res_sum / user_count\n    test_df['content_count'] = content_count\n    test_df['content_id'] = content_sum / content_count\n    test_df['prior_question_elapsed_time_mean'] = prior_question_elapsed_time_sum / prior_question_elapsed_time_count\n    \n    test_df['prior_question_elapsed_time'].fillna(train_df['prior_question_elapsed_time'].mean(), inplace=True)\n    \n    test_df[cat_features] = test_df[cat_features].apply(pd.to_numeric, downcast='integer')\n    test_df.fillna(0, inplace=True)\n       \n    test_df[target] = np.average([\n        clf_tabnet.predict_proba(test_df[features].values)[:,1],\n        lin_model.predict(test_df[features]),\n        learn.predict_batch(learn, test_df[features])[:,0],\n        lgb_model.predict(test_df[features])\n    ], weights=[0.4, 0.0, 0.0, 0.6], axis=0)\n    \n    env.predict(test_df[['row_id', target]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[target] = np.average([\n    clf_tabnet.predict_proba(test_df[features].values)[:,1],\n    lin_model.predict(test_df[features]),\n    learn.predict_batch(learn, test_df[features])[:,0],\n    lgb_model.predict(test_df[features])\n], weights=[0.4, 0.0, 0.0, 0.6], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[['row_id','answered_correctly']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}