{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Introduction\n\nThis is the best I could do with a fastai tabular model. It contains a number of features particular to both the user and the question at that point in time. Then for inference there is a dictionary filled with each user's history and their current state. This helped dramatically in improving my LB score. I did pull code from a bunch of different notebooks, so I'll reference the original material as it comes up. \n\nA big problem with using a fastai tabular model is building the TabularPandas during inference. It takes way too long with certain categorical features, so we just can't use them. In training I was able to use user_id as a categorical feature, which seemed really useful. But it's too slow to use during inference.\n\nA big thanks to all the users out there sharing kernels, I've learned a ton from these notebooks and this competition. "},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score     \nfrom fastai.tabular.all import *\nimport gc\nimport fastai\nfrom tqdm.notebook import tqdm as tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Initial Load"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_df = pd.read_feather('/kaggle/input/riid-with-attempts-feather/train_with_attempts')\n# The dataframe is already sorted by time\ntrain_df.drop('timestamp', axis=1, inplace=True)\ntrain_df.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Feature generation"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/shoheiazuma/riiid-lgbm-starter\n# This notebook helped me create features that didn't leak future information, which was very helpful\ncontent_agg = pd.read_feather('/kaggle/input/riid-with-attempts-feather/content_agg_feather')\ncontent_agg.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nquestions_df = pd.read_csv(\n    '../input/riiid-test-answer-prediction/questions.csv', \n    usecols=[0, 1, 3, 4],\n    dtype={'question_id': 'int16', 'bundle_id': 'int16', 'part': 'int8', 'tags': 'object'}\n)\n\n# Split the tags into their own individual columns\ntags = questions_df['tags'].str.split(' ', n=10, expand=True)\ntags.columns = ['tags1', 'tags2', 'tags3', 'tags4', 'tags5', 'tags6']\n# Drop the original tags column plus unimportant tags from feature importance\nquestions_df.drop(['tags'], axis=1, inplace=True)\n# While investgating feature importance, tags2 and tags3 seemed to be the most useful. \ntags.drop(['tags1', 'tags4', 'tags5', 'tags6'], axis=1, inplace=True)\n# Concat the individual tag columns\nquestions_df = pd.concat([questions_df, tags], axis=1)\n\nquestions_df['tags2'] = pd.to_numeric(questions_df['tags2'], errors='coerce',downcast='integer').fillna(-1)\nquestions_df['tags3'] = pd.to_numeric(questions_df['tags3'], errors='coerce',downcast='integer').fillna(-1)\nquestions_df['tags2'] = questions_df['tags2'].astype('uint16')\nquestions_df['tags3'] = questions_df['tags3'].astype('uint16')\n\ntrain_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')\ntrain_df.drop(columns=['question_id'], inplace=True)\n\n# del questions_df\ndel tags\ngc.collect()\ntrain_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat = ['part', 'tags2', 'tags3', 'bundle_id']\ncont = ['user_count', 'user_correctness', 'content_count', 'content_correctness', 'prior_question_elapsed_time', \n        'attempts', 'hmean_user_content']\n\nfeatures = cat + cont","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def replace_na(x):\n    x = x.replace([np.inf, -np.inf], np.nan)\n\n    x['tags2'].fillna(255, inplace=True)\n    x['tags3'].fillna(255, inplace=True)\n\n    return x.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get rid of all the columns we won't be needing\ntrain_df = train_df[features + [target]]\ntrain_df = replace_na(train_df)\ntrain_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# I use the last quarter of the data to train on, with the last tenth of that as validation. Given time and spatial restrictions, this seems the most useful\nthresh = int(0.75 * len(train_df))\ntrain_df = train_df.iloc[thresh:].reset_index(drop=True)\ntrain_range = list(range(len(train_df) - int(len(train_df) / 10)))\nvalid_range = list(range(len(train_df) - int(len(train_df) / 10), len(train_df)))\nsplits = (train_range, valid_range)\n\ngc.collect()\ntrain_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nEPOCHS=5\nBATCH_SIZE=2048\n\npd.options.mode.chained_assignment=None\nto = TabularPandas(train_df, [Categorify, Normalize, FillMissing], cat, cont, target, splits=splits, inplace=True)\ndls = to.dataloaders(BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/gilfernandes/fastai-single-nn\n# my_auc and predict_batch are both from this kernel\ndef my_auc(inp, targ):\n    \"Simple wrapper around scikit's roc_auc_score function for regression problems\"\n    inp,targ = flatten_check(inp,targ)\n    return roc_auc_score(targ.cpu().numpy(), inp.cpu().numpy())\n\nlearn = tabular_learner(dls, layers=[200,200], metrics=my_auc)\n\nlr_find_res = learn.lr_find()\n\nlearn.fit_one_cycle(EPOCHS, lr_find_res.lr_min / 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_batch(self, df):\n    dl = self.dls.test_dl(df)\n    dl.dataset.conts = dl.dataset.conts.astype(np.float32)\n    inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)\n    return preds.numpy()\n\nsetattr(learn, 'predict_batch', predict_batch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del dls\ndel to\ndel train_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_df = pd.read_feather('/kaggle/input/riid-with-attempts-feather/train_with_attempts')\ntrain_df = train_df.loc[train_df['answered_correctly'] != -1]\ntrain_df.sort_values('timestamp', inplace=True)\ntrain_df.drop(['prior_question_had_explanation', 'prior_question_elapsed_time', 'timestamp', 'attempts', \n               'user_correctness', 'user_count', 'content_correctness', 'content_count', 'hmean_user_content'], axis=1, inplace=True)\ntrain_df.reset_index(drop=True, inplace=True)\ngc.collect()\ntrain_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# https://www.kaggle.com/markwijkhuizen/riiid-training-and-prediction-using-a-state\n# This is an altered piece from that notebook to be keep track of user states\n\n# Creates a dictionary of dictionaries for each user and their attributes\ndef user_states(df):\n    mean_user_accuracy = train_df.groupby('user_id')[target].mean().values\n    answered_correctly_user = train_df.groupby('user_id')[target].sum().values\n    answered_user = train_df.groupby('user_id')[target].count().values  \n\n    state = dict()\n    for user_id in train_df['user_id'].sort_values().unique():\n        state[user_id] = {}\n    total = len(state.keys())\n\n    user_content = train_df.groupby('user_id')['content_id'].apply(np.array).apply(np.sort).apply(np.unique)\n    # The number of attempts made for each of those unique combinations\n    user_attempts = train_df.groupby(['user_id', 'content_id'])[target].count().astype(np.uint8).groupby('user_id').apply(np.array).values\n\n    for user_id, content, attempt in tqdm(zip(state.keys(), user_content, user_attempts),total=total):\n        state[user_id]['user_content_attempts'] = dict(zip(content, attempt))\n\n    del user_content, user_attempts\n    gc.collect()\n\n    for idx, user_id in enumerate(state.keys()):\n        state[user_id]['user_correctness'] = mean_user_accuracy[idx]\n        state[user_id]['user_sum'] = answered_correctly_user[idx]\n        state[user_id]['user_count'] = answered_user[idx]\n        \n    return state\n  \nuser_states = user_states(train_df)\ndel train_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\n\niter_test = env.iter_test()\nprior_test_df = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_sum_dict = content_agg['sum'].astype('int32').to_dict(defaultdict(int))\ncontent_count_dict = content_agg['count'].astype('int32').to_dict(defaultdict(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/shoheiazuma/riiid-lgbm-starter\n# The skeleton for this inference part was taken from the above notebook, including a number of modifications.\n\nfor (test_df, sample_prediction_df) in iter_test:\n    y_preds = []\n    \n    # If test_df includes information about the previous chunk of data, update our states to reflect that\n    if prior_test_df is not None:   \n        prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)\n        \n        user_ids = prior_test_df['user_id'].values\n        content_ids = prior_test_df['content_id'].values\n        targets = prior_test_df[target].values\n        \n        for user_id, content_id, answered_correctly in zip(user_ids, content_ids, targets):\n            user_states[user_id]['user_sum'] += answered_correctly\n            user_states[user_id]['user_count'] += 1\n            user_states[user_id]['user_content_attempts'][content_id] += 1\n            content_sum_dict[content_id] += answered_correctly\n            content_count_dict[content_id] += 1\n      \n    prior_test_df = test_df.copy()\n    \n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    \n    test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')\n    \n    user_sum = np.zeros(len(test_df), dtype=np.int16)\n    user_count = np.zeros(len(test_df), dtype=np.int16)\n    content_sum = np.zeros(len(test_df), dtype=np.int32)\n    content_count = np.zeros(len(test_df), dtype=np.int32)\n    user_content_attempts = np.zeros(len(test_df), dtype=np.int8)\n    \n    for i, (user_id, content_id) in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values)):\n        # Handle new users and questions\n        if user_id not in user_states.keys():\n            user_states[user_id] = {'user_content_attempts':{user_id:0}, \n                                    'user_correctness':0, \n                                    'user_count':0, \n                                    'user_sum':0}\n            \n        if content_id not in user_states[user_id]['user_content_attempts'].keys():\n            user_states[user_id]['user_content_attempts'][content_id] = 0\n            \n        user_sum[i] = user_states[user_id]['user_sum']\n        user_count[i] = user_states[user_id]['user_count']\n        user_content_attempts[i] = user_states[user_id]['user_content_attempts'][content_id]\n\n        content_sum[i] = content_sum_dict[content_id]\n        content_count[i] = content_count_dict[content_id]\n \n    test_df['user_correctness'] = user_sum / user_count\n    test_df['user_count'] = user_count\n    \n    test_df['content_count'] = content_count\n    test_df['content_correctness'] = content_sum / content_count\n    test_df['attempts'] = user_content_attempts\n    test_df['hmean_user_content'] = 2 * (\n                                            (test_df['user_correctness'] * test_df['content_correctness']) /\n                                            (test_df['user_correctness'] + test_df['content_correctness'])\n                                        ).astype('float32')\n\n    test_df = replace_na(test_df)\n    test_df[target] = learn.predict_batch(learn, test_df[features])\n\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}