{"cells":[{"metadata":{},"cell_type":"markdown","source":"- First of all, congratulations to all kagglers and thanks to organizers. I have learned a lot from this competition, so I would like to share my solution here. I hope some pepole find new ideas from my solution.\n- I got some ideas from the following great notebooks. \nhttps://www.kaggle.com/yanamal/learning-factor-analysis-are-tags-skills/notebook \nhttps://www.kaggle.com/gilfernandes/riiid-self-attention-transformer \nhttps://www.kaggle.com/its7171/cv-strategy"},{"metadata":{"trusted":true},"cell_type":"code","source":"# installation without internet\n!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl\n!pip install ../input/pandarallel151whl/pandarallel-1.5.1-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nimport os\nimport gc\nimport cv2\nimport joblib\nimport random\nimport warnings\nimport numpy as np \nimport pandas as pd\nimport datatable as dt\nimport tensorflow as tf \nimport matplotlib.pyplot as plt \nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nfrom datetime import datetime\nfrom IPython.display import display\nfrom pandarallel import pandarallel\n\nfrom sklearn.metrics import roc_curve, roc_auc_score\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, KFold, TimeSeriesSplit, GroupKFold, GroupShuffleSplit\n\nimport lightgbm as lgb\nimport optuna\nimport optuna.integration.lightgbm as optuna_lgb\nfrom optuna.visualization import plot_optimization_history\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\ndef seed_everything(seed=42):\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n tf.random.set_seed(seed)\n session_conf = tf.compat.v1.ConfigProto(\n intra_op_parallelism_threads=1,\n inter_op_parallelism_threads=1\n )\n sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(), config=session_conf)\n tf.compat.v1.keras.backend.set_session(sess)\n\nwarnings.simplefilter('ignore')\npd.set_option(\"max_columns\", 150)\npd.set_option('display.max_rows', 150)\nseed_everything(42)\npandarallel.initialize()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_deviation_value(ans_cor_rate, std, method=\"p\"):\n if method == \"p\":\n return (1-ans_cor_rate)*(std/1.5+1)\n else:\n return (ans_cor_rate)*(std/1.5+1)\n\ndev_vals = []\nfor a in np.arange(0,1.01,0.1):\n a = round(a,1)\n for s in np.arange(0, 1.501, 0.1):\n s = round(s,1)\n dev_vals.append([a, s, round(get_deviation_value(a,s),1)])\n\nplt.plot(np.array(dev_vals)[:,2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True, y=[]):\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n start_mem = df.memory_usage().sum() / 1024**2 \n for col in df.columns:\n col_type = df[col].dtypes\n if col in y or col_type not in numerics:\n continue\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)\n end_mem = df.memory_usage().sum() / 1024**2\n if verbose:\n print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n return df\n\ndef showStats(df):\n stats = []\n for col in df.columns:\n stats.append((col,\n df[col].nunique(),\n df[col].value_counts().index[0],\n df[col].value_counts().values[0],\n df[col].isnull().sum() * 100 / df.shape[0],\n df[col].value_counts(normalize=True, dropna=False).values[0] * 100,\n df[col].dtype))\n df_stats = pd.DataFrame(stats, columns=['Feature name', 'Unique values', 'Most frequent item', 'Freuquence of most frequent item',\n 'Missing values(%)', 'Values in the biggest category(%)', 'Type'])\n display(df_stats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# You can only call make_env() once, so don't lose it!\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#=========================\nDEBUG = False\nTUNE = False\nENSEMBLE = True\nS = 0.3\nL = 0.7\n#=========================","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain = reduce_mem_usage(dt.fread(\"../input/riiid-test-answer-prediction/train.csv\").to_pandas())\ndf_qs = reduce_mem_usage(pd.read_csv('../input/riiid-test-answer-prediction/questions.csv'))\ndf_lecs = reduce_mem_usage(pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape, df_qs.shape, df_lecs.shape)\ndisplay(train.head(2))\ndisplay(df_qs.head(2))\ndisplay(df_lecs.head(2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Modify some features\ntrain.prior_question_elapsed_time = (train.prior_question_elapsed_time.fillna(0)).astype(int)\ntrain.prior_question_had_explanation = (train.prior_question_had_explanation.fillna(False) * 1).astype(int)\ntrain.content_type_id = (train.content_type_id * 1).astype(int)\ntrain = reduce_mem_usage(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split to qa data and lecture data\ntrain_lec = train.query(\"content_type_id==1\").copy()\ntrain_lec.reset_index(drop=True, inplace=True)\n\ntrain.drop(train.loc[train.content_type_id==1].index, inplace=True)\ntrain.reset_index(drop=True, inplace=True)\n\n# Rename to question_id\ntrain .rename(columns={\"content_id\": \"question_id\"}, inplace=True)\ntrain_lec.rename(columns={\"content_id\": \"question_id\"}, inplace=True)\n\ngc.collect()\ntrain.shape, train_lec.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature enginnering"},{"metadata":{"trusted":true},"cell_type":"code","source":"def intervaled_cumsum(ar, idx):\n # Make a copy to be used as output array\n out = ar.copy()\n # Get cumumlative values of array\n arc = ar.cumsum()\n # Place differentiated values that when cumumlatively summed later on would\n # give us the desired intervaled cumsum\n out[idx[0]] = ar[idx[0]] - arc[idx[0]-1]\n out[idx[1:-1]] = ar[idx[1:-1]] - np.diff(arc[idx[:-1]-1])\n return out.cumsum()\n\ndef shift_to_prior(ary, sizes_cumsum):\n shifted = np.r_[np.array([0]), ary]\n shifted[sizes_cumsum] = 0\n return shifted[:-1]\n\ndef get_hist_and_training_data(ary):\n return ary[sizes_qcumsum-1], ary[ext_user_bol] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cut(df, t_col, n_col):\n df = df.copy()\n df[n_col] = pd.cut(df[t_col], [-0.1, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.1], labels=False)\n df[n_col] = df[n_col].fillna(99).astype(\"int8\")\n return df\n\ndef add_que_rate_class(df):\n df = df.copy()\n df[\"question_rate_class\"] = 0\n for sta, end in zip(np.arange(0.3, 1.0, 0.1), np.arange(0.4, 1.1, 0.1)):\n sta = round(sta,1)\n end = round(end,1)\n end = 1.1 if end==1 else end\n df.loc[(sta <= df.question_rate)&(df.question_rate < end), \"question_rate_class\"] = int(sta*10)\n return df\n\ndef get_user_items_dict(ary, sizes_cumsum):\n end = sizes_cumsum\n sta = np.hstack([np.array([0]), end[:-1]])\n users = np.unique(ary[:,0])\n\n d = {}\n for u, s, e in zip(users, sta, end):\n d[u] = ary[s:e, 1:]\n return d\n\ndef change_rate_with_num(df, t_col, n_col, method=\"respectively\", n1=5, n2=3):\n df = df.copy()\n if method == \"respectively\":\n df.loc[(df[n_col] < n1)&(df[t_col] <= 0.3), t_col] = 0.3\n df.loc[(df[n_col] < n1)&(0.3 < df[t_col])&(df[t_col] <= 0.7), t_col] = 0.5\n df.loc[(df[n_col] < n1)&(0.7 < df[t_col]), t_col] = 0.7\n else:\n df.loc[(df[n_col] < n2), t_col] = 0.5\n return df\n\ndef change_rate_high_low(df, t_col, h=0.9, l=0.1):\n df = df.copy()\n df.loc[df[t_col] < l, t_col] = l\n df.loc[df[t_col] > h, t_col] = h\n return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define some parameters for training models\nCV = 4\nMODEL_NUM = 1\nUSE_USER_NUM = 10000\nif DEBUG:\n USE_USER_NUM = 500\nif TUNE:\n TRAIN_RATE, TUNE_RATE, TEST_RATE = 0.4, 0.3, 0.3\nelse:\n TRAIN_RATE, TEST_RATE = 0.7, 0.3\n \nprint(\"%s users reduce to %s\" % (train.user_id.nunique(), USE_USER_NUM))\nuser_ids = train.user_id.unique()\n# Random shuffle\nrandom.shuffle(user_ids)\n# Get user ids for training and test\next_user = user_ids[:USE_USER_NUM]\next_user_bol = np.isin(np.array(train.user_id), np.sort(ext_user))\n\n# Get the number of user row\nsizes_quser = np.array(train.user_id.value_counts().sort_index().values, dtype=\"int16\")\nsizes_qcumsum = sizes_quser.cumsum()\n# Get unique user ids\nunique_user_ids = np.array(train.user_id)[sizes_qcumsum-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Caluculate elapsed time stats\ndf_etime = pd.DataFrame({\"question_id\" : np.array(train.question_id[:-1]),\n \"prior_question_elapsed_time\": np.array(train.prior_question_elapsed_time[1:])})\ndf_etime = df_etime.loc[df_etime.prior_question_elapsed_time!=0].copy()\ndf_etime.prior_question_elapsed_time = df_etime.prior_question_elapsed_time / (1000*3600)\ndf_etime[\"etime_kurt\"] = df_etime.prior_question_elapsed_time\ndf_etime[\"etime_std\"] = df_etime.prior_question_elapsed_time\ndf_etime[\"etime_skew\"] = df_etime.prior_question_elapsed_time\ndf_etime_kurt = df_etime.groupby(\"question_id\", as_index=False).etime_kurt.apply(pd.DataFrame.kurt)\ndf_etime_rate = df_etime.groupby(\"question_id\", as_index=False).agg({\"prior_question_elapsed_time\": \"mean\",\n \"etime_std\" : \"std\",\n \"etime_skew\": \"skew\"})\ndf_etime_rate.rename(columns={\"prior_question_elapsed_time\": \"etime_mean\"}, inplace=True)\ndf_etime_rate = df_etime_rate.merge(df_etime_kurt, on=\"question_id\")\ndf_etime_rate.fillna(0, inplace=True)\ndf_etime_rate.question_id = df_etime_rate.question_id.astype(\"int32\")\ndf_etime_rate.etime_mean = df_etime_rate.etime_mean .astype(\"float32\")\ndf_etime_rate.etime_std = df_etime_rate.etime_std .astype(\"float32\")\ndf_etime_rate.etime_kurt = df_etime_rate.etime_kurt .astype(\"float32\")\ndf_etime_rate.etime_skew = df_etime_rate.etime_skew .astype(\"float32\")\n\n# Make dictionary for prediction phase\nd_etime = dict(zip(df_etime_rate.question_id, df_etime_rate.etime_mean))\n\ndel df_etime, df_etime_kurt\ngc.collect()\n\nprint(df_etime_rate.shape)\ndf_etime_rate.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate the difference between a user elapsed time and the average elapsed time\ndf_etmie_rate = train[[\"row_id\",\"user_id\",\"question_id\",\"prior_question_elapsed_time\"]].copy()\ndf_etmie_rate.rename(columns={\"prior_question_elapsed_time\":\"lag_etime\"}, inplace=True)\ndf_etmie_rate.lag_etime = df_etmie_rate.lag_etime / (1000*3600)\ndf_etmie_rate.lag_etime = df_etmie_rate.groupby(\"user_id\").lag_etime.shift(-1)\ndf_etmie_rate = df_etmie_rate.merge(df_etime_rate[[\"question_id\",\"etime_mean\"]], on=\"question_id\")\ndf_etmie_rate.lag_etime = df_etmie_rate.lag_etime - df_etmie_rate.etime_mean\ndf_etmie_rate.sort_values(\"row_id\", inplace=True)\ndf_etmie_rate.reset_index(drop=True, inplace=True)\n\n# Calculate elapsed time rate\ndf_etmie_rate[\"num\"] = 1\ndf_etmie_rate[\"num\"] = df_etmie_rate.groupby(\"user_id\").num.shift(1)\ndf_etmie_rate[\"num\"] = df_etmie_rate.groupby(\"user_id\").num.cumsum()\ndf_etmie_rate[\"num_etime\"] = df_etmie_rate.groupby(\"user_id\").lag_etime.shift(1)\ndf_etmie_rate[\"num_etime\"] = df_etmie_rate.groupby(\"user_id\").num_etime.cumsum()\ndf_etmie_rate[\"etime_rate\"] = df_etmie_rate.num_etime / df_etmie_rate.num\ndf_etmie_rate.fillna(0, inplace=True)\n\n# Save memory\ndf_etmie_rate.num_etime = df_etmie_rate.num_etime .astype(\"float32\")\ndf_etmie_rate.etime_rate = df_etmie_rate.etime_rate.astype(\"float32\")\ndf_etmie_rate.drop([\"row_id\",\"user_id\",\"question_id\",\"lag_etime\",\"etime_mean\",\"num\"], axis=1, inplace=True)\n\nprint(df_etmie_rate.shape)\ndisplay(df_etmie_rate.head(3))\n\n# Store for history and training\nar_etmie_rate = np.array(df_etmie_rate)\nhist_ar_etmie_rate, ar_etmie_rate = get_hist_and_training_data(ar_etmie_rate)\nhist_ar_etmie_rate, ar_etmie_rate = hist_ar_etmie_rate[:,0], ar_etmie_rate[:,1]\n\ndel df_etmie_rate\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Run a one hot encoding for lecture features\ndf_lecs = df_lecs.join(pd.DataFrame(pd.get_dummies(df_lecs.type_of)))\ndf_lecs.rename(columns={\"lecture_id\" :\"question_id\",\n \"solving question\":\"solving_question\"}, inplace=True)\ndf_lecs.drop([\"tag\",\"type_of\"], axis=1, inplace=True)\ndf_lecs = reduce_mem_usage(df_lecs)\n\n# Make tag features and merge with question dataframe\ntags = sum(df_qs.tags.apply(lambda x: [] if x is np.nan else x.split(\" \")), [])\ndf_tags = pd.DataFrame(pd.Series(tags).value_counts(), columns=[\"tag\"])\ntags_num = df_tags.to_dict()[\"tag\"]\ndf_tags.reset_index(inplace=True)\ndf_tags.columns = [\"tag\",\"num\"]\ndf_tags[\"total\"] = df_qs.shape[0]\ndf_tags[\"tfidf\"] = 1 * np.log(df_tags.total / df_tags.num) # TF is always 1\ndf_tags.index = df_tags.tag\ntags_tfidf = df_tags[[\"tfidf\"]].to_dict()[\"tfidf\"]\ndf_tags = pd.DataFrame(list(df_qs.tags.apply(lambda x: [] if x is np.nan else x.split(\" \"))),\n columns=[\"tag\"+str(t) for t in range(1,7)])\ndf_tags = df_tags.fillna(0).astype(\"int16\")\ndf_tags = df_qs[[\"question_id\"]].join(df_tags)\ndf_qs[\"tag_rel\"] = df_qs.tags.apply(lambda x: 0 if x is np.nan else sum([tags_num[t] for t in x.split(\" \")]))\ndf_qs[\"tag_tfidf_sum\"] = df_qs.tags.apply(lambda x: 0 if x is np.nan else sum([tags_tfidf[t] for t in x.split(\" \")]))\n#df_qs[\"tag_tfidf_max\"] = df_qs.tags.apply(lambda x: 0 if x is np.nan else max([tags_tfidf[t] for t in x.split(\" \")]))\n#df_qs[\"tag_tfidf_min\"] = df_qs.tags.apply(lambda x: 0 if x is np.nan else min([tags_tfidf[t] for t in x.split(\" \")]))\ndf_qs[\"num_tag\"] = df_qs.tags.apply(lambda x: 0 if x is np.nan else len(x.split(\" \")))\ndf_qs.drop([\"correct_answer\",\"tags\"], axis=1, inplace=True)\ndf_qs = df_qs.merge(df_tags, on=\"question_id\")\ndf_qs = reduce_mem_usage(df_qs)\n\ndel df_tags, tags_num, tags_tfidf\ngc.collect()\n\nprint(df_lecs.shape, df_qs.shape)\ndisplay(df_lecs.head(2))\ndisplay(df_qs.head(2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merge with question dataframe\ntrain = train.merge(df_qs[[\"question_id\",\"bundle_id\",\"part\",\"tag1\"]], on='question_id', how='left')\ntrain = reduce_mem_usage(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate answering stats for each question and explanation\ndf_que_exp_rate = train[[\"question_id\",\"prior_question_had_explanation\",\"answered_correctly\"]].copy()\ndf_que_exp_rate[\"cnt\"] = 1\ndf_que_exp_rate = df_que_exp_rate.groupby([\"question_id\",\"prior_question_had_explanation\"], as_index=False).agg({\"answered_correctly\":\"mean\", \"cnt\":\"count\"})\ndf_que_exp_rate.rename(columns={\"answered_correctly\":\"question_rate\"}, inplace=True)\ndf_que_exp_rate = change_rate_with_num(df_que_exp_rate, \"question_rate\", \"cnt\", n1=20)\ndf_que_exp_rate = change_rate_with_num(df_que_exp_rate, \"question_rate\", \"cnt\", method=\"bundle\", n2=5)\ndf_que_exp_rate.drop(\"cnt\", axis=1, inplace=True)\n# Merge with base dataframe\ndf_que_exp_base = pd.DataFrame({\"question_id\": np.repeat(np.array(df_qs.question_id), 2),\n \"prior_question_had_explanation\": np.tile([0,1], df_qs.shape[0])})\ndf_que_exp_rate = df_que_exp_base.merge(df_que_exp_rate, on=[\"question_id\",\"prior_question_had_explanation\"], how=\"left\")\ndf_que_exp_rate.fillna(0.5, inplace=True)\ndf_que_exp_rate = add_que_rate_class(df_que_exp_rate)\ndf_que_exp_rate.question_rate = df_que_exp_rate.question_rate.astype(\"float32\")\ndf_que_exp_rate.question_rate_class = df_que_exp_rate.question_rate_class.astype(\"int8\")\ndf_que_exp_rate = reduce_mem_usage(df_que_exp_rate)\n\nprint(df_que_exp_rate.shape)\ndf_que_exp_rate.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate answering stats for each question and bundle id\ndf = train[[\"user_id\",\"question_id\",\"bundle_id\",\"user_answer\",\"answered_correctly\"]].copy()\ndf_question_rate = df_qs[[\"question_id\",\"bundle_id\"]].copy()\nfor c in [\"question_id\", \"bundle_id\"]:\n c_name = c[:-3]\n df[c_name+\"_kurt\"] = df.user_answer\n df[c_name+\"_skew\"] = df.user_answer\n df_kurt = df.groupby(c, as_index=False)[c_name+\"_kurt\"].apply(pd.DataFrame.kurt)\n df_rate = df.groupby(c, as_index=False).agg({\"answered_correctly\": \"mean\",\n \"user_answer\" : \"std\",\n c_name+\"_skew\" : \"skew\"})\n df_rate.rename(columns={\"answered_correctly\":c_name+\"_rate\",\n \"user_answer\" :c_name+\"_std\"}, inplace=True)\n df_rate = df_rate.merge(df_kurt, on=c)\n df_question_rate = df_question_rate.merge(df_rate, on=c, how=\"left\")\n df_question_rate.fillna(0, inplace=True)\n \n# Rate 0 question will be replaced to second min value\nmin2_val = sorted(df_question_rate.question_rate.unique())[1]\nins_row = df_question_rate[df_question_rate.question_rate == min2_val].iloc[:,2:]\ndf_question_rate.loc[df_question_rate.question_rate == 0, ins_row.columns] = ins_row.values\n\n# Add deviation calculated by mean and std\ndf_question_rate[\"deviation_p\"] = df_question_rate[[\"question_rate\",\"question_std\"]].apply(lambda x: get_deviation_value(x[0],x[1], \"p\"), axis=1)\ndf_question_rate[\"deviation_m\"] = df_question_rate[[\"question_rate\",\"question_std\"]].apply(lambda x: get_deviation_value(x[0],x[1], \"m\"), axis=1)\n# Add question rate class\ndf_question_rate = add_que_rate_class(df_question_rate)\ndf_question_rate.bundle_id = df_question_rate.bundle_id.astype(\"int16\")\ndf_question_rate.question_rate_class = df_question_rate.question_rate_class.astype(\"int8\")\ndf_question_rate = reduce_mem_usage(df_question_rate)\n\n# Calculate answering stats for each part\ndf_part_rate = train.groupby(\"part\", as_index=False).answered_correctly.mean()\ndf_part_rate.rename(columns={\"answered_correctly\": \"part_rate\"}, inplace=True)\ndf_part_rate = reduce_mem_usage(df_part_rate)\n\ndel df, df_kurt, df_rate, min2_val, ins_row\ngc.collect()\n\nprint(df_question_rate.shape, df_part_rate.shape)\ndisplay(df_question_rate.head(2))\ndisplay(df_part_rate.head(2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_question_rate = df_question_rate.merge(df_que_exp_rate, on=\"question_id\", suffixes=(\"\", \"_exp\"))\n\ndel df_que_exp_rate\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(13, 5))\ndf_question_rate.question_rate.hist(ax=axes[0,0])\ndf_question_rate.question_std .hist(ax=axes[0,1])\ndf_question_rate.question_skew.hist(ax=axes[0,2])\ndf_question_rate.question_kurt.hist(ax=axes[1,0])\ndf_question_rate.deviation_p .hist(ax=axes[1,1])\ndf_part_rate.part_rate .plot(ax=axes[1,2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merge with DataFrame for caluculating answer rate and deviation\ntrain = train.merge(df_question_rate[[\"question_id\",\"prior_question_had_explanation\",\"question_rate_exp\",\"deviation_p\",\"deviation_m\"]],\n on=[\"question_id\",\"prior_question_had_explanation\"])\n\n# Sort to original order\ntrain.sort_values(by=\"row_id\", inplace=True)\ntrain.reset_index(drop=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate correct answer rate and deviation for each user\nans = np.array(train.answered_correctly, dtype=\"int16\")\nnum = np.ones (train.shape[0], dtype=\"int16\")\nexp = np.array(train.prior_question_had_explanation, dtype=\"int16\")\ndevp = ans * np.array(train.deviation_p, dtype=\"float64\")\ndevm = devp + np.where(ans==0, -1, 0) * np.array(train.deviation_m, dtype=\"float64\")\n\nans_cumsum = intervaled_cumsum(ans, sizes_qcumsum).astype(\"int16\")\nnum_cumsum = intervaled_cumsum(num, sizes_qcumsum).astype(\"int16\")\nexp_cumsum = intervaled_cumsum(exp, sizes_qcumsum).astype(\"int16\")\ndevp_cumsum = intervaled_cumsum(devp, sizes_qcumsum).astype(\"float32\")\ndevm_cumsum = intervaled_cumsum(devm, sizes_qcumsum).astype(\"float32\")\nans_rate = (ans_cumsum / num_cumsum).astype(\"float32\")\nstudy_rate = (exp_cumsum / shift_to_prior(num_cumsum, sizes_qcumsum)).astype(\"float32\")\nstudy_rate[np.isnan(study_rate)] = 0\nstudy_rate[np.isinf(study_rate)] = 0\n\n# Store for history and training\nhist_ans_cumsum, ans_cumsum = get_hist_and_training_data(ans_cumsum)\nhist_num_cumsum, num_cumsum = get_hist_and_training_data(num_cumsum)\nhist_exp_cumsum, exp_cumsum = get_hist_and_training_data(exp_cumsum)\nhist_devp_cumsum, devp_cumsum = get_hist_and_training_data(devp_cumsum)\nhist_devm_cumsum, devm_cumsum = get_hist_and_training_data(devm_cumsum)\nhist_ans_rate, ans_rate = get_hist_and_training_data(ans_rate)\nhist_study_rate, _ = get_hist_and_training_data(study_rate)\n\ntrain.drop([\"deviation_p\",\"deviation_m\"], axis=1, inplace=True)\ndel ans, num, exp, devp, devm\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Make column names to merge with train later\nnum_part_cols = [\"num_part\"+str(p) for p in range(1,8)]\nans_part_cols = [\"ans_part\"+str(p) for p in range(1,8)]\npart_rate_cols = [\"part\"+str(p)+\"_rate\" for p in range(1,8)]\npart_cols = ans_part_cols + num_part_cols\n\n# Make cumsum array for each part\nar_part_num = np.array(pd.get_dummies(train.part))\nar_part_ans = ar_part_num * np.array(train.answered_correctly).reshape(-1,1)\nar_part = np.hstack([ar_part_ans, ar_part_num]).astype(\"int16\")\ndel ar_part_num, ar_part_ans\ngc.collect()\n\n# Calculate correct answer rate for each part\nfor i in range(ar_part.shape[1]):\n ar_part[:,i] = intervaled_cumsum(ar_part[:,i], sizes_qcumsum)\nar_part_rate = (ar_part[:,:7] / ar_part[:,7:]).astype(\"float32\")\nnp.nan_to_num(ar_part_rate, copy=False)\n\n# Store for history and training\nhist_ar_part_rate, _ = get_hist_and_training_data(ar_part_rate)\nhist_ar_part, ar_part = get_hist_and_training_data(ar_part)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate answering ability for each question difficulty\nabi_cols = []\nabi_rate_cols = []\nfor sta, end in zip([0.0] + list(np.arange(0.3, 1.0, 0.1)),\n [0.3] + list(np.arange(0.4, 1.1, 0.1))):\n sta = round(sta,1)\n end = round(end,1)\n abi_cols += [\"ans_abi\" +str(int(sta*10)), \"num_abi\"+str(int(sta*10))]\n abi_rate_cols += [\"abi_rate\"+str(int(sta*10))]\n end = 1.1 if end == 1 else end\n print(\"Target question rate is from %s to %s\" % (sta, end))\n \n # Replace the ranged data to 0\n df_ability = train[[\"answered_correctly\",\"question_rate_exp\"]].copy()\n df_ability[\"num\"] = 1\n df_ability.loc[(df_ability[\"question_rate_exp\"] <= sta)|(end < df_ability[\"question_rate_exp\"]), [\"answered_correctly\",\"num\"]] = 0\n abians = np.array(df_ability.answered_correctly, dtype=\"int16\")\n abinum = np.array(df_ability.num, dtype=\"int16\")\n del df_ability\n gc. collect()\n \n # Calculate cumsum\n abians_cumsum = intervaled_cumsum(abians, sizes_qcumsum).astype(\"int16\")\n abinum_cumsum = intervaled_cumsum(abinum, sizes_qcumsum).astype(\"int16\")\n del abians, abinum\n gc. collect()\n \n # Store for history and training\n abians_num = np.array([abians_cumsum, abinum_cumsum]).T\n hist_abians_num, abians_num = get_hist_and_training_data(abians_num)\n if sta == 0:\n hist_ar_ability = hist_abians_num\n ar_ability = abians_num\n else:\n hist_ar_ability = np.hstack([hist_ar_ability, hist_abians_num])\n ar_ability = np.hstack([ar_ability, abians_num])\n del abians_cumsum, abinum_cumsum, abians_num, hist_abians_num\n gc. collect()\n \n# Sort to ans -> num order\nabi_cols = sorted(abi_cols)\nabi_rate_cols = sorted(abi_rate_cols)\nhist_ar_ability = np.hstack([hist_ar_ability[:,::2], hist_ar_ability[:,1::2]])\nar_ability = np.hstack([ar_ability[:,::2], ar_ability[:,1::2]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make correct answer rate classes(i.e. class 0 is based on from 0 to 0.1)\nar_class = np.hstack([unique_user_ids.reshape(-1,1),\n hist_num_cumsum.reshape(-1,1),\n hist_ans_rate .reshape(-1,1),\n hist_study_rate.reshape(-1,1),\n hist_ar_part[:,7:],\n hist_ar_part_rate])\ndf_class = pd.DataFrame(ar_class, columns=[\"user_id\",\"num_que\",\"ans_rate\",\"study_rate\"]+num_part_cols+part_rate_cols)\ndel ar_class, hist_ans_rate, hist_study_rate, hist_ar_part_rate\ngc.collect()\n\n# The rates that the number of answering is small will replace to Nan\ndf_class.loc[df_class.num_que < 5, [\"ans_rate\",\"study_rate\"]] = np.nan\nfor p in range(1,8):\n df_class.loc[df_class[\"num_part\"+str(p)] < 5, \"part\"+str(p)+\"_rate\"] = np.nan\n\n# Make classes and calculate correct answer rate based on them\ndf_class = cut(df_class, \"ans_rate\", \"ans_class\")\ndf_class = cut(df_class, \"study_rate\", \"study_class\")\nfor p in range(1,8):\n df_class = cut(df_class, \"part\"+str(p)+\"_rate\", \"part\"+str(p)+\"_class\")\n \n# Save memory\ndf_class.user_id = df_class.user_id.astype(\"int32\")\ndf_class.ans_rate = df_class.ans_rate.astype(\"float32\")\ndf_class.study_rate = df_class.study_rate.astype(\"float32\")\ndf_class = df_class[[\"user_id\"] + [c for c in df_class.columns if -1 < c.find(\"class\")]]\ndf_class = reduce_mem_usage(df_class)\n\nprint(df_class.shape)\ndf_class.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndef make_rate_class(df, m_col):\n df_rc = df.groupby([m_col, \"question_id\"], as_index=False).agg(aggs)\n df_rc = df_rc.loc[df_rc[m_col] != 99].copy()\n df_rc = change_rate_with_num(df_rc, \"ans_rate_class\", \"ans_cnt\")\n df_rc.drop(\"ans_cnt\", axis=1, inplace=True)\n return reduce_mem_usage(df_rc)\n\ndf_ans_class = train[[\"user_id\",\"question_id\",\"answered_correctly\"]].merge(df_class, on=\"user_id\")\ndf_ans_class[\"ans_cnt\"] = df_ans_class.answered_correctly\ndf_ans_class.rename(columns={\"answered_correctly\": \"ans_rate_class\"}, inplace=True)\ndel df_class\ngc.collect()\n\n# Make dataframe that grouped by each class feature\n# Answer rate that the number of answering is so small will be changed to 0.3/0.5/0.7\ndf_parts = []\naggs = {\"ans_rate_class\":\"mean\", \"ans_cnt\":\"count\"}\ndf_ans_rate_class = make_rate_class(df_ans_class, \"ans_class\")\ndf_study_rate_class = make_rate_class(df_ans_class, \"study_class\")\ndf_study_rate_class.rename(columns={\"ans_rate_class\":\"study_rate_class\"}, inplace=True)\nfor p in range(1,8):\n df_part_rate_class = make_rate_class(df_ans_class, f\"part{p}_class\")\n df_part_rate_class.rename(columns={\"ans_rate_class\":f\"part{p}_rate_class\"}, inplace=True)\n df_parts.append(df_part_rate_class)\n \n# Merge with all dataframes\nlo = [\"base_class\",\"question_id\"]\ndf_ans_class = pd.DataFrame({\"base_class\" : np.repeat(np.arange(0,10), df_qs.question_id.nunique()),\n \"question_id\": np.tile (df_qs.question_id.unique(), 10)})\ndf_ans_class = df_ans_class.merge(df_ans_rate_class, left_on=lo, right_on=[\"ans_class\", \"question_id\"], how=\"left\")\ndf_ans_class.drop(\"ans_class\", axis=1, inplace=True)\ndel df_ans_rate_class\n\ndf_ans_class = df_ans_class.merge(df_study_rate_class, left_on=lo, right_on=[\"study_class\", \"question_id\"], how=\"left\")\ndf_ans_class.drop(\"study_class\", axis=1, inplace=True)\ndel df_study_rate_class\n\nfor p, df in enumerate(df_parts):\n df_ans_class = df_ans_class.merge(df, left_on=lo, right_on=[f\"part{p+1}_class\",\"question_id\"], how=\"left\")\n df_ans_class.drop(f\"part{p+1}_class\", axis=1, inplace=True)\ndel df_parts\n\n# Modify\ndf_ans_class.fillna(0.5, inplace=True)\ndf_ans_class = reduce_mem_usage(df_ans_class)\n\ngc.collect()\n\nprint(df_ans_class.shape)\ndf_ans_class.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c = 5; r = 2; u = 0\nfig, axes = plt.subplots(nrows=r, ncols=c, figsize=(13, 6))\n\nfor _r in range(r):\n for _c in range(c):\n if 9 < u: continue\n df_ans_class.loc[df_ans_class.base_class==u].ans_rate_class.hist(ax=axes[_r,_c])\n u += 1\ndel fig, axes\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c = 5; r = 2; u = 0\nfig, axes = plt.subplots(nrows=r, ncols=c, figsize=(13, 6))\n\nfor _r in range(r):\n for _c in range(c):\n if 9 < u: continue\n df_ans_class.loc[df_ans_class.base_class==u].study_rate_class.hist(ax=axes[_r,_c])\n u += 1\ndel fig, axes\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c = 5; r = 2; u = 0\nfig, axes = plt.subplots(nrows=r, ncols=c, figsize=(13, 6))\n\nfor _r in range(r):\n for _c in range(c):\n if 9 < u: continue\n df_ans_class.loc[df_ans_class.base_class==u].part7_rate_class.hist(ax=axes[_r,_c])\n u += 1\ndel fig, axes\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Saved memory concat\nadd_a_cols = [c for c in train_lec.columns if c not in train.columns]\nadd_b_cols = [c for c in train.columns if c not in train_lec.columns]\nfor c in add_a_cols: train[c] = 0\nfor c in add_b_cols: train_lec[c] = 0\ntrain_lec = train_lec[train.columns]\ntrain_lec = reduce_mem_usage(train_lec)\ntrain = reduce_mem_usage(train)\ntrain = reduce_mem_usage(pd.concat([train, train_lec]))\n\ndel train_lec\ngc.collect()\n\n# Sort to original order\ntrain.sort_values(by=\"row_id\", inplace=True)\ntrain.reset_index(drop=True, inplace=True)\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Store all timestamps for calculating the gap of continuous\nsizes_all_user = np.array(train.user_id.value_counts().sort_index().values, dtype=\"int16\")\nsizes_all_cumsum = sizes_all_user.cumsum()\n\nt_ary = np.array(train[[\"user_id\",\"timestamp\"]])\nd_timestamp = get_user_items_dict(t_ary, sizes_all_cumsum)\nfor key, item in d_timestamp.items():\n d_timestamp[key] = np.unique(item)\n\ndel t_ary\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove lecture rows and some columns\ntrain.drop([\"bundle_id\",\"question_rate_exp\",\"user_answer\"], axis=1, inplace=True)\ntrain.drop(train.loc[train.content_type_id==1].index, inplace=True)\ntrain.drop([\"content_type_id\"], axis=1, inplace=True)\ntrain.reset_index(drop=True, inplace=True)\n\n# Make Listening(0) and Reading(1)\ntrain[\"LorR\"] = 0\ntrain.loc[train.part.isin([5,6,7]), \"LorR\"] = 1\n\ntrain = reduce_mem_usage(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Store latest data for prediction of real world data\nar_history = np.hstack([unique_user_ids.reshape(-1,1),\n hist_ans_cumsum.reshape(-1,1),\n hist_num_cumsum.reshape(-1,1),\n hist_exp_cumsum.reshape(-1,1),\n hist_devp_cumsum.reshape(-1,1),\n hist_devm_cumsum.reshape(-1,1),\n hist_ar_etmie_rate.reshape(-1,1),\n hist_ar_part,\n hist_ar_ability])\ndf_history = pd.DataFrame(ar_history, columns=[\"user_id\",\"num_ans\",\"num_que\",\"num_exp\",\"num_devp\",\"num_devm\",\"num_etime\"]+part_cols+abi_cols)\nfor c in df_history.columns:\n if c not in [\"num_devp\",\"num_devm\",\"num_etime\"]:\n df_history[c] = df_history[c].astype(int)\ndf_history = reduce_mem_usage(df_history)\ndf_history.sort_values(by=\"user_id\", inplace=True)\ndf_history.reset_index(drop=True, inplace=True)\n\ndel hist_ans_cumsum, hist_num_cumsum, hist_exp_cumsum, hist_devp_cumsum, hist_devm_cumsum, hist_ar_etmie_rate, hist_ar_part, hist_ar_ability\ngc.collect()\n\nprint(df_history.shape)\ndf_history.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make training data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_attempt_que(u, r, q):\n # u: user_id\n # r: row_id\n # q: question_id\n if u in d.keys():\n ua = d[u]\n return np.count_nonzero((ua[:,0] < r) * (ua[:,1] == q))\n return 0\n\ndef get_attempt_curt_part(u, r, p):\n # u: user_id\n # r: row_id\n # p: part\n if u in d.keys():\n ua = d[u]\n return np.count_nonzero(ua[ua[:,0]= 3:\n return ul[l]\n elif ul.shape[0] == 2 and l != -3:\n return ul[l]\n elif ul.shape[0] == 1 and l == -1:\n return ul[l]\n return 0\n\ndef get_noplay_days(u, t):\n # u: user_id\n # t: normalized timestamp\n if u in d_timestamp.keys():\n ul = d_timestamp[u] / (1000*3600)\n ul = ul[ul <= t]\n if 1 < ul.shape[0]:\n di = np.diff(ul)\n return np.where(di<24,0,di).sum()\n return 0\n\ndef get_attempt_thedaybefore(u, t, d):\n # u: user_id\n # t: normalized timestamp\n # d: days(7 is 1 week ago, 1 is 24 hours ago, 0.5 is 12 hours ago)\n if u in d_timestamp.keys():\n ul = d_timestamp[u] / (1000*3600)\n return np.count_nonzero((t-d <= ul) * (ul < t))\n return 0\n\ndef cut_and_merge_with_class(df, df_class, t_col, c_col):\n df = df.copy()\n df[\"base_class\"] = pd.cut(df[t_col], [-0.1, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.1], labels=False)\n return df.merge(df_class[[\"question_id\",\"base_class\",c_col]], on=[\"question_id\",\"base_class\"], sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = train[train.user_id.isin(ext_user)].copy()\ndata.reset_index(drop=True, inplace=True)\n\n# Store all user id for calculating attempt\nq_ary = np.array(train[[\"user_id\",\"row_id\",\"question_id\",\"part\",\"tag1\",\"answered_correctly\"]])\nq_ary = q_ary[np.argsort(q_ary[:,1])]\ndel train\ngc.collect()\n\ntrain = data.copy()\nsizes_train_user = np.array(train.user_id.value_counts().sort_index().values, dtype=\"int16\")\nsizes_train_cumsum = sizes_train_user.cumsum()\ndel data\ngc.collect()\n\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# CV strategy\ndef rand_time(max_time_stamp):\n interval = MAX_TIME_STAMP - max_time_stamp\n rand_time_stamp = random.randint(0,interval)\n return rand_time_stamp\n\nmax_timestamp_u = train[['user_id','timestamp']].groupby(['user_id']).agg(['max']).reset_index()\nmax_timestamp_u.columns = ['user_id', 'max_time_stamp']\nMAX_TIME_STAMP = max_timestamp_u.max_time_stamp.max()\n\nmax_timestamp_u['rand_time_stamp'] = max_timestamp_u.max_time_stamp.apply(lambda x: rand_time(x))\ntrain = train.merge(max_timestamp_u, on='user_id')\ntrain['viretual_time_stamp'] = train.timestamp + train['rand_time_stamp']\ntrain.sort_values(['viretual_time_stamp', 'row_id'], inplace=True)\ntrain.reset_index(drop=True, inplace=True)\n\ntrain_num = int(train.shape[0] * TRAIN_RATE)\nif TUNE:\n tune_num = int(train.shape[0] * TUNE_RATE)\n test_num = train.shape[0] - (train_num + tune_num)\nelse:\n test_num = train.shape[0] - train_num\n tune_num = 0\nprint(train_num, test_num, tune_num)\n\n# Add CV group\ntrain[\"cv_group\"] = 9\nsta = 0\nfor c in range(CV):\n end = int(train_num/CV)*(c+1) if (c+1) != CV else train_num\n train.loc[sta:end, \"cv_group\"] = c\n print(\"CV:%s -- Start with %07d and end with %07d. Num of user is %s and mean correct answer rate is %f\" % (c, sta, end, train.loc[train.cv_group==c].user_id.nunique(), train.loc[train.cv_group==c].answered_correctly.mean()))\n sta = end\nif TUNE:\n train.loc[train_num+1:train_num+tune_num, \"cv_group\"] = 5\n train.loc[train_num+tune_num+1:, \"cv_group\"] = 4\nelse:\n train.loc[train_num+1:train_num+test_num, \"cv_group\"] = 4\n\ntrain.sort_values('row_id', inplace=True)\ntrain.reset_index(drop=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Add correct answer rate after shifting to prior\ndevp_rate = (devp_cumsum / num_cumsum).astype(\"float32\")\ndevm_rate = (devm_cumsum / num_cumsum).astype(\"float32\")\ntrain[\"ans_rate\"] = shift_to_prior(ans_rate, sizes_train_cumsum).astype(\"float32\")\ntrain[\"devp_rate\"] = shift_to_prior(devp_rate, sizes_train_cumsum).astype(\"float32\")\ntrain[\"devm_rate\"] = shift_to_prior(devm_rate, sizes_train_cumsum).astype(\"float32\")\ntrain[\"num_que\"] = shift_to_prior(num_cumsum, sizes_train_cumsum).astype(\"int16\")\ntrain[\"num_exp\"] = exp_cumsum.astype(\"int16\")\ntrain[\"study_rate\"] = (train.num_exp / train.num_que).replace([np.inf,-np.inf], np.nan).fillna(0).astype(\"float32\")\n\ntrain = reduce_mem_usage(train)\n\ndel ans_rate, devp_rate, devm_rate, num_cumsum, exp_cumsum\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate correct answer rate for each part\nar_part_rate = ar_part[:,:7] / ar_part[:,7:]\nnp.nan_to_num(ar_part_rate, copy=False)\nar_part_rate = ar_part_rate.astype(\"float32\")\n\n# Shift to prior\nfor i in range(ar_part_rate.shape[1]):\n ar_part_rate[:,i] = shift_to_prior(ar_part_rate[:,i], sizes_train_cumsum)\nfor i in range(ar_part.shape[1]):\n ar_part[:,i] = shift_to_prior(ar_part[:,i], sizes_train_cumsum)\n\nar_part_rate.shape, ar_part.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate answering ability for each question difficulty\nar_ability_rate = ar_ability[:,:8] / ar_ability[:,8:]\nnp.nan_to_num(ar_ability_rate, copy=False)\nar_ability_rate = ar_ability_rate.astype(\"float32\")\n\n# Shift to prior\nfor i in range(ar_ability_rate.shape[1]):\n ar_ability_rate[:,i] = shift_to_prior(ar_ability_rate[:,i], sizes_train_cumsum)\n\nar_ability_rate.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.join(pd.DataFrame(ar_part_rate, columns=part_rate_cols))\ntrain = train.join(pd.DataFrame(ar_part[:,7:], columns=num_part_cols))\ntrain = train.join(pd.DataFrame(ar_ability_rate, columns=abi_rate_cols))\ntrain = train.join(pd.DataFrame(ar_etmie_rate, columns=[\"etime_rate\"]))\n\ntrain.sort_values(\"row_id\", inplace=True)\ntrain.reset_index(drop=True, inplace=True)\n\ndel ar_part_rate, ar_part, ar_ability_rate\ngc.collect()\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nprint(\"Calculating the gap of continuous\")\ntrain[\"lagtime1\"] = (train[[\"user_id\",\"timestamp\"]].parallel_apply(lambda x: get_lagtime(x[0], x[1], -1), axis=1) / (1000*3600)).astype(\"float32\")\ntrain[\"lagtime2\"] = (train[[\"user_id\",\"timestamp\"]].parallel_apply(lambda x: get_lagtime(x[0], x[1], -2), axis=1) / (1000*3600)).astype(\"float32\")\ntrain[\"lagtime3\"] = (train[[\"user_id\",\"timestamp\"]].parallel_apply(lambda x: get_lagtime(x[0], x[1], -3), axis=1) / (1000*3600)).astype(\"float32\")\ntrain[\"timestamp\"] = (train.timestamp / (1000*3600)).astype(\"float32\")\ntrain[\"lagtime1\"] = (train.timestamp - train.lagtime1).astype(\"float32\")\ntrain[\"lagtime2\"] = (train.timestamp - train.lagtime2).astype(\"float32\")\ntrain[\"lagtime3\"] = (train.timestamp - train.lagtime3).astype(\"float32\")\n\n#print(\"Calculating the number of attempt within 12 hours/24 hours/1 week\")\n#train[\"attempt_12hours\"] = train[[\"user_id\",\"timestamp\"]].apply(lambda x: get_attempt_thedaybefore(x[0], x[1], 0.5), axis=1)\n#train[\"attempt_24hours\"] = train[[\"user_id\",\"timestamp\"]].apply(lambda x: get_attempt_thedaybefore(x[0], x[1], 1), axis=1)\n#train[\"attempt_1week\"] = train[[\"user_id\",\"timestamp\"]].apply(lambda x: get_attempt_thedaybefore(x[0], x[1], 7), axis=1)\n\nprint(\"Calculating total of no play days\")\ntrain[\"noplay_days\"] = (train[[\"user_id\",\"timestamp\"]].parallel_apply(lambda x: get_noplay_days(x[0], x[1]), axis=1)).astype(\"float32\")\n\n# Add elapsed days\ntrain[\"elapsed_days\"] = (train.timestamp / 24).astype(\"int32\")\n\n# Add rate of timestamp\ntrain[\"noplay_rate\"] = (train.noplay_days / train.num_que).fillna(0).astype(\"float32\")\ntrain[\"timestamp_rate\"] = (train.timestamp / train.num_que).fillna(0).astype(\"float32\")\ntrain[\"elapsed_rate\"] = (train.elapsed_days / train.num_que).fillna(0).astype(\"float32\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nd = get_user_items_dict(q_ary, sizes_qcumsum)\n \nprint(\"Calculating the number of attempt against the same question\")\ntrain[\"attempt_que\"] = train[[\"user_id\",\"row_id\",\"question_id\"]].parallel_apply(lambda x: get_attempt_que(x[0], x[1], x[2]), axis=1)\n\ntags_cols = [\"tag\"+str(t) for t in range(1,7)]\nprint(\"Calculating the number of attempt against the same tag\")\ntrain[\"attempt_tags\"] = train[[\"user_id\",\"row_id\",\"tag1\"]].parallel_apply(lambda x: get_attempt_tags(x[0], x[1], x[2]), axis=1)\n\nprint(\"Calculating the current correct answer rate\")\ntrain[\"attempt_curtar\"] = train[[\"user_id\",\"row_id\"]].parallel_apply(lambda x: get_attempt_curt_ans_rate(x[0], x[1]), axis=1)\n\n#print(\"Calculating the number of attempt against the same part\")\n#train[\"attempt_curtp\"] = train[[\"user_id\",\"row_id\",\"part\"]].apply(lambda x: get_attempt_curt_part(x[0], x[1], x[2]), axis=1)\n\n#print(\"Calculating the number of attempt against the same question format\")\n#train[\"attempt_curtLR\"] = train[[\"user_id\",\"row_id\",\"LorR\"]].apply(lambda x: get_attempt_curt_LR(x[0], x[1], x[2]), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain = cut_and_merge_with_class(train, df_ans_class, \"ans_rate\", \"ans_rate_class\")\ntrain = cut_and_merge_with_class(train, df_ans_class, \"study_rate\", \"study_rate_class\")\nfor p in range(1,8):\n train = cut_and_merge_with_class(train, df_ans_class, f\"part{p}_rate\", f\"part{p}_rate_class\")\ntrain.drop(\"base_class\", axis=1, inplace=True)\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(df_qs.drop([\"part\",\"tag1\"], axis=1), on=\"question_id\")\ntrain = train.merge(df_question_rate, on=[\"question_id\",\"bundle_id\",\"prior_question_had_explanation\"])\ntrain = train.merge(df_etime_rate, on=\"question_id\")\ntrain = train.merge(df_part_rate, on=\"part\")\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_cols = [\"ans_rate_class\",\"study_rate_class\"] + [\"part\"+str(p)+\"_rate_class\" for p in range(1,8)]\nn_cols = [\"num_que\", \"num_que\"] + num_part_cols\nfor t, n in zip(t_cols, n_cols):\n train = change_rate_with_num(train, t, n, method=\"respectively\")\n train = change_rate_with_num(train, t, n, method=\"bundle\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make chance of making a mistake features\ntrain[\"prob_mistake_que\"] = train.num_que.apply(lambda x: 0.6*x**(-0.2) if x!=0 else 0.6)\npart_mistake_cols = []\nfor p in range(1,8):\n part_mistake_cols.append(\"prob_mistake_part\"+str(p))\n train[\"prob_mistake_part\"+str(p)] = train[\"num_part\"+str(p)].apply(lambda x: 0.6*x**(-0.2) if x!=0 else 0.6)\nall_prob_mistake = np.array(train[part_mistake_cols])\ntrain[\"prob_mistake_part_curt\"] = np.nanmax(np.identity(8) [train.part][:,1:]*all_prob_mistake, axis=1)\n \n# Make current part/part rate/ability features\nall_part_rate = np.array(train[[\"part1_rate\",\"part2_rate\",\"part3_rate\",\"part4_rate\",\"part5_rate\",\"part6_rate\",\"part7_rate\"]])\nall_part_ratec = np.array(train[[\"part1_rate_class\",\"part2_rate_class\",\"part3_rate_class\",\"part4_rate_class\",\"part5_rate_class\",\"part6_rate_class\",\"part7_rate_class\"]])\nall_abi_rate = np.array(train[[\"abi_rate0\",\"abi_rate3\",\"abi_rate4\",\"abi_rate5\",\"abi_rate6\",\"abi_rate7\",\"abi_rate8\",\"abi_rate9\"]])\ntrain[\"part_rate_curt\"] = np.nanmax(np.identity(8) [train.part][:,1:]*all_part_rate, axis=1)\ntrain[\"part_ratec_curt\"] = np.nanmax(np.identity(8) [train.part][:,1:]*all_part_ratec, axis=1)\ntrain[\"abi_rate_curt\"] = np.nanmax(np.identity(10)[train.question_rate_class_exp][:,[0,3,4,5,6,7,8,9]]*all_abi_rate, axis=1)\ntrain[\"part_ratec_mean\"] = np.nan_to_num(all_part_ratec).sum(1) / 7\n\ndel all_part_rate, all_part_ratec, all_abi_rate\ngc.collect()\n\n# Make harmonic features\n# Comparison between:\n# - a correct answer rate against all questions and a correct answer rate against the part\n# - a correct answer rate of someone who looks like you and a correct answer rate of the question\n# - a correct answer rate of someone who looks like you and a correct answer rate against the part\n# - a correct answer rate against the part and a correct answer rate of the part\n# - a correct answer rate of someone who looks like you and a correct answer rate of the question\n# - a correct answer rate of the part and a correct answer rate of the question\n# - a correct answer rate against questions difficulty and a correct answer rate of the question\ntrain[\"harmonic_ar_prc\"] = 2 * (train.ans_rate * train.part_rate_curt) / (train.ans_rate + train.part_rate_curt)\ntrain[\"harmonic_arc_qr\"] = 2 * (train.ans_rate_class * train.question_rate_exp) / (train.ans_rate_class + train.question_rate_exp)\ntrain[\"harmonic_arc_prcc\"] = 2 * (train.ans_rate_class * train.part_ratec_curt) / (train.ans_rate_class + train.part_ratec_curt)\ntrain[\"harmonic_prc_pr\"] = 2 * (train.part_rate_curt * train.part_rate) / (train.part_rate_curt + train.part_rate)\ntrain[\"harmonic_prcc_qr\"] = 2 * (train.part_ratec_curt * train.question_rate_exp) / (train.part_ratec_curt + train.question_rate_exp)\ntrain[\"harmonic_qr_pr\"] = 2 * (train.part_rate * train.question_rate_exp) / (train.part_rate + train.question_rate_exp)\ntrain[\"harmonic_abrc_qr\"] = 2 * (train.abi_rate_curt * train.question_rate_exp) / (train.abi_rate_curt + train.question_rate_exp)\ntrain[\"harmonic_arc_src\"] = 2 * (train.ans_rate_class * train.study_rate_class) / (train.ans_rate_class + train.study_rate_class)\ntrain[\"harmonic_prcc_src\"] = 2 * (train.part_ratec_curt * train.study_rate_class) / (train.part_ratec_curt + train.study_rate_class)\ntrain[\"harmonic_src_qr\"] = 2 * (train.study_rate_class * train.question_rate_exp) / (train.study_rate_class + train.question_rate_exp)\n\n# Make rate features\ntrain[\"rate_arc_prcc\"] = train.ans_rate_class / train.part_ratec_curt\ntrain[\"rate_arc_src\"] = train.ans_rate_class / train.study_rate_class\ntrain[\"rate_prcc_src\"] = train.part_ratec_curt / train.study_rate_class\ntrain[\"rate_arc_qr\"] = train.ans_rate_class / train.question_rate_exp\ntrain[\"rate_qr_pr\"] = train.part_rate / train.question_rate_exp\ntrain[\"rate_prcc_prccm\"] = train.part_ratec_curt / train.part_ratec_mean","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.fillna(0, inplace=True)\ntrain.sort_values(by=\"row_id\", inplace=True)\ntrain.reset_index(drop=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot(rs, re, c, t_col):\n u = 0\n for _r in range(rs, re):\n for _c in range(c):\n train.loc[train.user_id==ids[u]].plot(x=\"num_que\", y=t_col, ax=axes[_r,_c])\n u += 1\nc = 4; r = 2\nids = train.user_id.unique()\nrandom.shuffle(ids)\nfig, axes = plt.subplots(nrows=r*3, ncols=c, figsize=(14, 10))\n\nplot(0, r, c, \"ans_rate\")\nplot(r, r*2, c, \"devm_rate\")\nplot(r*2, r*3, c, \"devp_rate\")\n\ndel ids, fig, axes\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(nrows=3, ncols=3, figsize=(13, 6))\ntrain.part_ratec_mean .hist(ax=axes[0,0])\ntrain.harmonic_ar_prc .hist(ax=axes[0,1])\ntrain.harmonic_arc_qr .hist(ax=axes[0,2])\ntrain.harmonic_arc_prcc.hist(ax=axes[1,0])\ntrain.harmonic_prc_pr .hist(ax=axes[1,1])\ntrain.harmonic_prcc_qr .hist(ax=axes[1,2])\ntrain.harmonic_qr_pr .hist(ax=axes[2,0])\ntrain.rate_arc_qr .hist(ax=axes[2,1])\ntrain.rate_qr_pr .hist(ax=axes[2,2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#showStats(train.drop([\"row_id\",\"user_id\"], axis=1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LGBM training "},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"class BaseModel(object):\n \"\"\"\n Base Model Class:\n\n train_df : train pandas dataframe\n test_df : test pandas dataframe\n target : target column name (str)\n features : list of feature names\n categoricals : list of categorical feature names\n n_splits : K in KFold (default is 3)\n cv_method : options are .. KFold, StratifiedKFold, TimeSeriesSplit, GroupKFold, or GroupShuffleSplit\n group : group feature name when GroupKFold or StratifiedGroupKFold are used\n task : options are .. regression, multiclass, or binary\n param : dict of parameter, set that if you already define\n parameter_tuning : bool, only for LGB\n seed : seed (int)\n verbose : bool\n \"\"\"\n\n def __init__(self, train_df, test_df, target, features, \n valid_df=None, tune_df=None, categoricals=[], alpha=0, \n n_splits=3, cv_method=\"KFold\", group=None,\n task=\"regression\", params=None, parameter_tuning=False,\n tuning_type=\"optuna_tuner\", seed=42, verbose=True):\n self.train_df = train_df\n self.test_df = test_df\n self.valid_df = valid_df\n self.tune_df = tune_df\n self.target = target\n self.features = features\n self.n_splits = n_splits\n self.categoricals = categoricals\n self.alpha = alpha\n self.cv_method = cv_method\n self.group = group\n self.task = task\n self.parameter_tuning = parameter_tuning\n self.seed = seed\n self.cv = self.get_cv()\n self.verbose = verbose\n self.tuning_type = tuning_type\n if params is None:\n self.params = self.get_params()\n else:\n self.params = params\n self.y_pred, self.y_valid, self.score, self.models, self.oof, self.y_val, self.fi_df = self.fit()\n\n def train_model(self, train_set, val_set):\n raise NotImplementedError\n\n def get_params(self):\n raise NotImplementedError\n\n def convert_dataset(self, x_train, y_train, x_val, y_val):\n raise NotImplementedError\n\n def calc_metric(self, y_true, y_pred): # this may need to be changed based on the metric of interest\n if self.task in (\"multiclass\",\"nn_multiclass\"):\n preds = np.argmax(y_pred, axis=1) if y_true.shape != y_pred.shape else y_pred\n return f1_score(y_true, preds, average='macro') \n elif self.task == \"binary\":\n return roc_auc_score(y_true, y_pred, average='macro')\n elif self.task in (\"regression\",\"quantile\"):\n return np.sqrt(mean_squared_error(y_true, y_pred))\n \n def get_cv(self):\n if self.cv_method == \"KFold\":\n cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=self.seed)\n return cv.split(self.train_df)\n if self.cv_method == \"StratifiedKFold\":\n cv = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=self.seed)\n return cv.split(self.train_df, self.train_df[self.target])\n if self.cv_method == \"TimeSeriesSplit\":\n cv = TimeSeriesSplit(max_train_size=None, n_splits=self.n_splits)\n return cv.split(self.train_df)\n if self.cv_method == \"GroupKFold\":\n if self.group in self.features:\n self.features.remove(self.group)\n if self.group in self.categoricals:\n self.categoricals.remove(self.group)\n cv = GroupKFold(n_splits=self.n_splits)\n return cv.split(self.train_df[self.features+self.categoricals], self.train_df[self.target], self.train_df[self.group])\n if self.cv_method == \"GroupShuffleSplit\":\n if self.group in self.features:\n self.features.remove(self.group)\n if self.group in self.categoricals:\n self.categoricals.remove(self.group)\n cv = GroupShuffleSplit(n_splits=self.n_splits, random_state=self.seed)\n return cv.split(self.train_df[self.features+self.categoricals], self.train_df[self.target], self.train_df[self.group])\n\n def fit(self):\n # Initialize\n y_vals = np.zeros((self.train_df.shape[0], ))\n if self.task in (\"multiclass\",\"nn_multiclass\"):\n col_len = self.train_df[self.target].nunique()\n else:\n col_len = 1\n oof_pred = np.zeros((self.train_df.shape[0], col_len))\n y_pred = np.zeros((self.test_df.shape[0], col_len))\n y_valid = np.zeros((self.valid_df.shape[0], col_len)) if self.valid_df is not None else None\n models = []\n \n if self.group is not None:\n if self.group in self.features:\n self.features.remove(self.group)\n if self.group in self.categoricals:\n self.categoricals.remove(self.group)\n \n fi = np.zeros((self.n_splits, len(self.features+self.categoricals)))\n if y_valid is not None:\n x_valid = self.valid_df[self.features+self.categoricals].copy()\n del self.valid_df\n gc.collect()\n x_test = self.test_df[self.features+self.categoricals]\n\n # Fitting with out of fold\n for fold, (train_idx, val_idx) in enumerate(self.cv):\n # Prepare train and test dataset\n x_train = self.train_df.iloc[train_idx, :][self.features+self.categoricals]\n y_train = self.train_df.iloc[train_idx, :][self.target]\n x_val = self.train_df.iloc[val_idx, :][self.features+self.categoricals]\n y_val = self.train_df.iloc[val_idx, :][self.target]\n train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)\n del x_train, y_train\n gc.collect()\n \n # Fit model\n model, importance = self.train_model(train_set, val_set)\n fi[fold, :] = importance\n y_vals[val_idx] = y_val\n \n # Get some scores\n oof_pred[val_idx] = model.predict(x_val, num_iteration=model.best_iteration).reshape(oof_pred[val_idx].shape)\n if y_valid is not None:\n y_valid += model.predict(x_valid, num_iteration=model.best_iteration).reshape(y_valid.shape) / self.n_splits\n y_pred += model.predict(x_test, num_iteration=model.best_iteration).reshape(y_pred.shape) / self.n_splits\n \n print('Partial score of fold {} is: {}'.format(fold, self.calc_metric(y_val, oof_pred[val_idx])))\n models.append(model)\n \n del train_set, val_set, x_val, y_val\n gc.collect()\n \n # Create feature importance data frame\n fi_df = pd.DataFrame()\n for n in np.arange(self.n_splits):\n tmp = pd.DataFrame()\n tmp[\"features\"] = self.features+self.categoricals\n tmp[\"importance\"] = fi[n, :]\n tmp[\"fold\"] = n\n fi_df = pd.concat([fi_df, tmp], ignore_index=True)\n gfi = fi_df[[\"features\", \"importance\"]].groupby([\"features\"]).mean().reset_index()\n fi_df = fi_df.merge(gfi, on=\"features\", how=\"left\", suffixes=('', '_mean'))\n \n # Calculate oof score\n loss_score = self.calc_metric(y_vals, oof_pred)\n print('Our oof loss score is: ', loss_score)\n \n return y_pred, y_valid, loss_score, models, oof_pred, y_vals, fi_df\n\n def plot_feature_importance(self, rank_range=[1, 100]):\n fig, ax = plt.subplots(1, 1, figsize=(16, 12))\n sorted_df = self.fi_df.sort_values(by=\"importance_mean\", ascending=False).reset_index()\n sns.barplot(data=sorted_df.iloc[self.n_splits*(rank_range[0]-1) : self.n_splits*rank_range[1]],\n x=\"importance\", y=\"features\", orient='h')\n ax.set_xlabel(\"feature importance\")\n ax.spines['top'].set_visible(False)\n ax.spines['right'].set_visible(False)\n return sorted_df\n \nclass LgbModel(BaseModel):\n \"\"\"\n LGB wrapper\n \"\"\"\n def train_model(self, train_set, val_set):\n verbosity = 100 if self.verbose else 0\n model = lgb.train(self.params, train_set, num_boost_round=3000,\n valid_sets=[train_set, val_set], verbose_eval=verbosity)\n fi = model.feature_importance(importance_type=\"gain\")\n return model, fi\n\n def convert_dataset(self, x_train, y_train, x_val=None, y_val=None):\n train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)\n if x_val is not None:\n val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)\n return train_set, val_set\n return train_set\n\n def get_params(self):\n # Fast fit parameters\n params = {\n 'boosting_type' : \"gbdt\",\n 'objective' : self.task,\n \"subsample\" : 0.4,\n \"subsample_freq\" : 1,\n 'max_depth' : 4,\n 'min_data_in_leaf' : 50,\n 'learning_rate' : 0.05,\n 'early_stopping_rounds' : 100,\n 'bagging_seed' : 11,\n 'random_state' : 42,\n 'verbosity' : -1\n }\n\n # List is here: https://lightgbm.readthedocs.io/en/latest/Parameters.html\n if self.task == \"regression\":\n params[\"metric\"] = \"regression_l2\"\n elif self.task == \"quantile\":\n params[\"metric\"] = \"quantile\"\n params[\"alpha\"] = self.alpha\n elif self.task == \"binary\":\n params[\"metric\"] = \"auc\" # binary_logloss\n elif self.task == \"multiclass\":\n params[\"metric\"] = \"multi_logloss\"\n params[\"num_class\"] = len(self.train_df[self.target].unique())\n \n # Bayesian Optimization by Optuna\n if self.parameter_tuning:\n # Define objective function\n def get_dataset():\n if self.tune_df is not None:\n tune = self.tune_df.copy()\n else:\n tune = self.train_df.copy()\n if self.group is not None:\n train_num = int(tune.shape[0] * 0.7)\n test_num = tune.shape[0] - train_num\n train_x = tune.head(train_num)[self.features+self.categoricals].reset_index(drop=True)\n train_y = tune.head(train_num)[[self.target]].reset_index(drop=True)\n test_x = tune.tail(test_num)[self.features+self.categoricals].reset_index(drop=True)\n test_y = tune.tail(test_num)[[self.target]].reset_index(drop=True)\n else:\n train_x, test_x, train_y, test_y = train_test_split(tune[self.features+self.categoricals], \n tune[self.target], test_size=0.3, random_state=self.seed)\n if self.categoricals != []:\n dtrain = lgb.Dataset(train_x, train_y, categorical_feature=self.categoricals)\n dtest = lgb.Dataset(test_x, test_y, categorical_feature=self.categoricals)\n else:\n dtrain = lgb.Dataset(train_x, train_y)\n dtest = lgb.Dataset(test_x, test_y)\n return dtrain, dtest, test_x, test_y\n \n def objective(trial):\n # Split train and test data\n dtrain, dtest, test_x, test_y = get_dataset()\n # Parameters to be explored\n hyperparams = {'max_depth' : trial.suggest_int('max_depth', 4, 16),\n 'max_bin' : trial.suggest_int('max_bin', 200, 1000),\n 'num_leaves' : trial.suggest_int('num_leaves', 200, 1000),\n 'min_data_in_leaf' : trial.suggest_int('min_data_in_leaf', 1, 300),\n 'min_child_samples' : trial.suggest_int('min_child_samples', 5, 100),\n 'feature_fraction' : trial.suggest_uniform('feature_fraction', 0.2, 1.0),\n 'bagging_fraction' : trial.suggest_uniform('bagging_fraction', 0.2, 1.0),\n 'bagging_freq' : trial.suggest_int('bagging_freq', 0, 7),\n 'lambda_l1' : trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),\n 'lambda_l2' : trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),\n 'early_stopping_rounds' : 150}\n # LGBM\n params.update(hyperparams)\n verbosity = 100 if self.verbose else 0\n model = lgb.train(params, dtrain, valid_sets=dtest, \n num_boost_round=1000, verbose_eval=verbosity)\n pred = model.predict(test_x)\n return self.calc_metric(test_y, pred)\n\n if self.tuning_type == \"optuna_tuner\":\n dtrain, dtest, _, _ = get_dataset()\n verbosity = 100 if self.verbose else 0\n tuner_params = {\"objective\": params[\"objective\"],\n \"metric\" : params[\"metric\"]}\n # Run optimization\n model = optuna_lgb.train(tuner_params, dtrain, valid_sets=dtest, \n num_boost_round=1000, early_stopping_rounds=50,\n verbose_eval=verbosity)\n print('Best params:')\n for key, value in model.params.items():\n print(' {}: {}'.format(key, value))\n params = model.params\n params[\"feature_pre_filter\"] = True\n else:\n # Run optimization\n study = optuna.create_study(direction='maximize') # if uses loss, should use minimize\n study.optimize(objective, n_trials=40)\n print('Number of finished trials: {}'.format(len(study.trials)))\n trial = study.best_trial\n print('Best trial:')\n print(' Value: {}'.format(trial.value))\n print(' Params: ')\n for key, value in trial.params.items():\n print(' {}: {}'.format(key, value))\n params.update(trial.params)\n # Plot history\n plot_optimization_history(study)\n \n return params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"not_use_cols = [\"row_id\",\"user_id\",\"user_answer\",\"prior_question_had_explanation\",\n \"question_rate_class\",\"question_rate_class_exp\",'part_rate_curt','part_ratec_curt',\"abi_rate_curt\",\n \"cv_group\",\"max_time_stamp\",\"rand_time_stamp\",\"viretual_time_stamp\"]\nnot_use_cols = not_use_cols + ans_part_cols + num_part_cols + part_mistake_cols + tags_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N = 20000 if 20000 < train.shape[0] else train.shape[0]\ndf_corr = train.drop([c for c in train.columns if c in not_use_cols], axis=1).sample(n=N).corr()\ndf_corr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_corr[[\"answered_correctly\"]].abs().sort_values(\"answered_correctly\")[[\"answered_correctly\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"not_use_cols += [\"prior_question_elapsed_time\",'bundle_id',\"bundle_skew\",\"num_solving_question\",\n \"num_intention\",\"num_lec\",\"num_concept\",\"num_concept\",\"num_tag\",'deviation_m',\"LorR\",'part']\ncategoricals = []\nkeep_cols = [\"user_id\"]\ntarget = \"answered_correctly\"\ngroup = \"cv_group\" #\"user_id\"\nfeatures = [c for c in train.columns if c not in not_use_cols+categoricals+[target]]\n\nprint(len(features+categoricals))\nprint(\"Numeric features\")\ndisplay(sorted(features))\nprint(\"Categorical features\")\nprint(sorted(categoricals))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_ids = train.user_id.unique()\nrandom.shuffle(user_ids)\n\nuse_features = [target, group] + keep_cols + features + categoricals\n\ntrains = []\nfor i in range(1, MODEL_NUM+1):\n print(\"Model %s will be from 0 to %s in cv_group\" % (i, CV-1))\n train_data = train.loc[train.cv_group.isin(np.arange(CV)), use_features].copy()\n train_data.reset_index(drop=True, inplace=True)\n trains.append(train_data)\n \ntest_data = train.loc[train.cv_group==CV, use_features].copy()\ntest_data.reset_index(drop=True, inplace=True)\n\nif TUNE:\n tune_data = train.loc[train.cv_group==CV+1, use_features].copy()\n tune_data.reset_index(drop=True, inplace=True)\nelse:\n tune_data = pd.DataFrame()\n\ndel train, train_data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Training data\")\nfor i, train_data in enumerate(trains):\n print(\"No.\", i+1)\n print(train_data.shape, train_data.cv_group.unique(), train_data.user_id.nunique())\n display(train_data.head())\n \nprint(\"Test data\")\nprint(test_data.shape, test_data.cv_group.unique(), test_data.user_id.nunique())\ndisplay(test_data.head())\n\nif TUNE:\n print(\"Tune data\")\n print(tune_data.shape, tune_data.cv_group.unique(), tune_data.user_id.nunique())\n display(tune_data.head())\n \ndel train_data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For making a baseline\nparams1 = {\n 'objective': 'binary',\n \"boosting_type\": \"gbdt\",\n \"metric\": 'auc',\n 'learning_rate': 0.1,\n \"max_depth\": 4,\n \"max_bin\": 853,\n \"num_leaves\": 492,\n \"min_data_in_leaf\": 229,\n \"min_child_samples\": 55,\n \"feature_fraction\": 0.7894275993264507,\n \"bagging_fraction\": 0.8305608180096481,\n \"bagging_freq\": 6,\n \"lambda_l1\": 0.00010208001912287773,\n \"lambda_l2\": 9.315928833771094,\n 'early_stopping_rounds' : 100,\n 'random_state': 42,\n \"bagging_seed\": 11,\n \"verbosity\": -1}\n\n# Got from public\nparams2 = {\n 'objective': 'binary',\n \"boosting_type\": \"gbdt\",\n \"metric\": 'auc',\n 'learning_rate': 0.05,\n 'max_depth': 4,\n 'max_bin': 700,\n 'num_leaves': 350,\n 'min_child_weight': 0.03454472573214212,\n 'feature_fraction': 0.58,\n 'bagging_fraction': 0.58,\n 'reg_alpha': 0.3899927210061127,\n 'reg_lambda': 0.6485237330340494,\n 'early_stopping_rounds' : 100,\n 'random_state': 42,\n \"bagging_seed\": 11,\n \"verbosity\": -1}\n\n# The best params from user_id group k-fold\nparams3 = {\n 'objective': 'binary',\n \"boosting_type\": \"gbdt\",\n \"metric\": 'auc',\n 'learning_rate': 0.05,\n \"max_depth\": 9,\n \"max_bin\": 872,\n \"num_leaves\": 743,\n \"min_data_in_leaf\": 274,\n \"min_child_samples\": 27,\n \"feature_fraction\": 0.5922083852690104,\n \"bagging_fraction\": 0.7638997668812837,\n \"bagging_freq\": 5,\n \"lambda_l1\": 4.603407547894731e-07,\n \"lambda_l2\": 4.833680631174746e-07,\n 'early_stopping_rounds' : 200,\n 'random_state': 42,\n \"bagging_seed\": 11,\n \"verbosity\": -1}\n\n# The best params from cv_group group k-fold\nparams4 = {\n 'objective': 'binary',\n \"boosting_type\": \"gbdt\",\n \"metric\": 'auc',\n 'learning_rate': 0.05,\n \"max_depth\": 8,\n \"max_bin\": 990,\n \"num_leaves\": 885,\n \"min_data_in_leaf\": 214,\n \"min_child_samples\": 89,\n \"feature_fraction\": 0.4990107527526446,\n \"bagging_fraction\": 0.9513631160877628,\n \"bagging_freq\": 7,\n \"lambda_l1\": 4.856465786762882e-05,\n \"lambda_l2\": 1.6804951000850403e-08,\n 'early_stopping_rounds' : 200,\n 'random_state': 42,\n \"bagging_seed\": 11,\n \"verbosity\": -1}\n\nif TUNE:\n params = [None] * MODEL_NUM\nelif MODEL_NUM == 1:\n params = [params3]\nelse:\n params = [params3, params4]\ntuning_type = \"optuna\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nlgbms = []\nfor idx, train_data in enumerate(trains):\n if DEBUG:\n lgbm = LgbModel(train_data.sample(n=30000).reset_index(drop=True), test_data,\n target, features, categoricals=categoricals,\n task=\"binary\", params=None, parameter_tuning=TUNE, tuning_type=tuning_type,\n tune_df=tune_data, cv_method=\"GroupKFold\", n_splits=4, group=group, verbose=False)\n else:\n lgbm = LgbModel(train_data, test_data, \n target, features, categoricals=categoricals,\n task=\"binary\", params=params[idx], parameter_tuning=TUNE, tuning_type=tuning_type,\n tune_df=tune_data, cv_method=\"GroupKFold\", n_splits=4, group=group, verbose=False)\n lgbms.append(lgbm)\n del lgbm\n gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for lgbm in lgbms:\n _ = lgbm.plot_feature_importance()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for lgbm in lgbms:\n score = roc_auc_score(test_data.answered_correctly, lgbm.y_pred)\n fpr, tpr, thresholds = roc_curve(test_data.answered_correctly, lgbm.y_pred)\n print(score)\n plt.plot(fpr, tpr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_cols = [\"correct\",\"answered_correctly\",\"predict\",\"predict_prob\",\"user_id\"]+features\ntest_data[\"predict_prob\"] = lgbm.y_pred\ntest_data[\"predict\"] = test_data.predict_prob.apply(lambda x: 1 if x>0.5 else 0)\ntest_data[\"correct\"] = test_data[[\"answered_correctly\",\"predict\"]].apply(lambda x: 1 if x[0]==x[1] else 0, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.sort_values(\"predict_prob\", ascending=False, inplace=True)\ntest_data[show_cols].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_res_sorted = test_data.loc[(test_data.correct==0), show_cols].sort_values(\"predict_prob\", ascending=False)\ndf_res_sorted.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for u in df_res_sorted.user_id.unique()[:3]:\n test_data.loc[test_data.user_id==u, show_cols].to_csv(\"debug_\"+str(u)+\".csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = []\nfor lgbm in lgbms:\n models += lgbm.models\nprint(len(models))\n\ndel lgbms, train_data, tune_data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load SAKT model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def future_mask(seq_length):\n future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n return torch.from_numpy(future_mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FFN(nn.Module):\n def __init__(self, state_size=200, forward_expansion=1, bn_size=(180-1), dropout=0.2):\n super(FFN, self).__init__()\n self.state_size = state_size\n \n self.lr1 = nn.Linear(state_size, forward_expansion * state_size)\n self.relu = nn.ReLU()\n self.bn = nn.BatchNorm1d(bn_size)\n self.lr2 = nn.Linear(forward_expansion * state_size, state_size)\n self.dropout = nn.Dropout(dropout)\n \n def forward(self, x):\n x = self.relu(self.lr1(x))\n x = self.bn(x)\n x = self.lr2(x)\n return self.dropout(x)\n \nclass TransformerBlock(nn.Module):\n def __init__(self, embed_dim, heads=8, dropout=0.1, forward_expansion=1):\n super(TransformerBlock, self).__init__()\n self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)\n self.dropout = nn.Dropout(dropout)\n self.layer_normal = nn.LayerNorm(embed_dim)\n self.ffn = FFN(embed_dim, forward_expansion=forward_expansion, dropout=dropout)\n self.layer_normal_2 = nn.LayerNorm(embed_dim)\n \n def forward(self, value, key, query, att_mask):\n att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)\n att_output = self.dropout(self.layer_normal(att_output + value))\n att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n x = self.ffn(att_output)\n x = self.dropout(self.layer_normal_2(x + att_output))\n return x.squeeze(-1), att_weight\n \nclass Encoder(nn.Module):\n def __init__(self, n_skill, max_seq=180, embed_dim=128, dropout=0.1, forward_expansion=1, num_layers=1, heads=8):\n super(Encoder, self).__init__()\n self.n_skill, self.embed_dim = n_skill, embed_dim\n self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)\n self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)\n self.e_embedding = nn.Embedding(n_skill + 1, embed_dim)\n self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion) for _ in range(num_layers)])\n self.dropout = nn.Dropout(dropout)\n \n def forward(self, x, question_ids):\n device = x.device\n x = self.embedding(x)\n pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n pos_x = self.pos_embedding(pos_id)\n x = self.dropout(x + pos_x)\n x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n e = self.e_embedding(question_ids)\n e = e.permute(1, 0, 2)\n for layer in self.layers:\n att_mask = future_mask(e.size(0)).to(device)\n x, att_weight = layer(e, x, x, att_mask=att_mask)\n x = x.permute(1, 0, 2)\n x = x.permute(1, 0, 2)\n return x, att_weight\n\nclass SAKTModel(nn.Module):\n def __init__(self, n_skill, max_seq=180, embed_dim=128, dropout=0.1, forward_expansion=1, enc_layers=1, heads=8):\n super(SAKTModel, self).__init__()\n self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)\n self.pred = nn.Linear(embed_dim+1, 1)\n \n def forward(self, x1, x2, question_ids):\n device = x1.device\n x, att_weight = self.encoder(x1, question_ids)\n x = torch.cat([x, x2.unsqueeze(2)], dim=2).to(device)\n x = self.pred(x)\n return x.squeeze(-1), att_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndevice = torch.device(\"cpu\")\nskills = joblib.load(\"/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip\")\nn_skill = len(skills)\ngroup = joblib.load(\"/kaggle/input/riiid-sakt-model-dataset-own/group.pkl.zip\")\nlags = joblib.load(\"/kaggle/input/riiid-sakt-model-dataset-own/lags.pkl.zip\")\n\nnn_model = SAKTModel(n_skill, max_seq=180, embed_dim=128, forward_expansion=1, enc_layers=1, heads=8, dropout=0.1)\nnn_model.load_state_dict(torch.load(\"/kaggle/input/riiid-sakt-model-dataset-own/sakt_model.pt\", map_location='cpu'))\n\nnn_model.to(device)\nnn_model.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n def __init__(self, samples, lags, test_df, max_seq=180): \n super(TestDataset, self).__init__()\n self.samples = samples\n self.lags = lags\n self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n self.test_df = test_df\n self.n_skill = n_skill\n self.max_seq = max_seq\n \n def __len__(self):\n return self.test_df.shape[0]\n \n def __getitem__(self, index):\n test_info = self.test_df.iloc[index]\n\n user_id = test_info[\"user_id\"]\n target_id = test_info[\"question_id\"]\n lagtime_mean = test_info[\"lagtime_mean\"]\n\n q = np.zeros(self.max_seq, dtype=int)\n qa = np.zeros(self.max_seq, dtype=int)\n l = np.zeros(self.max_seq, dtype=float)\n\n if user_id in self.samples.index:\n q_, qa_ = self.samples[user_id]\n l_ = self.lags[user_id]\n seq_len = len(q_)\n if seq_len >= self.max_seq:\n q = q_ [-self.max_seq:]\n qa = qa_[-self.max_seq:]\n l = l_ [-self.max_seq:]\n else:\n q [-seq_len:] = q_\n qa[-seq_len:] = qa_\n l [-seq_len:] = l_\n \n x1 = q[1:].copy()\n x1 += (qa[1:] == 1) * self.n_skill\n x2 = l[1:].copy()\n questions = np.append(q[2:], [target_id])\n \n return x1, x2, questions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"def insert_new_data_to_hist(ary, ins):\n ary = ary.copy()\n ary = np.vstack([ary, ins])\n ary = ary[np.argsort(ary[:,0])] # Sort by user_id\n return ary\n\ndef groupby_with_numpy(ary, t_idxs, qucnt):\n ary = ary.copy()\n g = sum([[i+1 for _ in range(c)] for i, c in enumerate(qucnt)], [])\n r = np.zeros((qucnt.shape[0], t_idxs.shape[0]))\n for i, t in enumerate(t_idxs):\n r[:,i] = np.bincount(g, ary[:,t])[1:]\n return r\n\ndef add_mistake_feat(df, t_col, n_col):\n df = df.copy()\n if df[n_col].min() == 0:\n df[t_col] = df[n_col].apply(lambda x: 0.6*x**(-0.2) if x!=0 else 0.6)\n else:\n df[t_col] = 0.6*df[n_col]**(-0.2)\n return df\n\ndef get_num_etimes(df):\n num_etimes = []\n for u, et in np.array(df[[\"user_id\",\"prior_question_elapsed_time\"]]):\n if u in d.keys():\n etime_mean = d_etime[d[u][-1, 1]]\n et_norm = et / (1000*3600)\n num_etimes.append(et_norm - etime_mean)\n else:\n num_etimes.append(0)\n return num_etimes\n\ndef add_to_timedict(df):\n for u, t in np.array(df[[\"user_id\",\"timestamp\"]]):\n if u in d_timestamp.keys():\n d_timestamp[u] = np.unique(np.hstack([d_timestamp[u], np.array([t])]))\n else:\n d_timestamp[u] = np.array([t])\n\ndef add_to_history(df, new_row_id):\n for r in np.array(df[[\"user_id\",\"question_id\",\"part\",\"tag1\"]]):\n if r[0] in d.keys():\n d[r[0]] = np.vstack([d[r[0]], np.hstack([np.array(new_row_id), r[1:], np.array(0)])])\n else:\n d[r[0]] = np.array([np.hstack([np.array(new_row_id), r[1:], np.array(0)])])\n \ndef upd_history_ans(df):\n for r in np.array(df[[\"user_id\",\"answered_correctly\"]]):\n if r[0] in d.keys() and r[1] == 1:\n d[r[0]][-1,-1] = r[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_lecs.drop([\"part\",\"starter\"], axis=1, inplace=True)\ndf_qs = df_qs.merge(df_question_rate, on=[\"question_id\",\"bundle_id\"]).merge(df_part_rate, on=\"part\").merge(df_etime_rate, on=\"question_id\")\ndf_qs.sort_values(\"question_id\", inplace=True)\ndf_qs.reset_index(drop=True, inplace=True)\ndf_qs[\"LorR\"] = 0\ndf_qs.loc[df_qs.part.isin([5,6,7]), \"LorR\"] = 1\n\nprint(df_lecs.shape, df_qs.shape)\ndisplay(df_lecs.head(2))\ndisplay(df_qs.head(2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sort_cols = ['user_id',\"num_exp\",'num_ans','num_devp','num_devm',\"num_etime\",'num_que'] + part_cols + abi_cols\nar_hist = np.array(df_history[sort_cols])\nar_qs = np.array(df_qs)[:,0]\nar_ans_class = np.array(df_ans_class)[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#(test_df, sample_prediction_df) = next(iter_test)\n#test_df_bk = test_df.copy()\n#idx = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_cols = [\"ans_rate_class\",\"study_rate_class\"] + [f\"part{p}_rate_class\" for p in range(1,8)]\nn_cols = [\"num_que\", \"num_que\"] + num_part_cols\nr_cols = ['row_id','timestamp','user_id','question_id','content_type_id','task_container_id','prior_question_elapsed_time','prior_question_had_explanation']\ndf_test = pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor idx, (test_df, sample_prediction_df) in enumerate(iter_test):\n # Modify DataFrame\n test_df.prior_question_had_explanation = (test_df.prior_question_had_explanation.fillna(False) * 1).astype(int)\n test_df.prior_question_elapsed_time = test_df.prior_question_elapsed_time.fillna(0).astype(int)\n \n # Get prior correct answer\n prior_ans = eval(test_df.prior_group_answers_correct.values[0])\n\n # DataFrame to array\n test_ary = np.array(test_df)[:,:8]\n lec_ary = test_ary[test_ary[:,4]==1, 2:4]\n test_ary = test_ary[test_ary[:,4]==0, :]\n\n # Make user information\n qusers, qucnt = np.unique(np.sort(test_ary[:,2]), return_counts=True)\n qusers = qusers.astype(\"int32\")\n existing_qusers_bol = np.isin(ar_hist[:,0], qusers)\n not_existing_qusers = qusers[~np.isin(qusers, ar_hist[existing_qusers_bol, 0])]\n\n # Store some information\n questions = np.unique(test_ary[:,3].astype(\"int32\"))\n group_num = test_df.index[:test_ary.shape[0]]\n \n # Add the number of correct answering to history\n if len(prior_ans) != 0 and prior_ans is not None and prior_ans is not np.nan:\n # Delete lecture data\n prior_ans = np.array([a for a in prior_ans if a != -1])\n if 0 < len(prior_ans) and len(prior_ans) == len(prior_test_user_ids) == len(prior_test_devp):\n df_prev['answered_correctly'] = prior_ans\n upd_history_ans(df_prev)\n # Update for SAKT model\n if ENSEMBLE:\n prev_group = df_prev.groupby('user_id').apply(lambda x: (x['question_id'].values,\n x['answered_correctly'].values,\n x[\"lagtime_mean\"].values))\n for prev_user_id in prev_group.index:\n if prev_user_id in group.index:\n group[prev_user_id] = (np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-180:], \n np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-180:])\n lags [prev_user_id] = (np.append(lags [prev_user_id], prev_group[prev_user_id][2])[-180:])\n\n else:\n group[prev_user_id] = (prev_group[prev_user_id][0], prev_group[prev_user_id][1])\n lags [prev_user_id] = prev_group[prev_user_id][2]\n # Update for LGBM\n ans_ary = np.array([prior_test_user_ids,\n prior_ans,\n prior_test_devp.fillna(9),\n prior_test_devm,\n prior_test_etime]).T\n ans_ary = ans_ary[ans_ary[:,2]!=9]\n if 0 < ans_ary.shape[0]:\n # The data answered correctly should be aggregated\n ans_ary[:,2] *= ans_ary[:,1]\n ans_ary[:,3] = ans_ary[:,2] + np.where(ans_ary[:,1]==0, -1, 0) * ans_ary[:,3]\n # Calculate ans and num for each part and user ability\n part_ans_ary = prior_test_part * ans_ary[:,1].reshape(-1,1)\n abi_ans_ary = prior_test_qc * ans_ary[:,1].reshape(-1,1)\n ans_ary = np.hstack([ans_ary,\n np.ones(ans_ary.shape[0]).reshape(-1,1),\n part_ans_ary,\n prior_test_part,\n abi_ans_ary,\n prior_test_qc])\n ans_ary = ans_ary[np.argsort(ans_ary[:,0])]\n if len(prior_test_user_ids) != len(np.unique(prior_test_user_ids)):\n # Group by user_id because of existing some questions in the user\n _, qacnt = np.unique(ans_ary[:,0], return_counts=True)\n ans_ary = groupby_with_numpy(ans_ary, np.arange(1, ans_ary.shape[1]), qacnt)\n else:\n ans_ary = ans_ary[:,1:]\n # Update existing data\n ar_hist[prior_hist_equb, 2:] += ans_ary\n \n # Add new users to history\n if 0 < not_existing_qusers.shape[0]:\n ins_exp = np.hstack([not_existing_qusers.reshape(-1,1),\n np.zeros((not_existing_qusers.shape[0], ar_hist.shape[1]-1))])\n ar_hist = insert_new_data_to_hist(ar_hist, ins_exp)\n existing_qusers_bol = np.isin(ar_hist[:,0], qusers)\n # Update existing data\n if 0 < test_ary.shape[0]:\n ary = test_ary[:,[2,7]].copy()\n ary = ary.astype(int)\n ary = ary[np.argsort(ary[:,0])]\n ar_hist[existing_qusers_bol, 1] += groupby_with_numpy(ary, np.array([1]), qucnt).flatten()\n\n # ========================================\n # Prediction with LGBM\n # Merge with some DataFrames\n df_hist = pd.DataFrame(ar_hist[existing_qusers_bol], columns=sort_cols)\n df_test = pd.DataFrame(test_ary, columns=r_cols)\n df_test = df_test.merge(df_hist, on=\"user_id\", sort=False)\n df_test = df_test.merge(df_qs[np.isin(ar_qs, questions)], on=[\"question_id\",\"prior_question_had_explanation\"], sort=False)\n \n # Make features\n df_test[\"ans_rate\"] = (df_test.num_ans / df_test.num_que).fillna(0)\n df_test[\"devp_rate\"] = df_test.num_devp / df_test.num_que\n df_test[\"devm_rate\"] = df_test.num_devm / df_test.num_que\n df_test[\"etime_rate\"] = df_test.num_etime / df_test.num_que\n ar_study_rate = np.array(df_test.num_exp / df_test.num_que)\n ar_study_rate[np.isnan(ar_study_rate)] = 0\n ar_study_rate[np.isinf(ar_study_rate)] = 0\n ar_study_rate[1 < ar_study_rate] = 1\n df_test[\"study_rate\"] = ar_study_rate\n df_class = df_ans_class[np.isin(ar_ans_class, questions)].copy()\n df_test = cut_and_merge_with_class(df_test, df_class, \"ans_rate\", \"ans_rate_class\")\n df_test = cut_and_merge_with_class(df_test, df_class, \"study_rate\", \"study_rate_class\")\n df_test = add_mistake_feat(df_test, \"prob_mistake_que\", \"num_que\")\n for p in range(0,10):\n if p in [0,3,4,5,6,7,8,9]:\n df_test[f\"abi_rate{p}\"] = (df_test[f\"ans_abi{p}\"] / df_test[f\"num_abi{p}\"]).fillna(0)\n if p in [1,2,3,4,5,6,7]:\n df_test[f\"part{p}_rate\"] = (df_test[f\"ans_part{p}\"] / df_test[f\"num_part{p}\"]).fillna(0)\n df_test = add_mistake_feat(df_test, f\"prob_mistake_part{p}\", f\"num_part{p}\")\n df_test = cut_and_merge_with_class(df_test, df_class, f\"part{p}_rate\", f\"part{p}_rate_class\")\n for t, n in zip(t_cols, n_cols):\n if df_test[n].min() < 5:\n # Replace to 0.3/0.5/0.7 for small answering\n df_test = change_rate_with_num(df_test, t, n, method=\"respectively\")\n df_test = change_rate_with_num(df_test, t, n, method=\"bundle\")\n df_test[\"lagtime1\"] = (df_test[[\"user_id\",\"timestamp\"]].apply(lambda x: get_lagtime(x[0], x[1], -1), axis=1) / (1000*3600)).astype(\"float32\")\n df_test[\"lagtime2\"] = (df_test[[\"user_id\",\"timestamp\"]].apply(lambda x: get_lagtime(x[0], x[1], -2), axis=1) / (1000*3600)).astype(\"float32\")\n df_test[\"lagtime3\"] = (df_test[[\"user_id\",\"timestamp\"]].apply(lambda x: get_lagtime(x[0], x[1], -3), axis=1) / (1000*3600)).astype(\"float32\")\n df_test[\"lagtime_mean\"] = (df_test.lagtime1 + df_test.lagtime2 + df_test.lagtime3) / 3\n df_test[\"timestamp\"] = (df_test.timestamp / (1000*3600)).astype(\"float32\")\n df_test[\"lagtime1\"] = (df_test.timestamp - df_test.lagtime1).astype(\"float32\")\n df_test[\"lagtime2\"] = (df_test.timestamp - df_test.lagtime2).astype(\"float32\")\n df_test[\"lagtime3\"] = (df_test.timestamp - df_test.lagtime3).astype(\"float32\")\n df_test[\"lagtime_mean\"] = np.log((df_test.timestamp - df_test.lagtime_mean) + 1).astype(\"float32\")\n df_test[\"noplay_days\"] = (df_test[[\"user_id\",\"timestamp\"]].apply(lambda x: get_noplay_days(x[0], x[1]), axis=1)).astype(\"float32\")\n df_test[\"elapsed_days\"] = (df_test.timestamp / 24).astype(\"int32\")\n df_test[\"noplay_rate\"] = (df_test.noplay_days / df_test.num_que).astype(\"float32\")\n df_test[\"timestamp_rate\"] = (df_test.timestamp / df_test.num_que).astype(\"float32\")\n df_test[\"elapsed_rate\"] = (df_test.elapsed_days / df_test.num_que).astype(\"float32\")\n df_test[\"attempt_que\"] = df_test[[\"user_id\",\"question_id\"]].apply(lambda x: get_attempt_que(x[0], 101230332+idx, x[1]), axis=1)\n df_test[\"attempt_tags\"] = df_test[[\"user_id\",\"tag1\"]].apply(lambda x: get_attempt_tags(x[0], 101230332+idx, x[1]), axis=1)\n df_test[\"attempt_curtar\"] = df_test[[\"user_id\"]].apply(lambda x: get_attempt_curt_ans_rate(x[0], 101230332+idx), axis=1)\n all_prob_mistake = np.array(df_test[part_mistake_cols])\n all_part_rate = np.array(df_test[[\"part1_rate\",\"part2_rate\",\"part3_rate\",\"part4_rate\",\"part5_rate\",\"part6_rate\",\"part7_rate\"]])\n all_part_ratec = np.array(df_test[[\"part1_rate_class\",\"part2_rate_class\",\"part3_rate_class\",\"part4_rate_class\",\"part5_rate_class\",\"part6_rate_class\",\"part7_rate_class\"]])\n all_abi_rate = np.array(df_test[[\"abi_rate0\",\"abi_rate3\",\"abi_rate4\",\"abi_rate5\",\"abi_rate6\",\"abi_rate7\",\"abi_rate8\",\"abi_rate9\"]])\n one_hot_part = np.identity(8) [df_test.part][:,1:]\n one_hot_ability = np.identity(10)[df_test.question_rate_class_exp][:,[0,3,4,5,6,7,8,9]]\n df_test[\"prob_mistake_part_curt\"] = np.nanmax(one_hot_part*all_prob_mistake, axis=1)\n df_test[\"part_rate_curt\"] = np.nanmax(one_hot_part*all_part_rate, axis=1)\n df_test[\"part_ratec_curt\"] = np.nanmax(one_hot_part*all_part_ratec, axis=1)\n df_test[\"abi_rate_curt\"] = np.nanmax(one_hot_ability*all_abi_rate, axis=1)\n df_test[\"part_ratec_mean\"] = np.nan_to_num(all_part_ratec).sum(1) / 7\n df_test[\"harmonic_ar_prc\"] = 2 * (df_test.ans_rate * df_test.part_rate_curt) / (df_test.ans_rate + df_test.part_rate_curt)\n df_test[\"harmonic_arc_qr\"] = 2 * (df_test.ans_rate_class * df_test.question_rate_exp) / (df_test.ans_rate_class + df_test.question_rate_exp)\n df_test[\"harmonic_arc_prcc\"] = 2 * (df_test.ans_rate_class * df_test.part_ratec_curt) / (df_test.ans_rate_class + df_test.part_ratec_curt)\n df_test[\"harmonic_prc_pr\"] = 2 * (df_test.part_rate_curt * df_test.part_rate) / (df_test.part_rate_curt + df_test.part_rate)\n df_test[\"harmonic_prcc_qr\"] = 2 * (df_test.part_ratec_curt * df_test.question_rate_exp) / (df_test.part_ratec_curt + df_test.question_rate_exp)\n df_test[\"harmonic_qr_pr\"] = 2 * (df_test.part_rate * df_test.question_rate_exp) / (df_test.part_rate + df_test.question_rate_exp)\n df_test[\"harmonic_abrc_qr\"] = 2 * (df_test.abi_rate_curt * df_test.question_rate_exp) / (df_test.abi_rate_curt + df_test.question_rate_exp)\n df_test[\"harmonic_arc_src\"] = 2 * (df_test.ans_rate_class * df_test.study_rate_class) / (df_test.ans_rate_class + df_test.study_rate_class)\n df_test[\"harmonic_prcc_src\"] = 2 * (df_test.part_ratec_curt * df_test.study_rate_class) / (df_test.part_ratec_curt + df_test.study_rate_class)\n df_test[\"harmonic_src_qr\"] = 2 * (df_test.study_rate_class * df_test.question_rate_exp) / (df_test.study_rate_class + df_test.question_rate_exp)\n df_test[\"rate_arc_prcc\"] = df_test.ans_rate_class / df_test.part_ratec_curt\n df_test[\"rate_arc_src\"] = df_test.ans_rate_class / df_test.study_rate_class\n df_test[\"rate_prcc_src\"] = df_test.part_ratec_curt / df_test.study_rate_class\n df_test[\"rate_arc_qr\"] = df_test.ans_rate_class / df_test.question_rate_exp\n df_test[\"rate_qr_pr\"] = df_test.part_rate / df_test.question_rate_exp\n df_test[\"rate_prcc_prccm\"] = df_test.part_ratec_curt / df_test.part_ratec_mean\n\n # Store prior infomation for next batch\n prior_hist_equb = existing_qusers_bol\n prior_test_user_ids = np.array(df_test.user_id).astype(\"int32\")\n prior_test_part = one_hot_part\n prior_test_qc = one_hot_ability\n prior_test_devp = df_test.deviation_p\n prior_test_devm = df_test.deviation_m\n prior_test_etime = get_num_etimes(df_test)\n\n # Fill Nan by 0\n df_test.fillna(0, inplace=True)\n \n # Sort to default order\n df_test.sort_values(\"row_id\", inplace=True)\n df_test.reset_index(drop=True, inplace=True)\n \n if idx < 1:\n display(test_df)\n display(df_test[features+categoricals])\n\n # Prediction\n reults = np.zeros(df_test.shape[0])\n for model in models:\n reults += model.predict(df_test[features+categoricals], num_iteration=model.best_iteration) / len(models)\n \n # ========================================\n # Prediction with SAKT model\n if ENSEMBLE:\n outs = []\n test_dataset = TestDataset(group, lags, df_test[[\"user_id\",\"question_id\",\"lagtime_mean\"]])\n test_dataloader = DataLoader(test_dataset, batch_size=df_test.shape[0], shuffle=False)\n for item in test_dataloader:\n x1 = item[0].to(device).long()\n x2 = item[1].to(device).float()\n target_id = item[2].to(device).long()\n with torch.no_grad():\n output, att_weight = nn_model(x1, x2, target_id)\n outs.extend(torch.sigmoid(output)[:, -1].view(-1).data.cpu().numpy())\n \n # ========================================\n # Submit and post processing for the next batch\n if ENSEMBLE:\n df_test['answered_correctly'] = S * np.array(outs) + L * reults\n else:\n df_test['answered_correctly'] = reults\n\n # Add current timestamp to time dict\n add_to_timedict(test_df)\n # Add current data to history\n add_to_history(df_test, 101230332+idx)\n # Store for updating knowledges for SAKT model\n df_prev = df_test[['user_id','question_id',\"lagtime_mean\"]].copy()\n \n # Submit to predict function\n df_test = df_test[['row_id', 'answered_correctly']].copy()\n df_test.index = group_num\n env.predict(df_test)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}