{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# DONE avg answer correctness for qn - better\n# DONE train mean instead of .5 for new users' user_avg - got worse\n# DONE avg correctness for user - worse\n#   DONE update stats on test - better\n# DONE pickle train.csv\n# DONE integrate alldata into non-submit training (exclude val from alldata)\n# DONE ditch pandas for inference - 20 times faster w/ python dicts\n# DONE remove content_id from rf\n# DONE remove task_container_id from rf\n# DONE total count of qns for user aka experience - worse\n# DONE total count of answers for qn from all users - better on val, no change on test\n# DONE qn stats updating - no change","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO if qns in containers always come in same groups renumber containers to same ids (is it bundle_id?)\n# TODO ts_diff taking lectures into account\n# TODO use lecture rows\n# TODO online learning during test\n# TODO avg correctness for specific qn for user (huge memory reqs!!!)\n# TODO prev answer to same qn (see archive) (huge memory reqs!!!)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit=True","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n# import matplotlib.pyplot as plt\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import roc_auc_score\nfrom time import time","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"PATH='/kaggle/input/riiid-test-answer-prediction/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nalldata=pd.read_pickle('/kaggle/input/riiidtrainpkl/riiid-train.pkl')\nalldata.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_rows=1*10**6\n# don't split at the middle of user's data, move to the next user boundary\nwhile alldata.user_id.iat[train_rows]==alldata.user_id.iat[train_rows-1]:\n    train_rows+=1\ndata=alldata.head(train_rows).copy()\ntrain_rows","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"last=data[['user_id','task_container_id']].drop_duplicates().groupby('user_id').tail(5)\nval=data.merge(last,how='inner',on=['user_id','task_container_id']).copy()\ndel last\ntr=data[~data.row_id.isin(val.row_id)].copy()\nif not submit:\n    alldata=alldata[~alldata.row_id.isin(val.row_id)].copy()\nlen(tr),len(val),len(alldata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def drop_lectures(df): return df.query('content_type_id==0').drop(columns='content_type_id').copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr=drop_lectures(tr)\nalldata=drop_lectures(alldata)\nlen(tr),len(alldata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stamps=tr.groupby(['user_id','timestamp'],as_index=False).timestamp.first()\nstamps=stamps.merge(stamps,how='left',left_on=['user_id',stamps.index],\n                    right_on=['user_id',stamps.index+1],suffixes=('','_lag'))\nstamps=stamps.drop(columns='key_1')\nstamps['timestamp_lag']=stamps.timestamp_lag.fillna(0).astype('int64')\nstamps['ts_diff']=stamps.timestamp-stamps.timestamp_lag\nstamps=stamps.drop(columns='timestamp_lag')\ntr=tr.merge(stamps,how='left',on=['user_id','timestamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"laststamps=alldata.groupby('user_id').timestamp.last()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qn_avgs=alldata.groupby('content_id').answered_correctly.mean().rename('qn_avg')\ntr=tr.join(qn_avgs,on='content_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_cnts=alldata.groupby('user_id').answered_correctly.count().rename('user_cnt')\nuser_avgs=alldata.groupby('user_id').answered_correctly.mean().rename('user_avg')\ntr=tr.join(user_avgs,on='user_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FillNa:\n    def __call__(self,df):\n        df.loc[:,'prior_question_elapsed_time'].fillna(self.elapsed_median,inplace=True)\n        df.loc[:,'prior_question_had_explanation'].fillna(True,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fillna=FillNa()\nfillna.elapsed_median=alldata.prior_question_elapsed_time.median()\nfillna(tr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del alldata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tr.describe(include='all').T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_Xy(df):\n    cols=['user_answer','answered_correctly']\n    y=df[cols].copy()\n    X=df.drop(columns=cols)\n    return X,y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X,y=split_Xy(tr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"drop_cols=['row_id','user_id','content_id','task_container_id']\nX=X.drop(columns=drop_cols)\ny=y.answered_correctly","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def enum_conts(df):\n    df.loc[:,'cont_num']=np.arange(len(df))\n    return df\n\ndef sort_by_cont(df):\n    conts=df[['user_id','task_container_id']].drop_duplicates()\n    conts['cont_num']=0\n    conts=conts.groupby('user_id').apply(enum_conts)\n    df=df.merge(conts,how='left',on=['user_id','task_container_id'])\n    df=df.sort_values(['cont_num','user_id','row_id'])\n    df=df.drop(columns='cont_num')\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not submit:\n    val=sort_by_cont(val)\n    nancol=np.full(len(val),float('nan'),dtype='object')\n    val['prior_group_responses']=nancol.copy()\n    val['prior_group_answers_correct']=nancol.copy()\n    del nancol","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not submit:\n    xv,yv=split_Xy(val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestIter:\n    def __init__(self,test,y,bs):\n        self.test=test.copy() # copy b/c we modify prior_group_* cols inplace\n        self.y,self.bs=y,bs\n        self.start=0\n        self.prior_ans=repr([])\n        self.prior_ans_idx=test.columns.get_loc('prior_group_answers_correct')\n    def __iter__(self): return self\n    def __next__(self):\n        if self.start>=len(self.test): raise StopIteration\n        end=self.start+self.bs\n        while end<len(self.test):\n            cur=self.test.iloc[end]\n            prev=self.test.iloc[end-1]\n            if (cur.user_id!=prev.user_id or\n                cur.task_container_id!=prev.task_container_id): break\n            end+=1\n        df=self.test.iloc[self.start:end]\n        df.iat[0,self.prior_ans_idx]=self.prior_ans\n        self.prior_ans=repr(self.y.answered_correctly.iloc[self.start:end].to_list())\n        self.start=end\n        return df,None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestEnv:\n    def __init__(self,y):\n        self.y=y\n        self.preds=pd.DataFrame()\n    def predict(self,preds):\n        self.preds=self.preds.append(preds)\n    def score(self):\n        preds=self.preds.answered_correctly\n        acc=(self.y==np.rint(preds)).mean()\n        auc=roc_auc_score(self.y,preds)\n        print(f'{acc:.03f} {auc:.03f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def do_submit(it,env,ret_full_test=False):\n    info={'laststamps':laststamps.to_dict(),\n          'qn_avgs':qn_avgs.to_dict(),\n          'user_cnts':user_cnts.to_dict(),\n          'user_avgs':user_avgs.to_dict()}\n    if ret_full_test: fulltest=[]\n\n    t0=time()\n    for test,_ in it:\n        test,sample=prepr_test(test,info)\n        if ret_full_test: fulltest.append(test)\n        sample['answered_correctly']=m.predict_proba(test)[:,1]\n        env.predict(sample)\n    print(f'{time()-t0:.03f}s')\n\n    if ret_full_test:\n        return pd.DataFrame(np.vstack(fulltest),columns=X.columns)\n    return test,sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepr_test(test,info):\n    index=test.index.to_list()\n    test=test.to_dict('list')\n    test['index']=index\n    \n    prior_ans=eval(test['prior_group_answers_correct'][0])\n    if prior_ans:\n        user_cnts,user_avgs=info['user_cnts'],info['user_avgs']\n        for user,ans in zip(info['prior_users'],prior_ans):\n            if ans<0: continue # lecture\n            cnt=user_cnts.get(user,0)\n            if cnt:\n                user_avgs[user]=(user_avgs[user]*cnt+ans)/(cnt+1)\n                user_cnts[user]=cnt+1\n            else:\n                user_cnts[user]=1\n                user_avgs[user]=ans\n    info['prior_users']=test['user_id']\n    \n    del test['prior_group_responses']\n    del test['prior_group_answers_correct']\n\n    ctype=test.pop('content_type_id')\n    test={k:[v for t,v in zip(ctype,vals) if t==0] for k,vals in test.items()}\n    \n    laststamps=info['laststamps']\n    test['timestamp_lag']=[laststamps.get(u,t)\n                           for u,t in zip(test['user_id'],test['timestamp'])]\n    test['ts_diff']=[t-l for t,l in zip(test['timestamp'],test['timestamp_lag'])]\n    del test['timestamp_lag']\n    laststamps.update(zip(test['user_id'],test['timestamp']))\n    \n    qn_avgs=info['qn_avgs']\n    test['qn_avg']=[qn_avgs.get(c,.5) for c in test['content_id']]\n    \n    user_avgs=info['user_avgs']\n    test['user_avg']=[user_avgs.get(u,.5) for u in test['user_id']]\n    \n    test['prior_question_elapsed_time']=[fillna.elapsed_median if t!=t else t\n                                         for t in test['prior_question_elapsed_time']]\n    test['prior_question_had_explanation']=[True if e is pd.NA else e\n                                            for e in test['prior_question_had_explanation']]\n\n    sample=pd.DataFrame(test['row_id'],columns=['row_id'],index=test['index'])\n    del test['index']\n    \n    for c in drop_cols: del test[c]\n    \n    test=np.column_stack([np.asarray(col,dtype=np.float32) for col in test.values()])\n    return test,sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nrf=RandomForestClassifier(n_estimators=100,n_jobs=-1,max_samples=.1,min_samples_leaf=40)\nm=rf.fit(X,y)\nm.n_jobs=1 # for small batches single job inference is the fastest","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not submit:\n    yv2=yv[yv.answered_correctly>=0].answered_correctly\n    env=TestEnv(yv2)\n    xv2=do_submit(TestIter(xv,yv,26),env,ret_full_test=True)\n    env.score()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#0.597 0.632 0.608 ver1  base\n#0.607 0.650 0.619 ver2  +timestamp_diff-user_id\n#0.647 0.701 0.710 ver3  +qn_avg\n#0.661 0.720 0.682 ver4  +user_avg\n#0.661 0.720 0.731 ver5  +user_avg w/ test stats updating \n#0.661 0.720 0.751 ver7  qn,usr stats from all data, not just train\n#0.664 0.723 0.755 ver8  4M train rows instead of 1M - 8h (but same inference duration)\n#0.664 0.723 0.748 ver9  w/out usr stats updating - !!!4h!!!\n#0.661 0.720 0.745 ver12 1M train rows\n#0.675 0.737 0.744 ver13 batched simulator instead of plain val\n#0.677 0.741 0.744 ver17 alldata for both training and submit - 3h\n#0.679 0.744 0.751 ver20 ditch pandas for inference, user stats updating - 40m\n#0.681 0.745 0.755 ver21 drop content_id,task_container_id, remove max_features from rf - 2h\n#0.682 0.746 0.755 ver23 +qn_cnt - 1.5h\n#0.680 0.746 0.755 ver25 -qn_cnt; qn stats updating - 35m\n#0.685 0.750 0.758 ver29 20M train rows\n#0.681 0.747       ver30 1M train rows, 100 (5x more) rf trees - ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_imp():\n    print(pd.Series(m.feature_importances_,X.columns).sort_values(ascending=False))\n\n    from sklearn.inspection import permutation_importance as pimp\n    def scorer(m,x,y): return roc_auc_score(y,m.predict_proba(x)[:,1])\n    imp=pimp(m,xv2,yv2,scoring=scorer,n_jobs=-1)\n    print(pd.DataFrame({'mean':imp['importances_mean'],\n                  'std':imp['importances_std']},xv2.columns).sort_values('mean',ascending=False))\n    \nif not submit: print_imp()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if submit:\n    import riiideducation\n    env=riiideducation.make_env()\n    test,sample=do_submit(env.iter_test(),env)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Archive"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # prev answer for same qn for same user (~20% of qns are answered at least twice)\n# # this gives around .005 of roc_auc increase (up to .007 w/ 'already' col)\n\n# cols=['user_id','content_id','timestamp','answered_correctly']\n# answers=data[cols].sort_values(cols[:-1],ignore_index=True)\n# answers.columns=cols[:-1]+['ans']\n# answers=answers.merge(answers,how='left',left_on=cols[:2]+[answers.index],\n#                       right_on=cols[:2]+[answers.index+1],suffixes=('','_lag'))\n# answers.drop(columns=['key_2','timestamp_lag','ans'],inplace=True)\n# answers['ans_lag']=answers.ans_lag.fillna(-1).astype('int8')\n# # answers['already']=~answers.ans_lag.isna()\n# # answers['ans_lag']=answers.ans_lag.fillna(0).astype('int8')\n# # print(answers.head())\n# data=data.merge(answers,how='left',on=['user_id','content_id','timestamp'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}