{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\n\n# Training data is in the competition dataset as usual\niter_test = env.iter_test()\nset_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Some preprocessing and selecting of data to feed to the collaborative filetering- Using the last 50 questions of every user, hoping to catch their current understanding and level.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\ndtypes = {\n    \"row_id\": \"uint32\",\n    \"timestamp\": \"float32\",\n    \"user_id\": \"int32\",\n    \"content_id\": \"int16\",\n    \"content_type_id\": \"boolean\",\n    \"task_container_id\": \"int16\",\n    \"user_answer\": \"int8\",\n    \"answered_correctly\": \"int8\",\n    \"prior_question_elapsed_time\": \"float16\", \n    \"prior_question_had_explanation\": \"boolean\"\n}\n\ndata = pd.read_csv(\"../input/riiid-test-answer-prediction/train.csv\", dtype=dtypes)\ndata.drop(columns=['timestamp','user_answer'],inplace=True)\nprint(\"Train size:\", data.shape)\nimport numpy as np\nimport gc\nimport time\nimport fastai\nfrom fastai.tabular.all import *\nfrom fastai.collab import *\n\nfastai.__version__\n\n\n#MakeQuestions since Lecture\nstart = time.time()\n\nnpdat=data[['user_id','content_type_id','answered_correctly']].to_numpy()\nnpdat=np.append(npdat, np.zeros((npdat.shape[0],1),dtype=int), axis=1)\nnpdat=np.append(npdat, np.zeros((npdat.shape[0],1),dtype=int), axis=1)\n\nfor i in range(1,npdat.shape[0],1):\n    if npdat[i-1,0]==npdat[i,0]:\n        if npdat[i,1]==False :\n            npdat[i,3]=(npdat[i-1,3] +1)\n            npdat[i,4]=(npdat[i-1,3] * npdat[i-1,4] + npdat[i-1,2]*100)/npdat[i,3]\n        else:\n            npdat[i,3]=0\n            npdat[i,4]=0\n    else:\n        if npdat[i,1]==False :\n            npdat[i,3]=1\n            npdat[i,4]=0\n        else:\n            npdat[i,3]=0\n            npdat[i,4]=0\ndata['qsincel']=npdat[:,3].astype(np.uint8)\ndata['avgcsincel']=npdat[:,4].astype(np.uint8)\ndel npdat\ngc.collect()\n\nprint('Done qsincel in :',(time.time() - start) ,'s')\n\n\n#Make Tasks since last lecture\nstart = time.time()\nnpa=data[['user_id','task_container_id','content_type_id']].to_numpy()\nnpa=np.append(npa, np.zeros((npa.shape[0],1) ,dtype=int), axis=1)\n\nfor i in range(1,npa.shape[0],1):\n    if npa[i-1,0]==npa[i,0]:\n        if npa[i,1]==npa[i-1,1]:\n            if npa[i,2]==0:\n                npa[i,3]=npa[i-1,3]\n            else:\n                 npa[i,3]=0\n        else:\n            if npa[i,2]==0:\n                npa[i,3]=npa[i-1,3]+1\n            else :\n                npa[i,3]=0\n    else:\n        if npa[i,2]==0 :\n            npa[i,3]= 1\n        else:\n            npa[i,3]= 0         \ndata['tsincel']=npa[:,3].astype(np.uint8)\ndel npa\ngc.collect()\nprint('Done tsincel in :',(time.time() - start) ,'s')\n\n# dls = CollabDataLoaders.from_df(data[['user_id','content_id','answered_correctly']],item_name='content_id',valid_pct=0,bs=2048).cuda()\n# learn1 = collab_learner(dls, n_factors=3, y_range=(0, 1),use_nn= True).to_fp16()\n# learn1.fit_one_cycle(1, 5e-3,wd=1e-1)\n\n#divides the questions\n\n\nfrom sklearn.decomposition import PCA\n\n\nquestions = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/questions.csv\")\n\nlst = []\nfor tags in questions[\"tags\"]:\n    ohe = np.zeros(188)\n    if str(tags) != \"nan\":\n        for tag in tags.split():\n            ohe += np.eye(188)[int(tag)]\n    lst.append(ohe)\ntags_df = pd.DataFrame(lst, columns=[f\"tag_{i}\" for i in range(188)]).astype(int)\ndel lst,ohe\n#WITH PCA\n# from sklearn.cluster import KMeans,MiniBatchKMeans\n# import numpy as np\n# pca = PCA(n_components=35)\n# b=pca.fit_transform(tags_df.values)\n\n# q=pd.DataFrame(b)\n# q['content_id']=q.index\n# #tda=data.iloc[int(9e7):,3]\n# tda=data.sample(n=int(3e6))['content_id']\n# tdat=pd.merge(tda,q,how='left',on='content_id').dropna().iloc[:,1:36]\n# nptda=tdat.to_numpy()\n# kmeans = MiniBatchKMeans(n_clusters=5,\n#                           random_state=0,\n#                           batch_size=256).fit(nptda)\n# del tdat,nptda,tda,tags_df,b\n# gc.collect()\n# npq= q.iloc[:,0:35]\n# q['group']=kmeans.predict(npq).astype(np.int8)\n# rq=q.iloc[:,35:]\n# dat=pd.merge(data[data['content_type_id']==False],rq,how='left',on='content_id',copy=False)\n# del npq,q,data,kmeans,rq,questions\n# gc.collect()\n\n\n#WITHOUT PCA\ntags_df['content_id']=tags_df.index\nfrom sklearn.cluster import KMeans,MiniBatchKMeans\n# tda=data.iloc[int(8e7):,3]\ntda=data.sample(n=int(3e6))['content_id']\ntdat=pd.merge(tda,tags_df,how='left',on='content_id').dropna().iloc[:,1:]\nnptda=tdat.to_numpy()\nkmeans = MiniBatchKMeans(n_clusters=50,reassignment_ratio=0.09,\n                          random_state=1,\n                          batch_size=256).fit(nptda)\ndel tdat,nptda,tda\ngc.collect()\nnpq= tags_df.iloc[:,0:188]\ntags_df['group']=kmeans.predict(npq).astype(np.uint8)\nrq=tags_df.loc[:,['group','content_id']]\n\n#Merge to get Question Groups \n\ntrain=pd.merge(data[data['content_type_id']==False],rq,how='left',on='content_id')\ndel npq,data ,tags_df \ngc.collect()\n\nbatchsize= int(5e6)\n\ntrain['mobile_mean6']= np.zeros(train.shape[0], dtype=np.uint8)\ntrain['mobile_mean9']= np.zeros(train.shape[0], dtype=np.uint8)\ntrain['mobile_mean12']= np.zeros(train.shape[0], dtype=np.uint8)\n\n\nrn=0\ndone=0\nprint('Calculating mean, while dividing into batches')\nwhile done ==0:\n    n= min(rn+batchsize,train.shape[0]-1)\n    us=train.loc[n,'user_id']\n    cutoff= train[train['user_id']==us].reset_index()['index'].max()\n    print('starting on ', train.loc[rn:cutoff].shape,'data' )\n    train.loc[rn:cutoff,'mobile_mean9'] = train.loc[rn:cutoff].groupby('user_id').shift(+1).rolling(9)['answered_correctly'].mean()*100\n    train.loc[rn:cutoff,'mobile_mean6'] = train.loc[rn:cutoff].groupby('user_id').shift(+1).rolling(6)['answered_correctly'].mean()*100\n    train.loc[rn:cutoff,'mobile_mean12'] = train.loc[rn:cutoff].groupby('user_id').shift(+1).rolling(12)['answered_correctly'].mean()*100\n\n    \n\n    \n    rn=cutoff+1\n    if rn>=train.shape[0]:\n        print('end')\n        done=1\n    \n    \n        \n# train['mobile_mean6'].fillna(0.6,inplace= True)\n# train['mobile_mean6']=(train['mobile_mean6']*100).astype(np.int8)\n\ntrain['mobile_mean9'].fillna(0.6,inplace= True)\n# train['mobile_mean9']=(train['mobile_mean9']*100).astype(np.int8)\ntrain['mobile_mean6'].fillna(0.6,inplace= True)\n# train['mobile_mean6']=(train['mobile_mean6']*100).astype(np.int8)\ntrain['mobile_mean12'].fillna(0.6,inplace= True)\n\nA=train.groupby('user_id').tail(30).copy()\ndel train\ngc.collect()\ntrain= A\n\nlastrec= train.groupby('user_id').tail(1).copy()\n\ntu=train['user_id'].unique().tolist()\ntc=train['content_id'].unique().tolist()\nquestions['content_id']=questions['question_id']\ntrain=pd.merge(train,questions.loc[:,['content_id','part']],how='left',on='content_id')\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**COLLABORATICE FILTERING FAST AI, TRAIN MODEL **"},{"metadata":{"trusted":true},"cell_type":"code","source":"start = time.time()\nnfact=3\ncdls = CollabDataLoaders.from_df(train.loc[:,['user_id','content_id','answered_correctly']],\n                            item_name='content_id',valid_pct=0,bs= 4096)#8192   16384\ncoll = collab_learner(cdls, n_factors=nfact, y_range=(-0.1, 1.1),wd=1e-1)\ncoll .fit_one_cycle(2, 5e-3)\nprint('Trained model total time :',(time.time() - start),'s')\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**GETTING THE MODELS CREATED BIASES AND VARIABLES AND  CALCULATING PREDICTIONS MANUALLY- FASTER WAY THAN METHOD MODEL.GET_PREDICTS - WITCH IS TOO SLOW TO USE LATER ON THE PRIVATE TEST SET**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['collabPreds']= np.zeros(train.shape[0], dtype=np.int16)\ntrain['collabUbias']= np.zeros(train.shape[0], dtype=np.int8)\ntrain['collabCbias']= np.zeros(train.shape[0], dtype=np.int16)\n\nstart = time.time()\n\nstepsize= int(5e6)+1 # how many samples i would like to really process at a time\n\nprint('Doing part ')\nrn=0\ndone=0\n\n\ncollabPred,collabCbias,collabUbias = pd.Series(dtype=np.int8),pd.Series(dtype=np.int8),pd.Series(dtype=np.int8)\n\nbatch=train\nu=batch.loc[:,'user_id'].to_numpy()\nprint(u.shape)\nc=batch.loc[:,'content_id'].to_numpy()    \ncb=coll.model.bias(c,is_item=True)\ncw=coll.model.weight(c,is_item=True)   \n#         print('content handeld')\nub=coll.model.bias(u,is_item=False)\nprint(ub.shape)\nuw=coll.model.weight(u,is_item=False)\npre=torch.einsum('ij,in->i',cw,uw)\npre=pre\n\nfor x in range(0, nfact):\n    train[\"User_weight{0}\".format(x)] =uw[:,x].numpy()\n    \ncollabPred=collabPred.append((pd.Series(pre)*1000000).astype(np.int16)) \ncollabCbias=collabCbias.append((pd.Series(cb)*100).astype(np.int8)) \ncollabUbias= collabUbias.append((pd.Series(ub)*100*50).astype(np.int16)) \n\n#         nwtestdf.loc[:,'qsincel']\n#         a['row_id']=gdata.loc[rn:n,'row_id'].values\n#         a['collabPreds']=collabPred.values\n#         a['collabCbias']=collabCbias.values\n#         a['collabUbias']=collabUbias.values\n#         del collabPred , collabCbias , collabUbias ,u ,c,cb,cw, ub,uw,pre ,gdata\n#         gc.collect()\n#         train=pd.merge(train,a,how='left',on='row_id')\n#         print('merged 10 m')\n\n#         train.loc[train['group']==mn slice(rn:n) ,'collabPreds']=collabPred.values\n#         train.loc[idx[train['group']==mn ],idx[rn:n,'collabPreds']]=collabPred.values\nprint(train.loc[:,'collabPreds'].shape,collabPred.values.shape)\ntrain.loc[:,'collabPreds']=collabPred.values\ntrain.loc[:,'collabUbias']=collabUbias.values\ntrain.loc[:,'collabCbias']=collabCbias.values\nprint('step done results handeld, out of :',train.shape[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**TABULAR MODEL TO USE THE OUTPUTS OF THE COLLABORATIVE FILTERING TOGETHER WITH THE OTHER CREAATED DATA**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['prior_question_elapsed_time']=train['prior_question_elapsed_time'].clip(lower=0, upper=80000)\ntrain['answered_correctly']=train['answered_correctly'].astype(np.bool_)\ntrain['prior_question_had_explanation'].fillna(0,inplace=True)\ntrain.dropna(inplace=True)\n\nfeatures=['prior_question_had_explanation',\"answered_correctly\",'task_container_id',\n          'prior_question_elapsed_time',\\\n          'content_id',\n        'collabPreds',\n          'collabCbias','collabUbias',\\\n        'group','qsincel','avgcsincel','tsincel',\\\n          'User_weight0',\n          'User_weight1',\n          'User_weight2',\n#         'mobile_mean3','Exmobile_mean3',#'mobile_mean3Diff',\\\n        'mobile_mean6',#'Exmobile_mean6',#'mobile_mean6Diff',\\\n        'mobile_mean9',#'Exmobile_mean9',#'mobile_mean9Diff',\\\n        'mobile_mean12',#'Exmobile_mean12',#'mobile_mean12Diff'\\\n         'part',]\n\n\ndls = TabularDataLoaders.from_df(train[features],\n    cat_names = ['prior_question_had_explanation','group',\n                 #                 'mobile_mean3','Exmobile_mean3',#'mobile_mean3Diff',\\\n                'mobile_mean6',#'Exmobile_mean6',#'mobile_mean6Diff',\\\n                'mobile_mean9',#'mobile_mean9Diff',\\\n                'mobile_mean12',#'Exmobile_mean12',#'mobile_mean12Diff'\\\n                'part','content_id'],\n    cont_names = ['task_container_id','prior_question_elapsed_time',\\\n                'collabPreds','collabCbias','collabUbias',\n                  'qsincel','avgcsincel','tsincel',\n                'User_weight0',\n                  'User_weight1',\n                  'User_weight2'\n                 # 'Exmobile_mean9'\n                 ],\n    y_names=\"answered_correctly\",\n    bs=512,\n    procs = [Categorify, FillMissing, Normalize])\n\n# nn_learn = tabular_learner(ndls,metrics=accuracy,layers=[50,100,100],wd=1e-1)\n# nn_learn.fit_one_cycle(2, lr=5e-2,pct_start=0.2)\nlearn = tabular_learner(dls, metrics=[accuracy,RocAucBinary()],layers=[100,200,200,100],wd=1e-1)\n# learn = tabular_learner(dls,layers=[100,200,200,100],wd=1e-1)\nlearn.fit_one_cycle(1)\n# print ('Trained NN')\n\nlastrec= train.groupby('user_id').tail(1).copy()\n\ntu=train['user_id'].unique().tolist()\ntc=train['content_id'].unique().tolist()\n\nkuserweights= coll.model.weight(lastrec['user_id'].to_numpy(),is_item=False)\nlastrec.reset_index(drop=True,inplace=True)\nlastrec['u_index_inrec']=lastrec.index\n\ncontent_df=pd.DataFrame()\ncontent_df['content_id']=train['content_id'].unique()\ncontent_df['collabCbias']=(coll.model.bias(train['content_id'].unique(),is_item=True).numpy()*100).astype(np.int8)\nkcontentweights= coll.model.weight(train['content_id'].unique(),is_item=True)\ncontent_df.reset_index(drop=True,inplace=True)\ncontent_df['c_index_inrec']=content_df.index\n\ntrainmedpred= train['collabPreds'].mean()\ntrainmedub= train['collabCbias'].mean()\ntrainmedcb= train['collabUbias'].mean()\ntrainmedtaskcont= train[\"task_container_id\"].median()\ntrainmeduw0= train['User_weight0'].mean()\ntrainmeduw1= train['User_weight1'].mean()\ntrainmeduw2= train['User_weight2'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#TESTING\n# for i in range(1):\nfor (test, sample_prediction_df) in iter_test:        \n\n    s=test.copy()\n    if type(test)== pd.core.series.Series:\n        test=pd.DataFrame(test).T\n        \n    test=test.loc[test['content_type_id']==0,:].copy()\n\n    \n#     test[\"row_id\"]=test[\"row_id\"].astype(np.uint32)\n    test[\"timestamp\"]=test[\"timestamp\"].astype(np.uint32).fillna(0)\n    test[\"user_id\"]=test[\"user_id\"].astype(np.uint32).fillna(0)\n    test[\"content_id\"]=test[\"content_id\"].astype(np.uint32).fillna(0)\n    test[\"content_type_id\"]=test[\"content_type_id\"].astype(np.uint32).fillna(0)\n    test[\"task_container_id\"]=test[\"task_container_id\"].astype(np.uint32).fillna(trainmedtaskcont)\n    test[\"prior_question_elapsed_time\"]=test[\"prior_question_elapsed_time\"].fillna(0).clip(lower=0, upper=80000).astype(np.uint32)\n    test[\"prior_question_had_explanation\"]=test[\"prior_question_had_explanation\"].fillna(0).astype(np.uint32)\n    \n    #group & part\n    test=pd.merge(test[test['content_type_id']==False],rq,how='left',on='content_id')\n    test=pd.merge(test,questions.loc[:,['content_id','part']],how='left',on='content_id')\n    # c_index_inrec\n    test=pd.merge(test,content_df,how='left',on='content_id')\n    # u_index_inrec\n    test=pd.merge(test,lastrec.loc[:,['user_id','mobile_mean9','mobile_mean6','mobile_mean12','qsincel','tsincel','avgcsincel','collabUbias','u_index_inrec','User_weight0','User_weight1', 'User_weight2']],how='left',on='user_id')\n    test.loc[:,'qsincel']= test.loc[:,'qsincel']+1\n    test.loc[:,'tsincel']= test.loc[:,'tsincel']+1\n    test.loc[:,'avgcsincel']= test.loc[:,'avgcsincel'].fillna(0) \n    test.loc[:,'qsincel']= test.loc[:,'qsincel'].fillna(0) \n    test.loc[:,'tsincel']= test.loc[:,'tsincel'].fillna(0)\n    test.loc[:,'mobile_mean6']= test.loc[:,'mobile_mean6'].fillna(60)\n    test.loc[:,'mobile_mean9']= test.loc[:,'mobile_mean9'].fillna(60)\n    test.loc[:,'mobile_mean12']= test.loc[:,'mobile_mean12'].fillna(60)\n    \n    \n    #create pre\n    l=torch.Tensor(test.loc[test['u_index_inrec'].notnull()& test['c_index_inrec'].notnull(),['u_index_inrec']].u_index_inrec.values).to(dtype=torch.int64)\n    kutest_weight=torch.index_select(kuserweights,0,l)\n    l=torch.Tensor(test.loc[test['u_index_inrec'].notnull() & test['c_index_inrec'].notnull(),['c_index_inrec']].c_index_inrec.values).to(dtype=torch.int64)\n    kcontent_weight=torch.index_select(kcontentweights,0,l)\n    if len(kutest_weight) !=  0:         \n        test.loc[test['u_index_inrec'].notnull() & test['c_index_inrec'].notnull(),['collabPreds']]=torch.einsum('ij,in->i',kutest_weight,kcontent_weight).numpy() *1000000\n        test['collabPreds']=test['collabPreds'].fillna(trainmedpred).astype(np.int16)\n    else:\n        test['collabPreds']=trainmedpred.astype(np.int16)\n    #elimnulls and amake sure of type\n#     test['collabPreds']=test['collabPreds'].fillna(trainmedpred).astype(np.int16)\n    test['collabCbias']=test['collabCbias'].fillna(trainmedcb).astype(np.int8)\n    test['collabUbias']=test['collabUbias'].fillna(trainmedub).astype(np.int16)\n    test['User_weight0']=test['User_weight0'].fillna(trainmeduw0).astype(np.int16)\n    test['User_weight1']=test['User_weight1'].fillna(trainmeduw1).astype(np.int16)\n    test['User_weight2']=test['User_weight2'].fillna(trainmeduw2).astype(np.int16)\n\n    \n    test.loc[:,'prior_question_elapsed_time']=test['prior_question_elapsed_time'].fillna (0)\n    pfeatures = ['prior_question_had_explanation','task_container_id',\n                    'prior_question_elapsed_time',\\\n                    'collabPreds','collabCbias','collabUbias',\\\n                    'group','qsincel','avgcsincel','tsincel',\\\n                     'User_weight0',\n                      'User_weight1',\n                      'User_weight2',\n                 \n    #                     'mobile_mean3','Exmobile_mean3',#'mobile_mean3Diff',\\\n                    'mobile_mean6',#'Exmobile_mean6',#'mobile_mean6Diff',\\\n                    'mobile_mean9',#'Exmobile_mean9',#'mobile_mean9Diff',\\\n                    'mobile_mean12',#'Exmobile_mean12',#'mobile_mean12Diff'\\\n                     'part','content_id']\n\n    dl=dls.test_dl(test.loc[test['content_type_id']==0,pfeatures])\n\n    #             print(learn.predict(X_test_value[pfeatures])[2][1].item())\n    test.loc[test['content_type_id']==0,'answered_correctly']=(learn.get_preds(dl=dl,with_input=True, with_decoded=True)[1][:,1]).numpy()\n    test['answered_correctly']=test['answered_correctly'].fillna(0.7)\n    set_predict(test.loc[test['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}