{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-11-12T20:18:33.489387Z","iopub.status.busy":"2020-11-12T20:18:33.488604Z","iopub.status.idle":"2020-11-12T20:18:33.496641Z","shell.execute_reply":"2020-11-12T20:18:33.497299Z"},"papermill":{"duration":0.030544,"end_time":"2020-11-12T20:18:33.497482","exception":false,"start_time":"2020-11-12T20:18:33.466938","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nmode='kaggle'\nif mode=='kaggle':\n    import numpy as np # linear algebra\n    import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n    import psutil\n    # Input data files are available in the read-only \"../input/\" directory\n    # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n    import os\n    for dirname, _, filenames in os.walk('/kaggle'):\n        for filename in filenames:\n\n            print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nfrom sklearn import metrics \nimport random\nfrom tqdm import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dt_df = dt.fread(\"/kaggle/working/hist.csv\")\n# dt_df.to_jay(derivedDataLoc+\"hist.jay\")\n# del [dt_df]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-12T20:18:33.531603Z","iopub.status.busy":"2020-11-12T20:18:33.530786Z","iopub.status.idle":"2020-11-12T20:18:35.566915Z","shell.execute_reply":"2020-11-12T20:18:35.566231Z"},"papermill":{"duration":2.055742,"end_time":"2020-11-12T20:18:35.567039","exception":false,"start_time":"2020-11-12T20:18:33.511297","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\nmode='kaggle'\nimport os\n#import riiideducation\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pickle\nfrom sklearn.model_selection import train_test_split\n#from dataExploration import distReports,plotGrabh\n#import lightgbm as lgb\nimport gc\nimport time\nimport numpy as np\nimport sys\nimport copy\nfrom matplotlib import pyplot\n#!{sys.executable} -m pip install datatable --target /kaggle/working/\n#sys.path.append(\"/kaggle/input/packages/\")\n#import datatable as dt\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn import preprocessing\nimport random\n\nbaseLoc=\"/kaggle/input/\"\ninputLoc=\"/kaggle/input/riiid-test-answer-prediction/\"\nhistoryLoc=\"/kaggle/working/\"\nworkingLoc=\"/kaggle/working/\"\nderivedDataLoc=\"/kaggle/working/\"\n\nif mode!='kaggle':\n    sys.path.append(\"/home/pooja/PycharmProjects/mlRepository/mlutomation/myCodes/\")#availble only for local run\n    from funcs import stratified_sample,stratified_sample_report #availble only for local run\n    from commonFuncs import packing\n    inputLoc=\"/kaggle/input/\"\n    historyLoc=\"/kaggle/working/\"\n    workingLoc=\"/kaggle/working/\"\n    derivedDataLoc=\"/kaggle/working/\"\n    historyLoc=\"/home/pooja/PycharmProjects/Riiid/baseDatasets/history/\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef stratified_sample(df, strata, size=None, seed=None, keep_index=True):\n    '''\n    It samples data from a pandas dataframe using strata. These functions use\n    proportionate stratification:\n    n1 = (N1/N) * n\n    where:\n        - n1 is the sample size of stratum 1\n        - N1 is the population size of stratum 1\n        - N is the total population size\n        - n is the sampling size\n    Parameters\n    ----------\n    :df: pandas dataframe from which data will be sampled.\n    :strata: list containing columns that will be used in the stratified sampling.\n    :size: sampling size. If not informed, a sampling size will be calculated\n        using Cochran adjusted sampling formula:\n        cochran_n = (Z**2 * p * q) /e**2\n        where:\n            - Z is the z-value. In this case we use 1.96 representing 95%\n            - p is the estimated proportion of the population which has an\n                attribute. In this case we use 0.5\n            - q is 1-p\n            - e is the margin of error\n        This formula is adjusted as follows:\n        adjusted_cochran = cochran_n / 1+((cochran_n -1)/N)\n        where:\n            - cochran_n = result of the previous formula\n            - N is the population size\n    :seed: sampling seed\n    :keep_index: if True, it keeps a column with the original population index indicator\n\n    Returns\n    -------\n    A sampled pandas dataframe based in a set of strata.\n    Examples\n    --------\n    >> df.head()\n    \tid  sex age city\n    0\t123 M   20  XYZ\n    1\t456 M   25  XYZ\n    2\t789 M   21  YZX\n    3\t987 F   40  ZXY\n    4\t654 M   45  ZXY\n    ...\n    # This returns a sample stratified by sex and city containing 30% of the size of\n    # the original data\n    >> stratified = stratified_sample(df=df, strata=['sex', 'city'], size=0.3)\n    Requirements\n    ------------\n    - pandas\n    - numpy\n    '''\n    population = len(df)\n    size = __smpl_size(population, size)\n    tmp = df[strata]\n    tmp['size'] = 1\n    tmp_grpd = tmp.groupby(strata).count().reset_index()\n    tmp_grpd['samp_size'] = round(size / population * tmp_grpd['size']).astype(int)\n\n    # controlling variable to create the dataframe or append to it\n    first = True\n    for i in range(len(tmp_grpd)):\n        # query generator for each iteration\n        qry = ''\n        for s in range(len(strata)):\n            stratum = strata[s]\n            value = tmp_grpd.iloc[i][stratum]\n            n = tmp_grpd.iloc[i]['samp_size']\n\n            if type(value) == str:\n                value = \"'\" + str(value) + \"'\"\n\n            if s != len(strata) - 1:\n                qry = qry + stratum + ' == ' + str(value) + ' & '\n            else:\n                qry = qry + stratum + ' == ' + str(value)\n\n        # final dataframe\n        if first:\n            stratified_df = df.query(qry).sample(n=n, random_state=seed).reset_index(drop=(not keep_index))\n            first = False\n        else:\n            tmp_df = df.query(qry).sample(n=n, random_state=seed).reset_index(drop=(not keep_index))\n            stratified_df = stratified_df.append(tmp_df, ignore_index=True)\n\n    return stratified_df\n\n\ndef stratified_sample_report(df, strata, size=None):\n    '''\n    Generates a dataframe reporting the counts in each stratum and the counts\n    for the final sampled dataframe.\n    Parameters\n    ----------\n    :df: pandas dataframe from which data will be sampled.\n    :strata: list containing columns that will be used in the stratified sampling.\n    :size: sampling size. If not informed, a sampling size will be calculated\n        using Cochran adjusted sampling formula:\n        cochran_n = (Z**2 * p * q) /e**2\n        where:\n            - Z is the z-value. In this case we use 1.96 representing 95%\n            - p is the estimated proportion of the population which has an\n                attribute. In this case we use 0.5\n            - q is 1-p\n            - e is the margin of error\n        This formula is adjusted as follows:\n        adjusted_cochran = cochran_n / 1+((cochran_n -1)/N)\n        where:\n            - cochran_n = result of the previous formula\n            - N is the population size\n    Returns\n    -------\n    A dataframe reporting the counts in each stratum and the counts\n    for the final sampled dataframe.\n    '''\n    population = len(df)\n    size = __smpl_size(population, size)\n    tmp = df[strata]\n    tmp['size'] = 1\n    tmp_grpd = tmp.groupby(strata).count().reset_index()\n    tmp_grpd['samp_size'] = round(size / population * tmp_grpd['size']).astype(int)\n    return tmp_grpd\n\n\ndef __smpl_size(population, size):\n    '''\n    A function to compute the sample size. If not informed, a sampling\n    size will be calculated using Cochran adjusted sampling formula:\n        cochran_n = (Z**2 * p * q) /e**2\n        where:\n            - Z is the z-value. In this case we use 1.96 representing 95%\n            - p is the estimated proportion of the population which has an\n                attribute. In this case we use 0.5\n            - q is 1-p\n            - e is the margin of error\n        This formula is adjusted as follows:\n        adjusted_cochran = cochran_n / 1+((cochran_n -1)/N)\n        where:\n            - cochran_n = result of the previous formula\n            - N is the population size\n    Parameters\n    ----------\n        :population: population size\n        :size: sample size (default = None)\n    Returns\n    -------\n    Calculated sample size to be used in the functions:\n        - stratified_sample\n        - stratified_sample_report\n    '''\n    if size is None:\n        cochran_n = round(((1.96) ** 2 * 0.5 * 0.5) / 0.02 ** 2)\n        n = round(cochran_n / (1 + ((cochran_n - 1) / population)))\n    elif size >= 0 and size < 1:\n        n = round(population * size)\n    elif size < 0:\n        raise ValueError('Parameter \"size\" must be an integer or a proportion between 0 and 0.99.')\n    elif size >= 1:\n        n = size\n    return n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndtype = {'row_id':'int32','timestamp':'int64', \n         'user_id':'int32' ,\n         'content_id':'int16',\n         'content_type_id':'int8',\n         'answered_correctly':'int8'}\nfullMode=False\nif True:\n    if fullMode:train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[0,1, 2, 3, 4, 7], dtype=dtype)#,nrows=6000000)\n    else:train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[0,1, 2, 3, 4, 7], dtype=dtype,nrows=6000000)\n    #train_df = train_df.sort_values(['timestamp'], ascending=True).reset_index(drop = True)\n\n    train_df['timestamp']=train_df['timestamp']/1000\n    train_df['timestamp']=train_df['timestamp'].astype('int32')\n    maxTimestamp=train_df['timestamp'].max()\n    #print(maxTimestamp)\n    train_df['timestamp']=train_df['timestamp']/87425772\n    train_df['timestamp']=train_df['timestamp'].astype('float32')\n    gc.collect()\nif True:#adding addtional tags\n    q=pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/questions.csv\")\n    len(q['bundle_id'].unique())\n    q['tagM']=q['tags'].map(lambda x:str(x).split(\" \"))\n    q['ntag']=q['tagM'].map(lambda x:len(x))\n    q['ntag'].max()\n    for i in range(q['ntag'].max()):\n        q['tag'+str(i)]=q.apply(lambda x:int(x['tagM'][i]) if i<x['ntag'] and x['tagM'][i]!='nan' else 0  ,axis=1).astype('int16')\n        #q['tag'+str(i)]=q['tag'+str(i)].fillna(0)\n        #print(q['tag'+str(i)].max())\n        #print(i)\n    q.set_index('question_id',inplace =True)\n    \n    train_df=train_df.join(q.drop(['tags','tagM','ntag','correct_answer'],axis=1),'content_id')\n    for col in ['tag0','tag1','tag2','tag3','tag4','tag5','part','bundle_id']:\n        train_df[col]=train_df[col].fillna(0).astype('int16')\nprint(train_df.memory_usage(deep=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#q=pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/questions.csv\")\n\ndef createl2conv() :\n    l=pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/lectures.csv\")\n\n    #set(q['question_id']).intersection(set(l['lecture_id']))\n    l_list=l['lecture_id']\n    #q_list=l['question_id']\n    lectureDict1={}\n    i=0\n    for e in l_list:\n        lectureDict1[e]=14000+i\n        i+=1\n    del [l_list,l]\n    return lectureDict1\nlectureDict1=  createl2conv() \ndef lectureDict(x):\n    try : return lectureDict1[x]\n    except : return 14498\ntrain_df.loc[train_df[train_df.content_type_id == True].index,'content_id']=train_df[train_df.content_type_id == True]['content_id'].map(lectureDict)\ntrain_df[train_df['content_id']==32736]\ntrain_df[train_df.content_type_id == False].index\ntrain_df['content_id'].max()\ndel lectureDict1\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#external merge\n#from other notebook\nimport pandas as pd\n#train_df=train_df1\n#train_df=train_df.set_index('row_id')\nif False:\n    train_df.set_index('row_id',inplace=True)\n    train_df.sort_index(inplace=True)\n    if fullMode:extra=pd.read_csv(\"/kaggle/input/sakt-vivekmodified/extra_info.csv\",index_col='row_id')#,nrows=6000000)\n    else:extra = pd.read_csv(\"/kaggle/input/sakt-vivekmodified/extra_info.csv\",index_col='row_id',nrows=6000000)\n    extra.memory_usage(deep=True)\n    extra[['relevantLectattended','lect_count']]=extra[['relevantLectattended','lect_count']].astype('int16')\n    extra[['content_type_id']]=extra[['content_type_id']].astype('int8')\n    extra[['quest_roll_average_50']]=extra[['quest_roll_average_50']].astype('float32').fillna(0.5)\n    extra=extra#[extra['content_type_id']==0]\n\n    gc.collect()\n    print(\"used ram:\"+str(psutil.virtual_memory().percent))\n    #sanity check\n    #print((train_df.index-extra.index).unique()[0:10])\n    # print(train_df1.shape,extra.shape)\n    #print(len(set(train_df.index)-set(extra.index)))\n    # extra[extra.index==1835009]\n    #train_df1[train_df1.index==1835009]\n    #print(\"remainign ram:\"+str(psutil.virtual_memory().percent))\n    #spaceDict={}\n    for c in ['relevantLectattended','lect_count','quest_roll_average_50']:\n        train_df[c]=extra[c]/extra[c].max()\n        train_df[c]=train_df[c].astype('float16')\n        extra=extra.drop(c,axis=1)\n        gc.collect()\n        print('dom')\n    del extra\n    gc.collect()\n    #preprocessing extra\n    #r['lect_count'].values,r['relevantLectattended'].values,r['quest_roll_average_50'].values\n\n    #train_df['newIndex']=train_df['newIndex'].astype('int8')\n    #train_df.set_index('content_type_id',inplace=True)#will chnage\n    #train_df=train_df.drop('row_id',axis=1)\n    #train_df['content_id']=train_df['content_id'].astype('int16')\n    train_df=train_df.reset_index()\n    print(train_df.memory_usage(deep=True))\n    gc.collect()\n    #print(train_df.isna().sum())\n\n    # train_df.shape,extra.shape\n    # extra.content_type_id.unique()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-12T20:18:35.601655Z","iopub.status.busy":"2020-11-12T20:18:35.600955Z","iopub.status.idle":"2020-11-12T20:18:48.799897Z","shell.execute_reply":"2020-11-12T20:18:48.798964Z"},"papermill":{"duration":13.218292,"end_time":"2020-11-12T20:18:48.800075","exception":false,"start_time":"2020-11-12T20:18:35.581783","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"%%time\n\n\nif False:#tito sampling didn't work as it was was pushing lot of user all rows in validation smale\n    train = dt.fread(baseLoc+\"baseDatasets/train.jay\")\n    #train.to_jay(baseLoc+\"baseDatasets/train.jay\")\n    train=train.to_pandas()#[0:1000000]\n    #train=train.set_index('row_id')\n\n    user=train.groupby('user_id')['content_id'].count().reset_index()\n    user=user#.sample(frac=0.65)\n    train=train[train['user_id'].isin(user['user_id'])]\n    print(train.shape)\n    max_timestamp_u = train[['user_id','timestamp']].groupby(['user_id']).agg(['max']).reset_index()\n    max_timestamp_u.columns = ['user_id', 'max_time_stamp']\n    MAX_TIME_STAMP = max_timestamp_u.max_time_stamp.max()\n\n    #(MAX_TIME_STAMP for all users) - (max_time_stamp for each user) is used for this interval.\n\n    def rand_time(max_time_stamp):\n        interval = MAX_TIME_STAMP - max_time_stamp\n        rand_time_stamp = random.randint(0,interval)\n        return rand_time_stamp\n\n    max_timestamp_u['rand_time_stamp'] = max_timestamp_u.max_time_stamp.apply(rand_time)\n    train = train.merge(max_timestamp_u, on='user_id', how='left')\n    train['viretual_time_stamp'] = train.timestamp + train['rand_time_stamp']\n    train = train.sort_values(['viretual_time_stamp'])#.reset_index()\n\n    train=train.drop(['max_time_stamp','rand_time_stamp','viretual_time_stamp'],axis=1)\n    train['prior_question_had_explanation'] = train['prior_question_had_explanation'].astype('bool')\n    val_size =int( 0.2*train.shape[0])\n    valid = train[-val_size:]\n    train = train[:-val_size]\n    print(\"train,valid size:\" +str(train.shape[0])+\",\"+str(valid.shape[0]))\n    valid=valid[valid['content_type_id']==False]\n    train.sort_values(['user_id','timestamp']).to_csv(derivedDataLoc+\"train_tito.csv\",index=False)\n    user=valid.groupby('user_id')['content_id'].count().reset_index()\n    user1=user.sample(frac=0.7)\n    remUser=user.drop(user1.index,axis=0)#.sample(frac=0.6)\n    user2=remUser.sample(frac=0.6)\n    user3=remUser.drop(user2.index,axis=0)\n    \n    seg='user_id'\n    count=0\n    for ele in [user1,user2,user3]:\n        count=count+1\n        temp=valid[valid['user_id'].isin(ele['user_id'])]\n        temp.sort_values(['user_id','timestamp']).to_csv(derivedDataLoc+str(count)+\"valid_tito.csv\",index=False)\n        print(temp.shape)\n        \n        \n \n    del [train,valid,temp,user1,user2,user3,user,remUser]\n    _ = gc.collect()\nif True:#creating the user to row mapping\n    train= train_df#pd.read_pickle(\"../input/riiid-train-data-multiple-formats/riiid_train.pkl.gzip\")\n    train=train#[0:1000000]#\n    for segregator in ['user_id']:#\n\n        df=train.reset_index()\n        train=0\n        df=df[['user_id','row_id']]\n        first=df.drop_duplicates(subset=['user_id'],keep='first').rename(columns={'row_id':'start_row_id'}).set_index('user_id')\n        last=df.drop_duplicates(subset=['user_id'],keep='last').rename(columns={'row_id':'last_row_id'}).set_index('user_id')\n        comb=first.join(last)\n        comb=comb.sort_values(by=['start_row_id'])#.reset_index()\n        comb[['start_row_id','last_row_id']].to_csv(derivedDataLoc+segregator+\"_mapping.csv\")\n        del [train,df,comb,first,last]\n        #comb.to_dict(orient='index')\nif True:#vivek sampling method\n    #step1 :generating sample from user_id and rows to be considered for final sampling\n    train= train_df#pd.read_pickle(\"../input/riiid-train-data-multiple-formats/riiid_train.pkl.gzip\")\n    train=train#[0:1000000]\n    train=train.set_index('row_id')#[0:100000]\n    er=train[train.content_type_id==0].groupby('user_id')['content_id'].agg(['count'])\n    sample=train[train.content_type_id==0].sample(frac=0.15)\n    user=sample.groupby('user_id')['content_id'].agg(['count']).rename(columns={'count':'sampleCount'})\n    user=user.join(er)\n    user['strata']=user['count'].apply(lambda row: row if row<1000 else 1000*int(row/1000))\n    #user=user.set_index('user_id')\n    dev=stratified_sample(user, strata=['strata'], size=0.8, seed=123,keep_index=True).set_index('user_id')\n    valid=stratified_sample(user.drop(dev.index,axis=0), strata=['strata'], size=0.7, seed=123, keep_index=True).set_index('user_id')\n    oot=user.drop(dev.index,axis=0).drop(valid.index,axis=0)\n    count=1\n    del [er,sample,user]\n    for d in [dev,valid,oot]:\n        temp=d.reset_index().groupby('strata')['sampleCount'].agg(['mean','count']).rename(columns={'count':'count'+str(count),'mean':'mean'+str(count)})#.reset_index().sort_values(by='content_id',ascending=True)\n        if count==1: report=temp\n        else : report=report.join(temp)\n        count=count+1\n    report.to_csv(derivedDataLoc+\"samplingReport.csv\")\n    \n    var='user_id'\n    userMapping=pd.read_csv(derivedDataLoc+var+\"_mapping.csv\",index_col='user_id')\n    final=[]\n    count=0\n    name=['dev.csv','valid.csv','oot.csv']\n    for d in [dev,valid,oot]:#,valid,oot\n        rows=[]\n        d=d.join(userMapping)\n        d['firstRowSample']=d['last_row_id']-d['sampleCount']+1\n        d['range']=d.apply(lambda row:range(row['firstRowSample'],row['last_row_id']+1),axis=1)\n        startRow=d['firstRowSample'].to_list()\n        lastRow=d['last_row_id'].to_list()\n        for i in range(0,len(startRow)):\n            rows.extend([j for j in range(startRow[i],lastRow[i]+1)])\n        train.loc[rows].sort_values(['user_id','timestamp']).to_csv(derivedDataLoc+name[count])\n        print(name[count]+str(len(rows)))\n        count=count+1\n        train=train.drop(rows,axis=0)\n    print(\"hist\"+str(train.shape[0]))\n    train.sort_values(['user_id','timestamp']).to_csv(derivedDataLoc+'hist.csv')\n    del [train,dev,valid,oot,d]#\n#     dt_df = dt.fread(\"/kaggle/working/hist.csv\")\n#     dt_df.to_jay(derivedDataLoc+\"hist.jay\")\n#     del [dt_df]\n        \n        \n        \n    \nif True:\n    hist579=derivedDataLoc+\"hist.csv\"\n    dev579=derivedDataLoc+\"dev.csv\"\n    valid579=derivedDataLoc+\"valid.csv\"\n    oot579=derivedDataLoc+\"oot.csv\"\n    \n    \n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nif True:\n    d=[]\n    for ele in [hist579,dev579]:#,valid579,oot579\n        data=pd.read_csv(ele)\n        d.append(data)\n#train = dt.fread(baseLoc+\"baseDatasets/train.jay\")\n# train=train.to_pandas()#[0:100000]\n#len(set(d[0]['user_id'].unique()).intersection(set(d[1]['user_id'].unique()))),len(d[1]['user_id'].unique())\n#d[1].head()\n#d[2][d[2]['user_id']==480368]\ntrain_df=d[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndef breakinUsers(r):\n    return (r['content_id'].values,r['answered_correctly'].values,r['timestamp'].values,\n            #r['lect_count'].values,r['relevantLectattended'].values,r['quest_roll_average_50'].values,\n           r['tag0'].values,r['tag1'].values,r['tag2'].values,r['tag3'].values,r['tag4'].values,r['tag5'].values,\n           r['part'].values,r['bundle_id'].values)\ngroup = train_df.groupby('user_id').apply(breakinUsers)\n# group = train_df[['user_id', 'content_id', 'answered_correctly','timestamp']].groupby('user_id').apply(lambda r: (\n    \n#             r['content_id'].values,\n#             r['answered_correctly'].values,r['timestamp'].values))\ndel train_df##########################change\ngc.collect()\nlectureDict1=  createl2conv()\n#train_df[train_df['content_id']==13522].sort_values(['content_id','row_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SAKTDataset(Dataset):\n    def __init__(self, group, n_skill, max_seq=MAX_SEQ): #HDKIM 100\n        super(SAKTDataset, self).__init__()\n        self.max_seq = max_seq\n        self.n_skill = n_skill\n        self.samples = group\n        \n#         self.user_ids = [x for x in group.index]\n        self.user_ids = []\n        for user_id in group.index:\n            tup= group[user_id]\n            #q, qa ,_=tup[0],tup[1],tup[2]\n            if len(tup[0]) < 2: #HDKIM 10\n                continue\n            self.user_ids.append(user_id)\n            \n            #HDKIM Memory reduction\n            #if len(q)>self.max_seq:\n            #    group[user_id] = (q[-self.max_seq:],qa[-self.max_seq:])\n\n    def __len__(self):\n        return len(self.user_ids)\n\n    def __getitem__(self, index):\n        user_id = self.user_ids[index]\n        tup = self.samples[user_id]\n        feature=11#len(tup[])##################chnage\n        #q_, qa_,times=tup[0],tup[1],tup[2]\n        seq_len = len(tup[0])\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = -1*np.ones(self.max_seq, dtype=int)\n        outs=[q,qa]\n        for i in range(2,feature):\n            x=np.zeros(self.max_seq, dtype=float)\n            outs.append(x)\n        \n        \n        if seq_len >= self.max_seq:\n            #HDKIM\n            if random.random()>0.95:\n                start = random.randint(0,(seq_len-self.max_seq))\n                end = start + self.max_seq\n                for i in range(feature):\n                    #print(i)\n                    outs[i][:]= tup[i][start:end]\n\n            else:\n                for i in range(feature):\n                    #print(i)\n                    outs[i][:] = tup[i][-self.max_seq:]\n\n        else:\n            #HDKIM\n            if random.random()>0.95:\n                #HDKIMHDKIM\n                start = 0\n                end = random.randint(2,seq_len)\n                seq_len = end - start\n                for i in range(feature):\n                    #print(i)\n                    outs[i][-seq_len:] = tup[i][0:seq_len]\n\n            else:\n                for i in range(feature):\n                    #print(i)\n                    outs[i][-seq_len:] =  tup[i]\n \n\n        target_id = outs[0][1:]\n        label = outs[1][1:]\n        label,weights=np.where(label==-1,0,label),np.where(label==-1,0,1)\n        #print(np.unique(label))\n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[:-1].copy()\n        x += (qa[:-1] == 1) * self.n_skill\n#         t=outs[2][1:]\n#         lect_count=outs[3][1:]\n#         relevantLectattended=outs[4][1:]\n#         quest_roll_average_50=outs[5][1:]\n        ret=[x, target_id, label,weights]\n        for i in range(2,feature):\n            ret.append(outs[i][1:])\n            \n        \n        #`print(tuple(ret))\n        return tuple(ret)#x, target_id, label,weights,t,lect_count,relevantLectattended,quest_roll_average_50\n    #train_df[train_df['user_id']==115]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = SAKTDataset(group, n_skill)\ndataloader = DataLoader(dataset, batch_size=2048, shuffle=False, num_workers=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PositionalEncoding(nn.Module):\n\n    def __init__(self, d_model, dropout=0.1, max_len=5000):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n\n        pe = torch.zeros(max_len, d_model)\n        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(position * div_term)\n#         for i in range(pe.shape[1]):\n#             print(pe[:, i])\n        pe[:, 1::2] = torch.cos(position * div_term)\n        pe = pe.unsqueeze(0).transpose(0, 1)\n        self.register_buffer('pe', pe)\n\n    def forward(self, x):\n        #print(x.shape,self.pe.shape)\n        x = x + self.pe.permute(1, 0, 2)\n        return self.dropout(x)\n\nclass FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(0.2)\n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\nclass FFN0(nn.Module):\n    def __init__(self, state_size=28):\n        super(FFN0, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(4, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.layer_normal = nn.LayerNorm(state_size) \n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        x=self.layer_normal(x)\n        #print(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass SAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128): #HDKIM 100->MAX_SEQ\n        super(SAKTModel, self).__init__()\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n        self.extraFeature=9\n        adjung=48-3\n        self.embedding = nn.Embedding(2*n_skill+1, embed_dim-adjung)\n        self.pos_embedding = PositionalEncoding(128, max_len=159)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim-adjung)\n        self.t_embedding = nn.Embedding(200, 4)\n        self.p_embedding = nn.Embedding(10, 4)\n        self.b_embedding = nn.Embedding(n_skill+1, 16)\n\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)\n\n        self.dropout = nn.Dropout(0.2)\n        self.layer_normal = nn.LayerNorm(embed_dim)# +28\n        self.layer_normal2 = nn.LayerNorm(embed_dim+adjung)# +28\n\n        self.ffn = FFN(embed_dim)\n        self.ffn0 = FFN0()\n        self.pred = nn.Linear(embed_dim, 1)#+28\n    \n    def forward(self, x, question_ids,*args):\n        see=False\n        outs=[]\n        loop=0\n        for i in args:\n            if loop<1:outs.append(i.unsqueeze(2))\n            else:outs.append(i)\n            loop+=1\n       \n        #times=times.unsqueeze(2)\n        device = x.device\n        #print(x[0])\n        #print(question_ids[0])\n        if see:print(x.shape,question_ids.shape,23)\n        x = self.embedding(x)\n        e = self.e_embedding(question_ids)\n        #pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n        #print(x.shape,a.shape,b.shape,c.shape,d.shape)\n        if False:\n            y=torch.cat(\n                (outs[0],outs[1],outs[2],outs[3],), axis=2)\n            z=self.ffn0(y)\n            if see:print(x.shape,z.shape,9)\n        \n            \n        if True:\n            for i in range(self.extraFeature):\n                if i>=1:\n                    temp0 = torch.tensor(outs[i]).to(torch.int64)\n\n                if i==7:\n                    temp=self.p_embedding(temp0)\n                    #e = torch.cat((e,temp,), axis=2)\n                elif i==8:\n                    temp=self.b_embedding(temp0)\n                    #e = torch.cat((e,temp,), axis=2)\n                elif i>=1 :\n                    temp=self.t_embedding(temp0)\n                    #print(temp.shape,e.shape,24)\n                    #e = torch.cat((e,temp,), axis=2)\n                else :\n                    temp=outs[i]\n                x = torch.cat((x,temp,), axis=2)\n                e = torch.cat((e,temp,), axis=2)\n                \n                \n                \n                    \n        elif False:\n            x = torch.cat(\n                (x,z,), axis=2)\n            #print(x.shape,z.shape)\n            \n                \n#         else:\n#             x = torch.cat(\n#                 (x,outs[0],outs[1],outs[2],outs[3],), axis=2)#xself.pos_embedding(x)\n        #print(x[0][0])\n        #print(x.shape)\n        #x = x + pos_x\n        \n        \n        \n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = e.permute(1, 0, 2)\n        att_mask = future_mask(x.size(0)).to(device)\n        #print(x.shape,e.shape,234)\n        att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)\n        if see:print(att_output.shape,z.shape)\n        \n        #print(x[0][0])\n        if see:print(att_output.shape,z.shape,1)\n        att_output = self.layer_normal(att_output +e)#+additona+ e\n        if see:print(att_output.shape,z.shape,2)\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n        if see:print(att_output.shape,3)\n        if False:\n            att_output = torch.cat(\n                (att_output,y,), axis=2)\n        if see:print(y[0],12)#,att_output)\n        x = self.ffn(att_output)\n        if see:print(att_output.shape,x.shape,4)\n        #x = self.layer_normal2(x +att_output)#x \n        x = self.pred(x)\n        #print(att_weight[0][0])\n        \n        return x.squeeze(-1), att_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nn_skill =14500\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel=SAKTModel(n_skill, embed_dim=128)\n\nmodel.load_state_dict(torch.load(\"/kaggle/input/model3/SAKT-HDKIM (2).pt\",map_location=torch.device('cpu')))\nmodel.eval()\n\n# optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.99, weight_decay=0.005)\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\ncriterion = nn.BCEWithLogitsLoss()\n\nmodel.to(device)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(model, train_iterator, optim, criterion, device=\"cpu\"):\n    model.train()\n\n    train_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n\n    tbar = tqdm(train_iterator)\n    for item in tbar:\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n        weights=item[3].to(device).float()\n        extra=[]\n        for i in range(4,4+model.extraFeature):\n            extra.append(item[i].to(device).float())\n        extra=tuple(extra)\n        #print(weights.shape)\n\n        optim.zero_grad()\n        #print(extra)\n        output, atten_weight = model(x, target_id,*extra)\n        #print(output.shape, label.shape)\n        #weights=torch.tensor(np.zeros(shape=output.shape))\n        criterion = nn.BCEWithLogitsLoss(weight=weights)#weight=weights\n        criterion.to(device)\n        loss = criterion(output, label)#\n        loss.backward()\n        optim.step()\n        train_loss.append(loss.item())\n        #print(loss)\n        output = output[:, -1]\n        label = label[:, -1] \n        pred = (torch.sigmoid(output) >= 0.5).long()\n        \n        num_corrects += (pred == label).sum().item()\n        num_total += len(label)\n\n        labels.extend(label.view(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n\n        tbar.set_description('loss - {:.4f}'.format(loss))\n\n    acc = num_corrects / num_total\n    auc = roc_auc_score(labels, outs)\n    loss = np.mean(train_loss)\n\n    return loss, acc, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#BASE 6M 40 0.761 80 0.787  120 0.812 #new 40 0.762 80:0.785  120 0.808\nepochs =1 #HDKIM 20 with 1 million accounts epoch - 34 train_loss - 0.23 acc - 0.609 auc - 0.647\n#base on 6 million 20 epoch is 0.72 auc. added mask on irrelvant prefex sequesnce and auc is 0.739\n#on full daset add mask doesnt hav any difference\ndataset = SAKTDataset(group, n_skill)\ndataloader = DataLoader(dataset, batch_size=2048, shuffle=False, num_workers=8)\ndel group\nfor epoch in range(epochs):\n    loss, acc, auc = train_epoch(model, dataloader, optimizer, criterion, device)\n    print(\"epoch - {} train_loss - {:.2f} acc - {:.3f} auc - {:.3f}\".format(epoch, loss, acc, auc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#validation check\n#grp=group\n#group=grp.copy()\nif False:#validation data split\n    valids=[]\n    dev=d[1].sort_values('timestamp')\n    batch=int(dev.shape[0]/10)\n    for i in range(batch):\n        valids.append(dev[i*batch:(i+1)*batch])\n    \nif True:\n    grpLen=len(group[115])\n    #on all  new\n    #test_df=pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtype,nrows=10000000)[-100000:]\n    #om seggregated\n    prev_test_df=None\n    for test_df in valids:\n        if (prev_test_df is not None) :\n            prev_test_df = prev_test_df#[prev_test_df.content_type_id == False]\n            prev_group  = prev_test_df.groupby('user_id').apply(breakinUsers)\n        for prev_user_id in prev_group.index:\n            case=False\n            if prev_user_id in group.index :case=True\n            tup=[]\n            for i in range(grpLen):\n                if case:\n                    tup.append(np.append(group[prev_user_id][i],prev_group[prev_user_id][i]))\n                else:\n                    tup.append(prev_group[prev_user_id][i])\n\n            if len(group[prev_user_id][0])>MAX_SEQ:\n                for i in range(grpLen):tup[i]= tup[i][-MAX_SEQ:]\n            group[prev_user_id]=tuple(tup)\n\n    #prev_test_df = test_df.copy()\n        #test_df['timestamp']=test_df['timestamp']/maxTimestamp\n        test_df = test_df[test_df.content_type_id == False]\n        actual=test_df['answered_correctly']\n    \n    \n    #test_df.to_csv(str(loop)+\".csv\")\n    test_dataset = TestDataset(group, test_df, skills)\n    test_dataloader = DataLoader(test_dataset, batch_size=4096, shuffle=False)\n    \n    \n    outs = []\n \n    for item in tqdm(test_dataloader):\n        #print(\"done\")\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        #weights=item[3].to(device).float()\n        times=item[2].to(device).float()\n        \n\n        with torch.no_grad():\n            output, att_weight = model(x, target_id,times)\n        \n        \n        output = torch.sigmoid(output)\n        output = output[:, -1]\n\n        # pred = (output >= 0.5).long()\n        # loss = criterion(output, label)\n\n        # val_loss.append(loss.item())\n        # num_corrects += (pred == label).sum().item()\n        # num_total += len(label)\n\n        # labels.extend(label.squeeze(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n        \n    #print(type(outs),outs)\n    test_df['answered_correctly'] =  outs#[0.2 for c in outs ]\n    print(roc_auc_score(actual,test_df['answered_correctly'] ))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}