{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport random\nfrom tqdm import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn import datasets, linear_model\nfrom sklearn.linear_model import LinearRegression\nimport statsmodels.api as sm\nfrom scipy import stats","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"\ntrain_all = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',low_memory=False,nrows=10**6)\ntrain_q=train_all[train_all.content_type_id==0]\ntrain_q.fillna(0.0,inplace=True)\ntrain_q.prior_question_had_explanation=train_q.prior_question_had_explanation.astype(int)\ndtype = {'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','content_type_id': 'int8','answered_correctly':'int8'}\n\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3,4,7], dtype=dtype)\n\ntrain_q.head()\ng = sns.pairplot(train_q,\n                 vars = ['answered_correctly',#'content_id',\n                         'prior_question_elapsed_time' ,'prior_question_had_explanation'],\n                 kind='scatter',\n                 markers = '.')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%%time\n#dtype = {'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','content_type_id': 'int8','answered_correctly':'int8'}\n#train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3,4,7], dtype=dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_df = train_df[train_df.content_type_id == 0]\n\ntrain_df = train_df.sort_values(['timestamp'], ascending=True).reset_index(drop = True)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"raw","source":"train_df.shape"},{"metadata":{"trusted":true},"cell_type":"code","source":"n=train_df.shape\nn=n[0]\nn=n//10\ntrain1=train_df.iloc[0:n-1,:].copy()\ntrain2=train_df.iloc[n:2*n-1,:].copy()\ntrain3=train_df.iloc[2*n:3*n-1,:].copy()\ntrain4=train_df.iloc[3*n:4*n-1,:].copy()\ntrain5=train_df.iloc[4*n:5*n-1,:].copy()\ntrain6=train_df.iloc[5*n:6*n-1,:].copy()\ntrain7=train_df.iloc[6*n:7*n-1,:].copy()\ntrain8=train_df.iloc[7*n:8*n-1,:].copy()\ntrain9=train_df.iloc[8*n:9*n-1,:].copy()\ntrain10=train_df.iloc[9*n:10*n-1,:].copy()\n\nseparated=[train1,train2,train3,train4,train5,train6,train7,train8,train9,train10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ngroup = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values))\ncontent_id=train_df.content_id\nIDs=train_df.user_id\ndel train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\na,b=np.unique(group.iloc[0][0],return_counts=True)\nn=group.shape\nn=n[0]\nuser_ref=[]\nids=np.asarray(group.index)\nfor i in range(0,n):\n    temp=group.iloc[i][0]\n    unique, counts=np.unique(temp,return_counts=True)\n    del temp\n    types=len(unique)\n    all_q=sum(counts)\n    avg=group.iloc[i][1].mean()\n    temp=[ids[i],types,all_q,avg]\n    user_ref.append(temp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_ref=pd.DataFrame(user_ref, columns=['user_id', 'number_of_questions', 'number_of_total_tries','correct_rate'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntemp=separated[0].merge(user_ref,left_on='user_id', right_on='user_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nnew_trains=[]\nfor i in range(0,10):\n    new_trains.append(separated[0].merge(user_ref,left_on='user_id', right_on='user_id'))\n    del separated[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del new_trains","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"separated[0]['user_id'][1:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time \ntemp=user_ref[user_ref.user_id==separated[0].user_id]\ntemp=user_ref.iloc[content_id,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"k=10\nfor i in range(0,10):\n    temp=user_ref[user_ref.user_id==separated[i].user_id]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#question_df=pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\n#IDs=question_df.question_id\n#Set up tags\n#tags=question_df.tags\n#tags = tags.str.split().to_frame()\n#tags=tags.fillna(-1)\n#n=tags.shape\n#n=n[0]\n#tagsnew=[]\n#tags.iloc[0][0]\n#list(map(int, tags.iloc[0][0]))\n#count=0\n#for i in range(0,n):\n#    count=i\n#    temp=tags.iloc[i][0]\n#    if (not isinstance(temp,int)):\n#        temp=list(map(int, temp))\n#    else:\n#        temp=[temp]\n#    tagsnew.append(temp)\n    \n#tags = pd.DataFrame(tagsnew, columns=['tag1', 'tag2', 'tag3','tag4','tag5','tag6'])\n#tags=tags.fillna(-1).astype('int8')\n\n#Set up part\n#part=question_df.part\n#part=part.fillna(-1).astype('int8')\n\n#question_df=question_df[['question_id','part']]\n#question_df=question_df.fillna(-1).astype('int16')\n\n#question_df=pd.concat([IDs,part, tags], axis=1)\n#del part\n#del tags\n#del tagsnew","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%%time\n#question_df=question_df.iloc[content_id,:]\n#del content_id\n#IDs.to_csv('file.csv', index=False)\n#question_df.to_csv('file1.csv',index=False)\n#del IDs, question_df\n#del question_df\n#question_df=pd.concat([IDs,question_df],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%%time\n#n=content_id.shape\n#n=n[0]\n#question_df=question_df.iloc[content_id,:]\n#for i in range(0,n):\n#    question_df.iloc[i][0]=IDs[i]\n\n#del content_id\n#IDs.to_csv('file.csv', index=False)\n#question_df.to_csv('file1.csv',index=False)\n#del IDs, question_df\n#del question_df\n#question_df=pd.concat([IDs,question_df],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n=content_id.shape\nn=n[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IDs[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(IDs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(question_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ngroup2=question_df.groupby('question_id').apply(lambda r: (\n            r['part'].values\n#            r['tag1', 'tag2', 'tag3','tag4','tag5','tag6'].values\n))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n=group.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfinal_train=[]\ncount=-1\nfor i in range(0,n):\n  #  group.iloc[i][0]=len(group.iloc[i][0])\n #   group.iloc[i][1]=group.iloc[i][0].mean()\n#    group2.iloc[i][0]=len(group.iloc[i][0])\n    count=i\n    parts=np.zeros((7,), dtype=int)\n\n    unique, counts = np.unique(group2.iloc[i], return_counts=True)\n    for j in range(0,len(unique)):\n        index=unique[j]-1\n        value=unique[j]\n        parts[index]=value\n    \n    temp=np.append([len(group.iloc[i][0]),group.iloc[i][1].mean()],parts)\n    final_train.append(temp)\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group2.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_train=pd.DataFrame(final_train, columns=['number_of_questions', 'correct_rate', 'number_of_questions'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a = np.array([0, 3, 0, 1, 0, 1, 2, 1, 0, 0, 0, 0, 1, 3, 4])\nunique, counts = np.unique(a, return_counts=True)\nunique\n#counts","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}