{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n# import datatable as dt\nimport lightgbm as lgb\nfrom matplotlib import pyplot as plt\nfrom collections import defaultdict\nfrom sklearn.preprocessing import LabelEncoder\n\nimport riiideducation\n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler, MultiLabelBinarizer\nimport gc\nfrom keras.layers import Reshape\nimport gensim\nimport pickle\nfrom keras import layers\nimport tensorflow as tf\nimport gc\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv =  pd.read_csv(\"../input/riiid-test-answer-prediction/train.csv\",low_memory=False, nrows=10**7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = train_csv[['timestamp','user_id','content_id','answered_correctly','prior_question_elapsed_time','prior_question_had_explanation']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nwith open(r\"../input/riiid-results/embedding_matrix.pkl\", \"rb\") as input_file:\n    embedding_matrix = pickle.load(input_file) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocess"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = train_csv[train_csv['answered_correctly'] != -1].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv['lag'] = train_csv.groupby('user_id')['answered_correctly'].shift()\ncum = train_csv.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])\ntrain_csv['user_correctness'] = cum['cumsum'] / cum['cumcount']\ntrain_csv.drop(columns=['lag'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_agg = train_csv.groupby('user_id')['answered_correctly'].agg(['sum', 'count'])\ncontent_agg = train_csv.groupby('content_id')['answered_correctly'].agg(['sum', 'count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = train_csv.groupby('user_id').tail(24).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv['content_count'] = train_csv['content_id'].map(content_agg['count']).astype('int32')\ntrain_csv['content_id_correctness'] = train_csv['content_id'].map(content_agg['sum'] / content_agg['count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv['prior_question_had_explanation'].fillna(False, inplace=True)\ntrain_csv['prior_question_elapsed_time'].fillna(0,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lencoder = LabelEncoder()\ntrain_csv['prior_question_had_explanation_enc'] = lencoder.fit_transform(train_csv['prior_question_had_explanation'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\ntrain_csv['prior_question_elapsed_time']=scaler.fit_transform(train_csv['prior_question_elapsed_time'].values.reshape(-1,1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix[45][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix[185]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def func(content,pos):\n    return embedding_matrix[content][pos]\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i = 0\nwhile(i<20):\n    train_csv['embed'+str(i)] = train_csv['content_id']\n    train_csv['embed'+str(i)] = train_csv['embed'+str(i)].apply(lambda x : func(x,i))\n    i = i+1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_csv['lag1'] = train_csv.groupby('user_id')['embed0'].shift()\n# cum = train_csv.groupby('user_id')['lag1'].agg(['cumsum', 'cumcount'])\n# train_csv['embed0_avg'] = cum['cumsum'] / cum['cumcount']\n# train_csv.drop(columns=['lag1'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i = 0\nwhile(i<20):\n    train_csv['lag'] = train_csv.groupby('user_id')['embed'+str(i)].shift()\n    cum = train_csv.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])\n    train_csv['embed_avg'+str(i)] = cum['cumsum'] / cum['cumcount']\n    train_csv.drop(columns=['lag'], inplace=True)\n    i = i+1\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ni = 0\nwhile(i<20):\n    train_csv['embed_avg'+str(i)][0] = 0  #to ask - how to fill first entry which has no history\n    i = i+1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#embed_avg - history avg of q embedding\n#embed - cuurent q embedding ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_csv = train_csv.groupby('user_id').tail(6)\ntrain_csv.drop(valid_csv.index, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_csv[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_csv.drop(columns = 'embed0_avg',inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n    'prior_question_elapsed_time',\n    'prior_question_had_explanation_enc',\n    'user_correctness',\n    'content_count',\n    'content_id_correctness',\n    'embed0', 'embed1', 'embed2',\n    'embed3', 'embed4', 'embed5', \n    'embed6', 'embed7', 'embed8', \n    'embed9','embed10', 'embed11',\n    'embed12', 'embed13', 'embed14',\n    'embed15','embed16', 'embed17',\n    'embed18', 'embed19',\n    'embed_avg0',\n    'embed_avg1', 'embed_avg2', 'embed_avg3', 'embed_avg4', 'embed_avg5',\n    'embed_avg6', 'embed_avg7', 'embed_avg8', 'embed_avg9', 'embed_avg10',\n    'embed_avg11', 'embed_avg12', 'embed_avg13', 'embed_avg14',\n    'embed_avg15', 'embed_avg16', 'embed_avg17', 'embed_avg18',\n    'embed_avg19'\n     ]\ntarget = 'answered_correctly'\n\nparams = {\n    'objective': 'binary',\n    'seed': 42,\n    'metric': 'auc',\n    'device' : 'gpu',\n    'learning_rate': 0.05,\n    'max_bin': 100,\n    'num_leaves': 80\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_data = lgb.Dataset(train_csv[features], label=train_csv[target])\nva_data = lgb.Dataset(valid_csv[features], label=valid_csv[target])\n\nmodel = lgb.train(\n    params, \n    tr_data, \n    num_boost_round=10000,\n    valid_sets=[tr_data, va_data], \n    early_stopping_rounds=10,\n    verbose_eval=50\n)\n\n# model.save_model(f'model.txt')\nlgb.plot_importance(model, importance_type='gain')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}