{"cells":[{"metadata":{},"cell_type":"markdown","source":"- This Notebook is a continuation of [the notebook](https://www.kaggle.com/imazekishota/riiid-word2vec-with-content-id)\n- Base Notebook: [Riiid! LGBM bagging2](https://www.kaggle.com/zephyrwang666/riiid-lgbm-bagging2)"},{"metadata":{},"cell_type":"markdown","source":"# Import"},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = True","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport random\nimport pandas as pd\nimport datatable as dt\nimport gc\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict\n\nfrom gensim.models import Word2Vec, KeyedVectors\nimport lightgbm as lgb\nfrom matplotlib import pyplot as plt\nimport riiideducation\nfrom sklearn.metrics import roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N = 3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_types_dict = {\n    'timestamp': 'int64',\n    'user_id': 'int32', \n    'content_id': 'int16', \n    'content_type_id':'int8', \n    'task_container_id': 'int16',\n    #'user_answer': 'int8',\n    'answered_correctly': 'int8', \n    'prior_question_elapsed_time': 'float32', \n    'prior_question_had_explanation': 'bool'\n}\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = dt.fread('../input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# delete lecture rows\ntrain_df = train_df[~train_df['content_type_id']]\ntrain_df.drop('content_type_id', axis=1, inplace=True)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature Engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['word'] = train_df['content_id'] + train_df['answered_correctly'] * 100000 # 6digit is answered_correctly","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for n in range(1, N):\n    train_df[f'lag_word{n}'] = train_df.groupby('user_id')['word'].shift(n)\n    train_df[f'lag_word{n}'].fillna(-1, inplace=True)\n    train_df[f'lag_word{n}'] = train_df[f'lag_word{n}'].astype('int32')\n\n# sampling\nif DEBUG:\n    train_df=train_df[:10000]\nelse:\n    train_df = train_df[1200*10000:2*1200*10000]\n\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## word2vec Feature\n- It takes a lot of time to create this feature..."},{"metadata":{"trusted":true},"cell_type":"code","source":"wv = KeyedVectors.load_word2vec_format('../input/riiid-word2vec/vec.pt', binary=True)\n\ndef cosine_similarity(v1, v2):\n    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))\n\ndef apply_cosine_similarity(row, is_correct=True):\n#     print('lag_word1', row['lag_word1'])\n#     print(row['content_id'])\n#     print('content_id')\n    arr = []\n    for col in cols:\n        if row[col]==-1:\n            continue\n        else:\n            v = wv.get_vector(str(row[col]))\n            arr.append(v)\n\n    if is_correct:\n        word = str(row['content_id'] + 100000)\n    else:\n        word = str(row['content_id'])\n    \n    if len(arr) > 0 and word in wv.vocab:\n        v1 = np.mean(arr, axis=0)\n        v2 = wv.get_vector(word)\n        return cosine_similarity(v1, v2)\n    else:\n        return 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = []\nfor n in range(1, N):\n    cols.append(f'lag_word{n}')\n\n\ntrain_df[[\"cos_sim_correct\"]] = train_df[cols + ['content_id']].apply(lambda row: apply_cosine_similarity(row), axis=1)\ntrain_df[[\"cos_sim_incorrect\"]] = train_df[cols + ['content_id']].apply(lambda row: apply_cosine_similarity(row, is_correct=False), axis=1)\ntrain_df.drop(cols + ['word'], axis=1, inplace=True)\ntrain_df['cos_sim_correct'] = train_df['cos_sim_correct'].astype('float32')\ntrain_df['cos_sim_incorrect'] = train_df['cos_sim_incorrect'].astype('float32')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"features_dict = {\n    'timestamp':'float16',\n    'content_id':'int16',\n    'task_container_id':'int16',\n    'prior_question_elapsed_time': 'float32',\n    'prior_question_had_explanation': 'int16',\n    'cos_sim_correct': 'float32',\n    'cos_sim_incorrect': 'float32',\n}\n\ncategorical_columns= [\n    'content_id',\n    'task_container_id',\n]\n\nfeatures=list(features_dict.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"flag_lgbm=True\nclfs = list()\nparams = {\n'num_leaves': 300,\n'max_bin':450,\n'feature_fraction': 0.52,\n'bagging_fraction': 0.52,\n'objective': 'binary',\n'learning_rate': 0.05,\n\"boosting_type\": \"gbdt\",\n\"metric\": 'auc',\n}\ntrains=list()\nvalids=list()\nnum=1\nfor i in range(0,num):\n    \n    users=train_df['user_id'].drop_duplicates()\n    users=users.sample(frac=0.08)\n    users_df=pd.DataFrame()\n    users_df['user_id']=users.values\n    valid_df_newuser = pd.merge(train_df, users_df, on=['user_id'], how='inner',right_index=True)\n    del users_df\n    del users\n    gc.collect()\n\n    train_df.drop(valid_df_newuser.index, inplace=True)\n    valid_df=train_df.sample(frac=0.1)\n    train_df.drop(valid_df.index, inplace=True)\n   \n    valid_df = valid_df.append(valid_df_newuser)\n    del valid_df_newuser\n    gc.collect()\n\n    trains.append(train_df)\n    valids.append(valid_df)\n    print('train_df_clf length：',len(train_df))\n    print('valid_df length：',len(valid_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df\ndel valid_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(0,num):\n    X_train_np = trains[i][features].values.astype(np.float32)\n    X_valid_np = valids[i][features].values.astype(np.float32)\n    tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))\n    va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))\n\n    del trains, valids, X_train_np, X_valid_np\n    gc.collect()\n\n    model = lgb.train(\n        params, \n        tr_data,\n        num_boost_round=5000,\n        valid_sets=[tr_data, va_data],\n        early_stopping_rounds=50,\n        feature_name=features,\n        categorical_feature=categorical_columns,\n        verbose_eval=50\n    )\n    clfs.append(model)\n    print('Training done!!!')\n\n    fig,ax = plt.subplots(figsize=(15,15))\n    lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)\n    plt.show()\n\n    del tr_data, va_data\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}