{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Thanks!!\n\nThanks to tylerpike-san for this great script ( https://www.kaggle.com/tylerpike/riid-submission-notebook-simple-ensemble )\n\n## このnotebookの目的\n\nランクは狙わず、tagsをone-hotエンコードし、LGBMを回す。\nクロスバリデーションは、user_idをひとかたまりとし、そこからランダムにチョイスする。"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport random\nimport pandas as pd\nimport joblib\nimport os\nimport sys\npd.set_option('display.max_columns', 40)\nfrom collections import defaultdict\n#import datatable as dt\nimport lightgbm as lgb\nfrom matplotlib import pyplot as plt\nif os.path.exists(\"/kaggle/input/riiid-test-answer-prediction\"):\n    sys.path.append(\"/kaggle/input/riiid-test-answer-prediction\")\nimport riiideducation\nfrom sklearn.metrics import roc_auc_score\nimport gc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare training data"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Preparing training data...')\n\n# import training data\ndata_types_dict = {\n    'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    'content_type_id': 'int8',\n    'task_container_id': 'int16',\n    'user_answer': 'int8',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float16',\n    'prior_question_had_explanation': 'boolean'\n}\n\ntrain_df = pd.read_csv('../input/riiid-test-answer-prediction/train.csv', \n                   nrows=10**6,\n                   dtype = data_types_dict)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 一回も答えていないuser idを調べ、train_dfからdropする"},{"metadata":{"trusted":true},"cell_type":"code","source":"## user_idの数を確認\nuserid_counts = train_df[\"user_id\"].value_counts()\n\n## user_idの数が一つの行のuser_idをPickUp\nuser_ids_one_response = set(userid_counts[userid_counts == 1].index)\n\n## 一回しか出てこないuser_idをtrain_dfのindexに置き換え\nindex_one_response = set()\nfor i in user_ids_one_response:\n    [id] = train_df[train_df[\"user_id\"] == i].index.values\n    index_one_response.add(id)\n    \n## 一回しか出てこないuser_id をlist up したindexを使ってtrain_dfからdrop\nfor i in index_one_response:\n    train_df.drop(index=i, inplace=True)\n    \ntrain_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2回以上IDが出てくるuser_idのリストを作成\nこのリストの中からランダムに取得して、User_id単位で学習することを考える。"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_ids_set = set(train_df[\"user_id\"].values) - user_ids_one_response","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## questionsでtagをonehot エンコーディング\n\nデータ追加参考URL<br>\nhttps://qiita.com/studio_haneya/items/35951c56decd212ba41e\n\ndtype変換参考URL<br>\nhttps://note.nkmk.me/python-pandas-dtype-astype/"},{"metadata":{"trusted":true},"cell_type":"code","source":"## question.csv読み込み\nquestions_df = pd.read_csv(\n    '../input/riiid-test-answer-prediction/questions.csv', \n    usecols=[0, 1,3,4],\n    dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}\n)\n\n## tagだけのdf準備\nquestions_tags_df = pd.DataFrame()\nfor index, row in questions_df.iterrows():\n    for tag in [\"tag_\" + str(i) for i in str(row[\"tags\"]).split(\" \")]:\n        questions_tags_df.loc[index,tag] = int(1)\nquestions_tags_df.fillna(0,inplace=True)\nquestions_tags_df = questions_tags_df.astype(\"int8\")\n\n## questions_dfに、one-hotエンコーディングしたtagを追加し、不要になった\"tags\"をdrop\nquestions_df = pd.concat([questions_df,questions_tags_df],axis=1).drop(columns='tags')\nquestions_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## train_dfにquestion_dfをマージし、不要なものなnanの処理をし、train用dfの準備"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='lecture_id', how='left',right_index=True)\n\n## 不要な行データの削除\ntrain_df = train_df[train_df['answered_correctly'] != -1]\ntrain_df = train_df[train_df['content_type_id'] == 0]\n\n## Nan処理  \ntrain_df['prior_question_had_explanation'].fillna(False, inplace = True)\ntrain_df = train_df.replace([np.inf, -np.inf], np.nan)\ntrain_df['prior_question_elapsed_time'].fillna(0, inplace = True)\n\n# cast variables as factors (keep code in case needed)\ntrain_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].astype(int)\n\n## user setの準備\nuser_set = set(train_df[\"user_id\"].unique())\n\n\n## 学習で不要なcolumnの削除 \ntrain_df.drop(columns=['row_id', 'user_answer', 'question_id', 'task_container_id', 'content_id', 'content_type_id'], inplace = True)\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## training 実行"},{"metadata":{"trusted":true},"cell_type":"code","source":"N = 5\nmodels = list()\nfeatures = set(train_df.columns) - set([\"user_id\",\"answered_correctly\"])\n\nfor i in range(N):\n\n    ## userをランダムにチョイス\n    train_user_set = set(random.sample(user_set, int(len(user_set) * 0.8)))\n    varid_user_set = user_set - train_user_set\n\n    ## train_dfから、user_idが一致するものをPickUp\n    lgb_train = lgb.Dataset(\n        np.array(train_df[train_df[\"user_id\"].isin(train_user_set)].drop(columns=['user_id', 'answered_correctly'])),\n        label=np.array(train_df[train_df[\"user_id\"].isin(train_user_set)]['answered_correctly'])\n    )\n    \n    lgb_eval = lgb.Dataset(\n        np.array(train_df[train_df[\"user_id\"].isin(varid_user_set)].drop(columns=['user_id', 'answered_correctly'])),\n        label=np.array(train_df[train_df[\"user_id\"].isin(varid_user_set)]['answered_correctly']),\n        reference=lgb_train\n    )\n\n    params = {\n        'num_leaves': 200,\n        'max_bin':450,\n        'feature_fraction': 0.52,\n        'bagging_fraction': 0.52,\n        'objective': 'binary',\n        'learning_rate': 0.05,\n        \"boosting_type\": \"gbdt\",\n        \"metric\": 'auc'\n    }\n\n    model = lgb.train(\n        params, \n        lgb_train,\n        num_boost_round=5000,\n        valid_sets=[lgb_train, lgb_eval],\n        early_stopping_rounds=50,\n        verbose_eval=50\n    )\n\n    models.append(model)\n    \n    print(f\"\\nInfo :: {i} run finish!\\n\")\n    \nprint(\"\\nInfo :: All finished!!\\n\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test実行"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\n\nfor (test_df, sample_prediction_df) in iter_test:\n    # Merge in extra features\n    test_df = test_df.merge(questions_df, how = 'left', left_on = 'content_id', right_on = 'question_id')\n    \n    \n    ## 不要な行データの削除\n    #test_df = test_df[test_df['answered_correctly'] != -1]\n    test_df = test_df[test_df['content_type_id'] == 0]\n    \n    ## 提出データで必要なものをpickup\n    row_id = test_df['row_id']\n    content_type_id = test_df['content_type_id']\n\n    \n    ## Nan処理  \n    test_df['prior_question_had_explanation'].fillna(False, inplace = True)\n    test_df['prior_question_elapsed_time'].fillna(0, inplace = True)\n\n    # cast variables as factors (keep code in case needed)\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype(int)\n\n    ## 不要なcolumnの削除\n    drop_columns = ['user_id','row_id','question_id','task_container_id','content_id','content_type_id',\"prior_group_answers_correct\",\"prior_group_responses\"]\n    test_df.drop(columns=drop_columns,inplace=True)\n\n    ## 推論\n    sub_preds = np.zeros(test_df.shape[0])\n    for m in models:         \n        sub_preds += m.predict(test_df)\n        print(sub_preds)\n                 \n    test_df['answered_correctly'] = sub_preds / len(models)\n    test_df['answered_correctly'] = test_df['answered_correctly'].round().astype(\"int\")\n    print(test_df)\n\n    # Make predictions\n    test_df['row_id'] = row_id\n    test_df['content_type_id'] = content_type_id\n    \n    # Submit predictions\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create content group prior means\n# (https://www.kaggle.com/lgreig/simple-lgbm-baseline)\n# I have created this based on the entire training sample and import\n#     a saved copy of it now to save time\n#results_c = pd.read_csv('../input/riid-questions-priorscsv/question_priors.csv')\n#ansered_correctly_unconditional = results_c['answered_correctly_content'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create data regarding questions\n# (https://www.kaggle.com/jsylas/riiid-lgbm-starter)\n#Data_questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',\n#                             usecols=[0,1,3,4],\n#                             dtype={'question_id': 'int16','part': 'int8','bundle_id': 'int8','tags': 'str'})#\n#\n#tag = Data_questions[\"tags\"].str.split(\" \", n = 10, expand = True) \n#tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']\n\n#Data_questions =  pd.concat([Data_questions,tag],axis=1)\n#Data_questions['tags1'] = pd.to_numeric(Data_questions['tags1'], errors='coerce')\n#Data_questions['tags2'] = pd.to_numeric(Data_questions['tags2'], errors='coerce')\n#Data_questions['tags3'] = pd.to_numeric(Data_questions['tags3'], errors='coerce')\n#Data_questions['tags4'] = pd.to_numeric(Data_questions['tags4'], errors='coerce')\n#Data_questions['tags5'] = pd.to_numeric(Data_questions['tags5'], errors='coerce')\n#Data_questions['tags6'] = pd.to_numeric(Data_questions['tags6'], errors='coerce')\n\n#Data_questions['tags1'].fillna(0, inplace = True)\n#Data_questions['tags2'].fillna(0, inplace = True)\n#Data_questions['tags3'].fillna(0, inplace = True)\n#Data_questions['tags4'].fillna(0, inplace = True)\n#Data_questions['tags5'].fillna(0, inplace = True)\n#Data_questions['tags6'].fillna(0, inplace = True)\n\n#Data_questions.drop(['tags'], axis =1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Data_questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_test.csv', \n#                   nrows=10**3)\n#Data_questions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge data together\n#Data = Data.merge(results_c, on = 'content_id', how = 'left')\n#Data = Data.merge(Data_questions, how = 'left', left_on = 'content_id', right_on = 'question_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# remove invalid 'answered_correctly' entries and keep only questions\n#Data = Data[Data['answered_correctly'] != -1]\n#Data = Data[Data['content_type_id'] == 0]\n\n# impute missing values  \n#Data['prior_question_had_explanation'].fillna(False, inplace = True)\n#Data = Data.replace([np.inf, -np.inf], np.nan)\n#Data['prior_question_elapsed_time'].fillna(0, inplace = True)\n\n# drop unwanted columns \n#Data.drop(['user_id', 'row_id', 'user_answer', 'question_id', 'task_container_id',\n#           'content_id', 'content_type_id'], \n#          axis = 1, inplace = True)\n\n# cast variables as factors (keep code in case needed)\n#Data['prior_question_had_explanation'] = Data['prior_question_had_explanation'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create vector or Y\n# store correct answers for later\n#labels = np.array(Data['answered_correctly'])\n\n# Create matrix of regressors\n# store regressors\n#Data = Data.drop('answered_correctly', axis = 1)\n# store regressor names\n#feature_list = list(Data.columns)# Convert to numpy array\n# cast regressors as numpy array for scikit learn\n#Data = np.array(Data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Using Skicit-learn to split data into training and testing sets\n#from sklearn.model_selection import train_test_split \n\n# Split the data into training and testing sets\n#train_features, test_features, train_labels, test_labels = train_test_split(Data, \n#                                                                            labels, \n#                                                                            test_size = 0.25, \n#                                                                            random_state = 42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fit the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"#print('Fitting the model...')\n\n# Import the model we are using\n#from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier\n#from sklearn.neural_network import MLPClassifier\n\n# instantiate the random forest classifier\n#model_rf = RandomForestClassifier(n_estimators = 10,\n#                                  max_depth = 3,\n#                                  random_state =1)\n\n# instantiate the random forest classifier\n#model_gbm = GradientBoostingClassifier(n_estimators = 10,\n#                                       max_depth = 1,\n#                                       random_state =1)\n\n#model_nn = MLPClassifier(hidden_layer_sizes = [50,25],\n#                         random_state=1)\n\n# package classifiers together in voting machine\n#model_voting = VotingClassifier(estimators=[('rf',  model_rf), \n#                                            ('gbm', model_gbm),\n#                                            ('nn',  model_nn)], \n#                                voting = 'soft',\n#                                n_jobs = 4)\n\n# fit the voting machine\n#model_voting_fit = model_voting.fit(Data, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\n# Evaluate the fit of the models \n# predictions  \n#predictions = model_voting_fit.predict(test_features)\n# calcualte AUC\n#roc_auc_score(predictions, test_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print('Submitting official predictions...')\n\n#import riiideducation\n#env = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#iter_test = env.iter_test()\n\n#for (test_df, sample_prediction_df) in iter_test:\n    \n    # Merge in extra features\n#    test_df = test_df.merge(results_c, on = 'content_id', how = 'left')\n#    test_df = test_df.merge(Data_questions, how = 'left', left_on = 'content_id', right_on = 'question_id')\n   \n    # Data cleaning\n    ## impute missing values  \n#    test_df['prior_question_had_explanation'].fillna(False, inplace = True)\n#    test_df['prior_question_elapsed_time'].fillna(0, inplace = True)\n#    with pd.option_context('mode.use_inf_as_na', True):\n#        test_df = test_df.dropna(subset = ['prior_question_elapsed_time'], how = 'all')\n\n    \n#    test_df['answered_correctly_content'].fillna(ansered_correctly_unconditional, inplace = True)\n    \n#    test_df['bundle_id'].fillna(0, inplace = True)\n#    test_df['part'].fillna(0, inplace = True)\n#    test_df['tags1'].fillna(0, inplace = True)\n#    test_df['tags2'].fillna(0, inplace = True)\n#    test_df['tags3'].fillna(0, inplace = True)\n#    test_df['tags4'].fillna(0, inplace = True)\n#   test_df['tags5'].fillna(0, inplace = True)\n#    test_df['tags6'].fillna(0, inplace = True)\n    \n    ## drop unwanted columns \n#    row_id = test_df['row_id']\n#    content_type_id = test_df['content_type_id']\n#    test_df = test_df[feature_list]\n\n    ## cast variables as factors\n#    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype(int)\n\n    # Make predictions\n#    test_df['answered_correctly'] = model_voting_fit.predict_proba(test_df)[:,1]\n#    test_df['row_id'] = row_id\n#    test_df['content_type_id'] = content_type_id\n    \n#    # Submit predictions\n#    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}