{"cells":[{"metadata":{},"cell_type":"markdown","source":"********* Base Notebook *********\n\nLGBM BASELINE: https://www.kaggle.com/sishihara/riiid-lgbm-5cv-benchmark\n\nBASE FE: https://www.kaggle.com/lgreig/simple-lgbm-baseline\n\nMEMORY USAGE: https://www.kaggle.com/sishihara/riiid-pandas-memory-usage-analysis\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport time\nimport pickle\nimport gc\nfrom tqdm.notebook import tqdm\n\nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as catbst\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.gaussian_process import GaussianProcessClassifier\nfrom sklearn.gaussian_process.kernels import RBF\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\nfrom sklearn.neural_network import MLPClassifier\n\n\nimport riiideducation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# You can only call make_env() once, so don't lose it!\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\ndef reduce_mem_usage(df, use_float16=False, log_name=None):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in tqdm(df.columns):\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]) or str(df[col].dtype)=='timedelta64[ns]':\n            # skip datetime type or categorical type\n            continue\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    # if log_name is not None:\n    #     get_logger(log_name).info('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    #     get_logger(log_name).info('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    # else:\n    #     print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    #     print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    print(\n        'Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(\n        100 * (start_mem - end_mem) / start_mem))\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                       low_memory=False,\n                       nrows=10**7,\n                       usecols=[\n                           #'timestamp', \n                           'user_id', \n                           'content_id', \n                           #'content_type_id', \n                           'task_container_id', \n                           'user_answer', \n                           'answered_correctly', \n                           #'prior_question_elapsed_time', \n                           #'prior_question_had_explanation'\n                       ],\n                       dtype={\n                        #'row_id': 'int64',\n                        #'timestamp': 'int64',\n                        'user_id': 'int32',\n                        'content_id': 'int16',\n                        #'content_type_id': 'int8',\n                        'task_container_id': 'int16',\n                        'user_answer': 'int8',\n                        'answered_correctly': 'int8',\n                        #'prior_question_elapsed_time': 'float32',\n                        #'prior_question_had_explanation': 'boolean',\n                             }\n                      )\n\ntrain_df = reduce_mem_usage(train_df)\ntrain_df = train_df.query('answered_correctly != -1').reset_index(drop=True)\n#train_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = train_df['answered_correctly']\nX_train = train_df.drop(['answered_correctly', 'user_answer'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_FEATS = list(X_train.columns)\nTRAIN_FEATS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = [\n    #{'model_name': 'KNeighborsClassifier', 'model_instance': KNeighborsClassifier(3), 'model_weight': None}, # This performs low score\n    #{'model_name': 'SVC_linear', 'model_instance': SVC(kernel=\"linear\", C=0.025), 'model_weight': 0.05}, # This took much time\n    #{'model_name': 'SVC_gamma', 'model_instance': SVC(gamma=2, C=1), 'model_weight': None}, # This took much time\n    #{'model_name': 'GaussianProcessClassifier', 'model_instance': GaussianProcessClassifier(1.0 * RBF(1.0)), 'model_weight': None}, # This cause memory leak\n    #{'model_name': 'DecisionTreeClassifier', 'model_instance': DecisionTreeClassifier(max_depth=5), 'model_weight': None},\n    #{'model_name': 'GaussianProcessClassifier', 'model_instance': GaussianProcessClassifier(1.0 * RBF(1.0)), 'model_weight': None},# This cause memory leak\n    #{'model_name': 'RandomForestClassifier', 'model_instance': RandomForestClassifier(max_depth=5, n_estimators=100, max_features=1), 'model_weight': None},\n    #{'model_name': 'MLPClassifier', 'model_instance': MLPClassifier(alpha=1, max_iter=1000), 'model_weight': None}, # This performs low score\n    #{'model_name': 'AdaBoostClassifier', 'model_instance': AdaBoostClassifier(), 'model_weight': None},\n    #{'model_name': 'GaussianNB', 'model_instance': GaussianNB(), 'model_weight': None}, # This performs low score\n    #{'model_name': 'QuadraticDiscriminantAnalysis', 'model_instance': QuadraticDiscriminantAnalysis(), 'model_weight': None}, # This performs low score\n    #{'model_name': 'lgb', 'model_instance': None, 'model_weight': 0.3},\n    {'model_name': 'xgb', 'model_instance': None, 'model_weight': 0.5},\n    {'model_name': 'catbst', 'model_instance': None, 'model_weight': 0.5},\n]\n\n\ndef arrange_weightNone(models):\n    weights = [m['model_weight'] if m['model_weight'] else 0 for m in models]\n    sum_weights = sum(weights)\n    null_weights = weights.count(0)\n    if null_weights > 0:\n        res_weights = (1 - sum_weights) / null_weights\n        weights = [res_weights if w==0 else w for w in weights]\n        for i in range(len(models)):\n            models[i]['model_weight'] = weights[i]\n    return models\n\nmodels = arrange_weightNone(models)\nmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_model(model, save_path):\n    with open(save_path, 'wb') as f:\n        pickle.dump(model, f)\n    del model\n    gc.collect()\n    return save_path\n\ndef read_model(model_path):\n    with open(model_path, 'rb') as f:\n        model = pickle.load(f)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trained_models = {}\n\noof_train = np.zeros((len(X_train),))\nN_SPLITS = 5 \ncv = KFold(n_splits=N_SPLITS, shuffle=True, random_state=0)\n\ncategorical_features = [\n    'user_id', \n    'content_type_id', \n    'task_container_id', \n    'prior_question_had_explanation'\n]\ncategorical_features = [c for c in categorical_features if c in TRAIN_FEATS]\n\nPRINT_FEATURE_IMPORTANCE = True\n\nfor m in tqdm(models):\n    model_name = m['model_name']\n    model_weight = m['model_weight']\n    print (f'******************************************************************')\n    print (f'*********** MODEL = {model_name} (weight = {model_weight}) start')\n    start_time = time.time()\n    \n    trained_models_sub = []\n    oof_train_sub = np.zeros((len(X_train),))\n\n    for fold_id, (train_index, valid_index) in enumerate(cv.split(X_train)):\n        model_path = f'./{model_name}_{fold_id}.pkl'\n        \n        X_tr = X_train.iloc[train_index, :]\n        X_val = X_train.iloc[valid_index, :]\n        y_tr = y_train[train_index]\n        y_val = y_train[valid_index]\n\n        if model_name == 'lgb':\n            params = {\n                'objective': 'binary',\n                'metric': 'auc',\n                'learning_rate': 0.05,\n                \"max_depth\": 7,\n                \"min_data_in_leaf\": 50, \n                \"reg_alpha\": 0.1, \n                \"reg_lambda\": 1, \n                \"num_leaves\" : 31,\n                \"bagging_fraction\" : 0.8,\n                \"feature_fraction\" : 0.8,\n                'seed': 123,\n            }\n\n            dtrain = lgb.Dataset(X_tr, y_tr, categorical_feature=categorical_features)\n            dvalid = lgb.Dataset(X_val, y_val, reference=dtrain, categorical_feature=categorical_features)\n\n            model = lgb.train(\n                params, \n                dtrain,\n                valid_sets=[dtrain, dvalid],\n                verbose_eval=100,\n                num_boost_round=1000,\n                early_stopping_rounds=100\n            )\n\n            oof_train_sub[valid_index] = model.predict(X_val, num_iteration=model.best_iteration)\n            \n            if PRINT_FEATURE_IMPORTANCE:\n                feature_importance = sorted(zip(model.feature_name(), model.feature_importance(importance_type='gain')),key=lambda x: x[1], reverse=True)[:]\n                for i, item in enumerate(feature_importance[:]):\n                    print('Feature importance {}: {}'.format(i, str(item)))\n                #feature_importance = pd.DataFrame(feature_importance, columns=['feature', 'importance'])\n                \n            model_path = save_model(model, model_path)\n            trained_models_sub.append(model_path)\n            \n        elif model_name == 'xgb':\n            params = {\n                'booster': 'gbtree',\n                'task': 'train',\n                'eval_metric': 'auc',\n                'objective': 'binary:logistic',\n                'base_score': 0.5,\n                'learning_rate': 0.05,\n                'max_depth': 5,\n                'alpha': 1,\n                'lambda': 20,\n                'gamma': 0.1,\n                'colsample_bytree': 0.2, \n                'colsample_bynode': 1,\n                'colsample_bylevel': 0.3,\n                'subsample': 0.85,\n                'scale_pos_weight': 10,\n                'min_child_weight': 30,\n                'seed': 123\n            }\n            \n            dtrain = xgb.DMatrix(X_tr, label=y_tr)\n            dvalid = xgb.DMatrix(X_val, label=y_val)\n            \n            model = xgb.train(\n                params, \n                dtrain=dtrain,\n                evals=[(dtrain, 'train'),(dvalid, 'eval')],\n                verbose_eval=100,\n                num_boost_round=1000,\n                early_stopping_rounds=100\n            )\n            \n            oof_train_sub[valid_index] = model.predict(xgb.DMatrix(X_val), ntree_limit=model.best_iteration)\n            model_path = save_model(model, model_path)\n            trained_models_sub.append(model_path)\n            \n        elif model_name == 'catbst':\n            params = {\n                'bootstrap_type': 'Bayesian',\n                'boosting_type': 'Plain',\n                'objective': 'Logloss',\n                'eval_metric': 'AUC',\n                'num_boost_round': 1000,\n                'learning_rate': 0.05,\n                'max_depth': 9,\n                'colsample_bylevel': 0.1,\n                'bagging_temperature': 0.3,\n                'random_seed': 123,\n                'use_best_model': True,\n                'od_type': 'Iter',\n                'od_wait': 100,\n            }\n            \n            cat_feats_index = np.array([i for i in range(len(TRAIN_FEATS)) if TRAIN_FEATS[i] in categorical_features])\n            X_tr[categorical_features] = X_tr[categorical_features].astype(str)\n            X_val[categorical_features] = X_val[categorical_features].astype(str)\n            \n            dtrain = catbst.Pool(X_tr, label=y_tr)\n            dvalid = catbst.Pool(X_val, label=y_val)\n            \n            model = catbst.CatBoostClassifier(**params, cat_features=cat_feats_index)\n            model.set_feature_names(TRAIN_FEATS)\n            model = model.fit(\n                X_tr,\n                y_tr,\n                eval_set=(X_val, y_val),\n                verbose=100,\n                early_stopping_rounds=100,\n            )\n            \n            oof_train_sub[valid_index] = model.predict_proba(X_val)[:, 1]\n            model_path = save_model(model, model_path)\n            trained_models_sub.append(model_path)\n            \n        else:\n            model = m['model_instance']\n            X_tr = X_tr.fillna(0)\n            X_val = X_val.fillna(0)\n            model = model.fit(X_tr, y_tr)\n            if hasattr(model, \"decision_function\"):\n                oof_train_sub[valid_index] = model.decision_function(X_val)\n            else:\n                oof_train_sub[valid_index] = model.predict_proba(X_val)[:, 1]\n            model_path = save_model(model, model_path)\n            trained_models_sub.append(model_path)\n        \n    oof_score = roc_auc_score(y_train, oof_train_sub)\n\n    trained_models[model_name] = [trained_models_sub, float(model_weight)]\n    \n    oof_train += oof_train_sub * model_weight\n    \n    end_time = time.time()\n    training_time = round((end_time - start_time)/60, 2)\n    \n    print (f'*********** MODEL = {model_name} end: score = {oof_score}, training_time = {training_time} min')\n    print (f'******************************************************************')\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_score = roc_auc_score(y_train, oof_train)\nprint (f'*********** ALL MODELS ENSEMBLE: score = {oof_score}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# user_id_train_last = train_df.groupby('user_id').last()[['user_id', 'answered_correctly_user_id_cumsum','answered_correctly_user_id_cummean']]\n# content_id_train_last = train_df.groupby('content_id').last()[['content_id', 'answered_correctly_content_id_cumsum','answered_correctly_content_id_cummean']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    y_preds = []\n    #test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype(float)\n    #test_df = test_df.merge(user_id_train_last, on='user_id', how='left')\n    #test_df = test_df.merge(content_id_train_last, on='content_id', how='left')\n    X_test = test_df[TRAIN_FEATS]\n    \n    for model_name, models in trained_models.items():\n        model_weight = models[1]\n        for model_path in models[0]:\n            model = read_model(model_path)\n            if model_name == 'lgb':\n                y_pred = model.predict(X_test, num_iteration=model.best_iteration) * model_weight / len(models)\n                y_preds.append(y_pred)\n                \n            elif model_name == 'xgb':\n                y_pred = model.predict(xgb.DMatrix(X_test), ntree_limit=model.best_iteration) * model_weight / len(models)\n                y_preds.append(y_pred)\n                \n            elif model_name == 'catbst':\n                X_test[categorical_features] = X_test[categorical_features].astype(str)\n                y_pred = model.predict_proba(X_test)[:, 1] * model_weight / len(models)\n                y_preds.append(y_pred)\n                \n            else:\n                X_test = X_test.fillna(0)\n                if hasattr(model, \"decision_function\"):\n                    y_pred = model.decision_function(X_test) * model_weight / len(models)\n                else:\n                    y_pred = model.predict_proba(X_test)[:, 1] * model_weight / len(models)\n                y_preds.append(y_pred)\n\n    y_preds = sum(y_preds)\n    test_df['answered_correctly'] = y_preds\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}