{"cells":[{"metadata":{"_uuid":"a9ef20fd4e6471fd2c9e3022edfdddc07a970aca"},"cell_type":"markdown","source":"## Thanks to this kernel:\n\n1. https://www.kaggle.com/fabiendaniel/elo-world\n2. https://www.kaggle.com/mfjwr1/simple-lightgbm-without-blending"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nimport warnings\nimport time\nimport sys\nimport datetime\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import BayesianRidge\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nfrom contextlib import contextmanager\nfrom pandas.core.common import SettingWithCopyWarning\nimport gc\nfrom tqdm import tqdm_notebook\nwarnings.simplefilter(action='ignore', category=SettingWithCopyWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"FEATS_EXCLUDED = ['first_active_month', 'target', 'card_id', 'outliers',\n                  'hist_purchase_date_max', 'hist_purchase_date_min', 'hist_card_id_size',\n                  'new_purchase_date_max', 'new_purchase_date_min', 'new_card_id_size',\n                  'OOF_PRED', 'month_0']\n@contextmanager\ndef timer(title):\n    t0 = time.time()\n    yield\n    print(\"{} - done in {:.0f}s\".format(title, time.time() - t0))\n\n# rmse\ndef rmse(y_true, y_pred):\n    return np.sqrt(mean_squared_error(y_true, y_pred))\n\n# One-hot encoding for categorical columns with get_dummies\ndef one_hot_encoder(df, nan_as_category = True):\n    original_columns = list(df.columns)\n    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n    df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)\n    new_columns = [c for c in df.columns if c not in original_columns]\n    return df, new_columns\n    \n# Display/plot feature importance\ndef display_importances(feature_importance_df_):\n    cols = feature_importance_df_[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:40].index\n    best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]\n\n    plt.figure(figsize=(8, 10))\n    sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\n    plt.title('LightGBM Features (avg over folds)')\n    plt.tight_layout()\n    plt.savefig('lgbm_importances.png')\n    \ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a940a9ce940fa489a5a5255f6d4523c9b7c3ffdc"},"cell_type":"code","source":"%%time\n# load csv\ndef load_data():\n    train_df = pd.read_csv('../input/elo-blending/train_feature.csv')\n    test_df = pd.read_csv('../input/elo-blending/test_feature.csv')\n    display(train_df.head())\n    display(test_df.head())\n    print(train_df.shape,test_df.shape)\n    train = pd.read_csv('../input/elo-merchant-category-recommendation/train.csv', index_col=['card_id'])\n    test = pd.read_csv('../input/elo-merchant-category-recommendation/test.csv', index_col=['card_id'])\n    train_df['card_id'] = train.index\n    test_df['card_id'] = test.index\n    train_df.index = train_df['card_id']\n    test_df.index = test_df['card_id']\n    display(train_df.head())\n    display(test.head())\n    print(train.shape,test.shape)\n    del train,test    \n    return (train_df,test_df)\n\nprint(gc.collect())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81da3c7b72a8a77db8180e10dff6c7952b5f849a"},"cell_type":"code","source":"# train_df,test_df = load_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf073c6a5dc22246b65f4a7a6d1c415c30b138c6"},"cell_type":"code","source":"boosting = [\"goss\",\"dart\"]\nboosting[0],boosting[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c0596e2e7f68c315c850f43ed0216cafe9ec4b9"},"cell_type":"code","source":"%%time\n\nboosting = [\"goss\",\"dart\"]\n# LightGBM GBDT with KFold or Stratified KFold\ndef kfold_lightgbm(train_df, test_df, num_folds, stratified = False, boosting = boosting[0]):\n    print(\"Starting LightGBM. Train shape: {}, test shape: {}\".format(train_df.shape, test_df.shape))\n\n    # Cross validation model\n    if stratified:\n        folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=326)\n    else:\n        folds = KFold(n_splits= num_folds, shuffle=True, random_state=2045)\n\n    # Create arrays and dataframes to store results\n    oof_preds = np.zeros(train_df.shape[0])\n    sub_preds = np.zeros(test_df.shape[0])\n    feature_importance_df = pd.DataFrame()\n    feats = [f for f in train_df.columns if f not in FEATS_EXCLUDED]\n\n    # k-fold\n    for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df[feats], train_df['outliers'])):\n        train_x, train_y = train_df[feats].iloc[train_idx], train_df['target'].iloc[train_idx]\n        valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['target'].iloc[valid_idx]\n\n        # set data structure\n        lgb_train = lgb.Dataset(train_x,label=train_y,free_raw_data=False)\n        lgb_test = lgb.Dataset(valid_x,label=valid_y,free_raw_data=False)\n\n        # params optimized by optuna\n        params ={\n                        'task': 'train',\n                        'boosting': 'goss',\n                        'objective': 'regression',\n                        'metric': 'rmse',\n                        'learning_rate': 0.005,\n                        'subsample': 0.9855232997390695,\n                        'max_depth': 8,\n                        'top_rate': 0.9064148448434349,\n                        'num_leaves': 87,\n                        'min_child_weight': 41.9612869171337,\n                        'other_rate': 0.0721768246018207,\n                        'reg_alpha': 9.677537745007898,\n                        'colsample_bytree': 0.5665320670155495,\n                        'min_split_gain': 9.820197773625843,\n                        'reg_lambda': 8.2532317400459,\n                        'min_data_in_leaf': 21,\n                        'verbose': -1,\n                        'seed':int(2**n_fold),\n                        'bagging_seed':int(2**n_fold),\n                        'drop_seed':int(2**n_fold)\n                        }\n\n        reg = lgb.train(\n                        params,\n                        lgb_train,\n                        valid_sets=[lgb_train, lgb_test],\n                        valid_names=['train', 'test'],\n                        num_boost_round=10000,\n                        early_stopping_rounds= 200,\n                        verbose_eval=100\n                        )\n\n        oof_preds[valid_idx] = reg.predict(valid_x, num_iteration=reg.best_iteration)\n        sub_preds += reg.predict(test_df[feats], num_iteration=reg.best_iteration) / folds.n_splits\n\n        fold_importance_df = pd.DataFrame()\n        fold_importance_df[\"feature\"] = feats\n        fold_importance_df[\"importance\"] = np.log1p(reg.feature_importance(importance_type='gain', iteration=reg.best_iteration))\n        fold_importance_df[\"fold\"] = n_fold + 1\n        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n        print('Fold %2d RMSE : %.6f' % (n_fold + 1, rmse(valid_y, oof_preds[valid_idx])))\n        del reg, train_x, train_y, valid_x, valid_y\n        gc.collect()\n\n    # display importances\n    display_importances(feature_importance_df)\n    \n        # save submission file\n    submission = pd.read_csv(\"../input/elo-merchant-category-recommendation/sample_submission.csv\")\n    submission['target'] = sub_preds\n    submission.to_csv(boosting+\".csv\", index=False)\n    display(submission.head())\n    return (submission)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a23af33845fc2d7da88b8eaab11daea986c22ad","scrolled":false},"cell_type":"code","source":"%%time\ntrain_df,test_df = load_data()\nprint(gc.collect())\nsubmission = kfold_lightgbm(train_df, test_df, num_folds=7, stratified=False, boosting=boosting[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb8de35a58fec93d4db148d2e324e15e4c36720c"},"cell_type":"code","source":"submission1 = kfold_lightgbm(train_df, test_df, num_folds=7, stratified=False, boosting=boosting[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9267a526dbbf7109a21bf0df6a8da029b9e7370e"},"cell_type":"code","source":"final = pd.read_csv(\"../input/elo-merchant-category-recommendation/sample_submission.csv\")\nfinal['target'] = submission['target'] * 0.5 + submission1['target'] * 0.5\nfinal.to_csv(\"blend.csv\",index = False)  ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}