{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm_notebook, tqdm_gui\n\nfrom catboost import CatBoostRegressor\nfrom sklearn.preprocessing import StandardScaler,LabelEncoder\nfrom sklearn.linear_model import LinearRegression, LassoCV\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold\npd.options.display.precision = 15\n\n%matplotlib inline\nimport lightgbm as lgb\nfrom xgboost.sklearn import XGBRegressor\nimport xgboost as xgb\nimport time\nimport datetime\nimport gc\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom scipy.signal import hilbert\nfrom scipy.signal import hann\nfrom scipy.signal import convolve\nfrom scipy import stats\n\nimport tsfresh\nfrom tsfresh import extract_features\nfrom tsfresh import select_features\nfrom tsfresh.utilities.dataframe_functions import impute\n\nprint('MODULES IMPORTED')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**FEATURES EXTRACTION**"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_scaled = pd.read_csv('../input/train_features.csv')\ny_train = pd.read_csv('../input/y_train.csv')\nX_test_scaled = pd.read_csv('../input/test_features.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_train_scaled.head())\nprint(y_train.head())\nprint(X_test_scaled.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_fold = 5\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=11)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(X=X_train_scaled, X_test=X_test_scaled, y=y_train, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):\n\n    oof = np.zeros(len(X))\n    prediction = np.zeros(len(X_test))\n    scores = []\n    feature_importance = pd.DataFrame()\n    for fold_n, (train_index, valid_index) in enumerate(folds.split(X)):\n        print('Fold', fold_n, 'started at', time.ctime())\n        X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n        y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n        \n        if model_type == 'lgb':\n            model = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1)\n            model.fit(X_train, y_train, \n                    eval_set=[(X_train, y_train), (X_valid, y_valid)], eval_metric='mae',\n                    verbose=10000, early_stopping_rounds=200)\n            \n            y_pred_valid = model.predict(X_valid)\n            y_pred = model.predict(X_test, num_iteration=model.best_iteration_)\n            \n        if model_type == 'lasso':\n            model = LassoCV(**params, n_jobs = -1)\n            model.fit(X_train, y_train)\n            \n            y_pred_valid = model.predict(X_valid).reshape(-1,)\n            score = mean_absolute_error(y_valid, y_pred_valid)\n            print(f'Fold {fold_n}. MAE: {score:.4f}.')\n            print('')\n            y_pred = model.predict(X_test)\n            \n        if model_type == 'xgb':\n            model = XGBRegressor(**params, n_estimators = 50000, n_jobs = -1)\n            model.fit(X_train, y_train, \n                    eval_set=[(X_train, y_train), (X_valid, y_valid)], eval_metric='mae',\n                    verbose=10000, early_stopping_rounds=200)\n            \n            y_pred_valid = model.predict(X_valid)\n            y_pred = model.predict(X_test)\n        \n    \n        \n        \n        oof[valid_index] = y_pred_valid.reshape(-1,)\n        scores.append(mean_absolute_error(y_valid, y_pred_valid))\n\n        prediction += y_pred    \n        \n        \n        if True:\n            # feature importance\n            fold_importance = pd.DataFrame()\n            fold_importance[\"feature\"] = X.columns\n            if model_type == 'lasso':\n                fold_importance[\"importance\"] = model.coef_\n            else :\n                fold_importance[\"importance\"] = model.feature_importances_\n            fold_importance[\"fold\"] = fold_n + 1\n            feature_importance = pd.concat([feature_importance, fold_importance], axis=0)\n            \n        \n\n    prediction /= n_fold\n    \n    print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))\n    \n    if plot_feature_importance:\n        feature_importance[\"importance\"] /= n_fold\n        if plot_feature_importance:\n            cols = feature_importance[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n                by=\"importance\", ascending=False)[:50].index\n\n            best_features = feature_importance.loc[feature_importance.feature.isin(cols)]\n\n            plt.figure(figsize=(16, 12));\n            sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False));\n            plt.title(model_type+' Features (avg over folds)');\n        \n            return oof, prediction, feature_importance\n        return oof, prediction\n    \n    else:\n        return oof, prediction","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**LG BOOST**"},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 128,\n          'min_data_in_leaf': 79,\n          'objective': 'huber',\n          'max_depth': -1,\n          'learning_rate': 0.01,\n          \"boosting\": \"gbdt\",\n          \"bagging_freq\": 5,\n          \"bagging_fraction\": 0.8126672064208567,\n          \"bagging_seed\": 11,\n          \"metric\": 'mae',\n          \"verbosity\": -1,\n          'reg_alpha': 0.1302650970728192,\n          'reg_lambda': 0.3603427518866501\n         }\noof_lgb, prediction_lgb, feature_importance_lgb = train_model(params=params, model_type='lgb', plot_feature_importance=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**XG BOOST**"},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_params = {'eta': 0.03,\n              'max_depth': 9,\n              'subsample': 0.85,\n              'objective': 'reg:linear',\n              'eval_metric': 'mae',\n              'silent': True,\n              'nthread': 4}\noof_xgb, prediction_xgb, feature_importance_xgb = train_model(X=X_train_scaled, X_test=X_test_scaled, params=xgb_params, model_type='xgb', plot_feature_importance=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**LASSO CV**"},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'cv':5}\noof_lasso, prediction_lasso, feature_importance_lasso = train_model(X=X_train_scaled, X_test=X_test_scaled, params=params, model_type='lasso', plot_feature_importance=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_importance_lgb.groupby([\"feature\"])[\"importance\"].mean().to_csv(\"lgb_importance.csv\")\nfeature_importance_xgb.groupby([\"feature\"])[\"importance\"].mean().to_csv(\"xgb_importance.csv\")\nfeature_importance_lasso.groupby([\"feature\"])[\"importance\"].mean().to_csv(\"lasso_importance.csv\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}