{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is a part of a series of copy or reproduction of below kernel  \nRef: https://www.kaggle.com/vettejeep/masters-final-project-model-lb-1-392  \n\nV3: Fix output format -> 2.533  \nV4: With Trim  "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport warnings\nimport traceback\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nimport scipy.signal as sg\nimport multiprocessing as mp\nfrom scipy.signal import hann\nfrom scipy.signal import hilbert\nfrom scipy.signal import convolve\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.preprocessing import StandardScaler\n\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\n\nfrom tqdm import tqdm_notebook\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/lanl-vettejeep-join\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = '../input/LANL-Earthquake-Prediction'\nTRAIN_DIR = '../input/lanl-vettejeep-join'\nTEST_DIR = '../input/lanl-vettejeep-join'\n# TRAIN_DIR = '../input/vettejeep-train-1'\n# TEST_DIR = '../input/vettejeep-test-1'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'), index_col='seg_id')\nsubmission = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'))\nscaled_train_X = pd.read_csv(os.path.join(TRAIN_DIR, 'scaled_train_X.csv'))\nscaled_test_X = pd.read_csv(os.path.join(TEST_DIR, 'scaled_test_X.csv'))\ntrain_y = pd.read_csv(os.path.join(TRAIN_DIR, 'train_y.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(submission.shape)\nprint(scaled_train_X.shape)\nprint(scaled_test_X.shape)\nprint(train_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# No need for '../input/lanl-vettejeep-join'\n# scaled_test_X.drop(['time_to_failure'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 21,\n         'min_data_in_leaf': 20,\n         'objective':'regression',\n         'learning_rate': 0.01,\n         'max_depth': 108,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.91,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.91,\n         \"bagging_seed\": 42,\n         \"metric\": 'mae',\n         \"lambda_l1\": 0.1,\n         \"verbosity\": -1,\n         \"random_state\": 42}\n\n# From my kernel\n# params = {'num_leaves': 51,\n#          'min_data_in_leaf': 10, \n#          'objective':'regression',\n#          'max_depth': -1,\n#          'learning_rate': 0.001,\n#          \"boosting\": \"gbdt\",\n#          \"feature_fraction\": 0.91,\n#          \"bagging_freq\": 1,\n#          \"bagging_fraction\": 0.91,\n#          \"bagging_seed\": 42,\n#          \"metric\": 'mae',\n#          \"lambda_l1\": 0.1,\n#          \"verbosity\": -1,\n#          \"nthread\": -1,\n#          \"random_state\": 42}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TEMP\n# n_fold = 8\n# folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)\n\n# for fold_, (trn_idx, val_idx) in enumerate(folds.split(scaled_train_X, train_y.values)):\n#         print('working fold %d' % fold_)\n#         strLog = \"fold {}\".format(fold_)\n#         print(trn_idx[:10])\n#         print(val_idx[:10])\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lgb_base_model():\n    maes = []\n    rmses = []\n    predictions = np.zeros(len(scaled_test_X))\n\n    n_fold = 8\n    folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)\n\n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"Feature\"] = scaled_train_X.columns\n\n    for fold_, (trn_idx, val_idx) in enumerate(folds.split(scaled_train_X, train_y.values)):\n        print('working fold %d' % fold_)\n        strLog = \"fold {}\".format(fold_)\n        print(strLog)\n\n        X_tr, X_val = scaled_train_X.iloc[trn_idx], scaled_train_X.iloc[val_idx]\n        y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]\n\n        model = lgb.LGBMRegressor(**params, n_estimators=80000, n_jobs=-1)\n        model.fit(X_tr, y_tr,\n                  eval_set=[(X_tr, y_tr), (X_val, y_val)], eval_metric='mae',\n                  verbose=1000, early_stopping_rounds=200)\n\n        # predictions\n        preds = model.predict(scaled_test_X, num_iteration=model.best_iteration_)\n        predictions += preds / folds.n_splits\n        preds = model.predict(X_val, num_iteration=model.best_iteration_)\n\n        # mean absolute error\n        mae = mean_absolute_error(y_val, preds)\n        print('MAE: %.6f' % mae)\n        maes.append(mae)\n\n        # root mean squared error\n        rmse = mean_squared_error(y_val, preds)\n        print('RMSE: %.6f' % rmse)\n        rmses.append(rmse)\n\n        fold_importance_df['importance_%d' % fold_] = model.feature_importances_[:len(scaled_train_X.columns)]\n\n    print('MAEs', maes)\n    print('MAE mean: %.6f' % np.mean(maes))\n    print('RMSEs', rmses)\n    print('RMSE mean: %.6f' % np.mean(rmses))\n\n    submission.time_to_failure = predictions\n    submission.to_csv('submission.csv', index=False)\n    fold_importance_df.to_csv('fold_imp.csv')  # index needed, it is seg id\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# lgb_base_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Verify\n# temp = pd.read_csv('submission.csv')\n# display(temp.head(5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature Reduction"},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 21,\n         'min_data_in_leaf': 20,\n         'objective':'regression',\n         'max_depth': 108,\n         'learning_rate': 0.001,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.91,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.91,\n         \"bagging_seed\": 42,\n         \"metric\": 'mae',\n         \"lambda_l1\": 0.1,\n         \"verbosity\": -1,\n         \"random_state\": 42}\n\n\ndef lgb_trimmed_model():\n    maes = []\n    rmses = []\n    tr_maes = []\n    tr_rmses = []\n\n    pcol = []\n    pcor = []\n    pval = []\n    \n    train_y = pd.read_csv(os.path.join(TRAIN_DIR, 'train_y.csv'))\n    y = train_y['time_to_failure'].values\n\n    for col in scaled_train_X.columns:\n        pcol.append(col)\n        pcor.append(abs(stats.pearsonr(scaled_train_X[col], y)[0]))\n        pval.append(abs(stats.pearsonr(scaled_train_X[col], y)[1]))\n\n    df = pd.DataFrame(data={'col': pcol, 'cor': pcor, 'pval': pval}, index=range(len(pcol)))\n    df.sort_values(by=['cor', 'pval'], inplace=True)\n    df.dropna(inplace=True)\n    df = df.loc[df['pval'] <= 0.05]\n\n    drop_cols = []\n\n    for col in scaled_train_X.columns:\n        if col not in df['col'].tolist():\n            drop_cols.append(col)\n\n    scaled_train_X.drop(labels=drop_cols, axis=1, inplace=True)\n    scaled_test_X.drop(labels=drop_cols, axis=1, inplace=True)\n\n    \n    predictions = np.zeros(len(scaled_test_X))\n    preds_train = np.zeros(len(scaled_train_X))\n\n    print('shapes of train and test:', scaled_train_X.shape, scaled_test_X.shape)\n\n    n_fold = 6\n    folds = KFold(n_splits=n_fold, shuffle=False, random_state=42)\n\n    for fold_, (trn_idx, val_idx) in enumerate(folds.split(scaled_train_X, train_y.values)):\n        print('working fold %d' % fold_)\n        strLog = \"fold {}\".format(fold_)\n        print(strLog)\n\n        X_tr, X_val = scaled_train_X.iloc[trn_idx], scaled_train_X.iloc[val_idx]\n        y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]\n\n        model = lgb.LGBMRegressor(**params, n_estimators=60000, n_jobs=-1)\n        model.fit(X_tr, y_tr,\n                  eval_set=[(X_tr, y_tr), (X_val, y_val)], eval_metric='mae',\n                  verbose=1000, early_stopping_rounds=200)\n\n        # model = xgb.XGBRegressor(n_estimators=1000,\n        #                                learning_rate=0.1,\n        #                                max_depth=6,\n        #                                subsample=0.9,\n        #                                colsample_bytree=0.67,\n        #                                reg_lambda=1.0, # seems best within 0.5 of 2.0\n        #                                # gamma=1,\n        #                                random_state=777+fold_,\n        #                                n_jobs=12,\n        #                                verbosity=2)\n        # model.fit(X_tr, y_tr)\n\n        # predictions\n        preds = model.predict(scaled_test_X)  #, num_iteration=model.best_iteration_)\n        predictions += preds / folds.n_splits\n        preds = model.predict(scaled_train_X)  #, num_iteration=model.best_iteration_)\n        preds_train += preds / folds.n_splits\n\n        preds = model.predict(X_val)  #, num_iteration=model.best_iteration_)\n\n        # mean absolute error\n        mae = mean_absolute_error(y_val, preds)\n        print('MAE: %.6f' % mae)\n        maes.append(mae)\n\n        # root mean squared error\n        rmse = mean_squared_error(y_val, preds)\n        print('RMSE: %.6f' % rmse)\n        rmses.append(rmse)\n\n        # training for over fit\n        preds = model.predict(X_tr)  #, num_iteration=model.best_iteration_)\n\n        mae = mean_absolute_error(y_tr, preds)\n        print('Tr MAE: %.6f' % mae)\n        tr_maes.append(mae)\n\n        rmse = mean_squared_error(y_tr, preds)\n        print('Tr RMSE: %.6f' % rmse)\n        tr_rmses.append(rmse)\n\n    print('MAEs', maes)\n    print('MAE mean: %.6f' % np.mean(maes))\n    print('RMSEs', rmses)\n    print('RMSE mean: %.6f' % np.mean(rmses))\n\n    print('Tr MAEs', tr_maes)\n    print('Tr MAE mean: %.6f' % np.mean(tr_maes))\n    print('Tr RMSEs', rmses)\n    print('Tr RMSE mean: %.6f' % np.mean(tr_rmses))\n\n    submission.time_to_failure = predictions\n    submission.to_csv('submission_2.csv')  # index needed, it is seg id\n\n    pr_tr = pd.DataFrame(data=preds_train, columns=['time_to_failure'], index=range(0, preds_train.shape[0]))\n    pr_tr.to_csv(r'preds_tr_xgb_slope_pearson_6fold.csv', index=False)\n    print('Train shape: {}, Test shape: {}, Y shape: {}'.format(scaled_train_X.shape, scaled_test_X.shape, train_y.shape))\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_trimmed_model()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}