{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Importing lib"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.feature_selection import SelectKBest,f_regression\nfrom sklearn.pipeline import Pipeline\n\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import GradientBoostingRegressor\nimport xgboost as xgb\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#  Loading & Preparing data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/train.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = train.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# making segments data into one data frame\ndef make_data():\n    global data\n    n = 0\n    for  i in data.segment_id :\n        file = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/train/{}.csv'.format(i))\n        for x in file.columns:\n            data.loc[n:n+1,x+'_mean'] = file[x].mean()\n            data.loc[n,x+'_std'] = file[x].std()\n            data.loc[n,x+'_min'] = file[x].min()\n            data.loc[n,x+'_20'] = file[x].quantile(0.20)\n            data.loc[n,x+'_40'] = file[x].quantile(0.40)\n            data.loc[n,x+'_50'] = file[x].quantile(0.50)\n            data.loc[n,x+'_60'] = file[x].quantile(0.60)\n            data.loc[n,x+'_80'] = file[x].quantile(0.80)\n            data.loc[n,x+'_max'] = file[x].max()\n        n +=1\n\n    data.fillna(0,inplace=True)\n    return data\nmake_data()    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train = data.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = data_train['time_to_eruption']\nsegment_id = data_train['segment_id']\ndata_train.drop(['time_to_eruption','segment_id'],axis = 1 , inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# finding and removing constant columns\ncols = []\nfor i in data_train.columns :\n    if data_train[i].min() == data_train[i].max():\n        cols.append(i)\ncols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train.drop(['sensor_1_50', 'sensor_2_50', 'sensor_3_50','sensor_4_50', 'sensor_5_50', 'sensor_6_50',\n                 'sensor_7_50', 'sensor_8_50', 'sensor_9_50', 'sensor_10_50'],axis = 1 , inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"std_scaler = StandardScaler()\nstd_data = std_scaler.fit_transform(data_train)\ndata_train = pd.DataFrame(std_data, columns = data_train.columns)\ndata_train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Models"},{"metadata":{},"cell_type":"markdown","source":"### 1. LinearRegression"},{"metadata":{"trusted":true},"cell_type":"code","source":"reg_model = LinearRegression()\nreg_model.fit(data_train,y_train)\nprint(reg_model.score(data_train,y_train))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2. RandomForestRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"forest_model = RandomForestRegressor()\nforest_model.fit(data_train,y_train)\nprint(forest_model.score(data_train,y_train))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"forest_score = cross_val_score(forest_model,data_train,y_train,scoring= 'neg_mean_squared_error', cv=20)\nforest_rmse_score = np.sqrt(-forest_score)\nprint(forest_rmse_score.mean())\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3. DecisionTreeRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"tree_model = DecisionTreeRegressor()\ntree_model.fit(data_train,y_train)\nprint(tree_model.score(data_train,y_train))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tree_score = cross_val_score(tree_model,data_train,y_train,scoring= 'neg_mean_squared_error', cv=20)\ntree_rmse_score = np.sqrt(-tree_score)\nprint(tree_rmse_score.mean())\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4. SVR"},{"metadata":{"trusted":true},"cell_type":"code","source":"svr_model = SVR()\nsvr_model.fit(data_train,y_train)\nprint(svr_model.score(data_train,y_train))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5. GradientBoostingRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"gbr_model = GradientBoostingRegressor()\ngbr_model.fit(data_train,y_train)\nprint(gbr_model.score(data_train,y_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gbr_score = cross_val_score(gbr_model,data_train,y_train,scoring= 'neg_mean_squared_error', cv=20)\ngbr_rmse_score = np.sqrt(-gbr_score)\nprint(gbr_rmse_score.mean())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 6. xgb"},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_model = xgb.XGBRegressor()\nxgb_model.fit(data_train,y_train)\nprint(xgb_model.score(data_train,y_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_score = cross_val_score(xgb_model,data_train,y_train,scoring= 'neg_mean_squared_error', cv=20)\nxgb_rmse_score = np.sqrt(-xgb_score)\nprint(xgb_rmse_score.mean())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 7. lgb"},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_model = lgb.LGBMRegressor()\nlgb_model.fit(data_train,y_train)\nprint(lgb_model.score(data_train,y_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_score = cross_val_score(lgb_model,data_train,y_train,scoring= 'neg_mean_squared_error', cv=20)\nlgb_rmse_score = np.sqrt(-lgb_score)\nprint(lgb_rmse_score.mean())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# FINE TUNE THE MODELS\n"},{"metadata":{},"cell_type":"markdown","source":"### 1. RandomForestRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"forest_tune_pipeline = Pipeline([\n     ('selector',SelectKBest(f_regression)),\n     ('model',RandomForestRegressor(random_state = 42))])\n\nforest_grid_search = GridSearchCV( estimator = forest_tune_pipeline, param_grid = {'selector__k':[70] , \n  'model__n_estimators':np.arange(500,550,50),'model__max_depth':[15]}, n_jobs=-1,\n                                  scoring=\"neg_mean_squared_error\", cv=20, verbose=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"forest_grid_search.fit(data_train,y_train)\nprint('the best parameters : ',forest_grid_search.best_params_)\nprint('the best score = ', np.sqrt(-forest_grid_search.best_score_))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2. DecisionTreeRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"tree_tune_pipeline = Pipeline([\n     ('selector',SelectKBest(f_regression)),\n     ('model',DecisionTreeRegressor(random_state = 42))])\n\ntree_grid_search = GridSearchCV( estimator = tree_tune_pipeline, param_grid = {'selector__k':[70,80] , \n     'model__max_depth':[12,13,15]}, n_jobs=-1,scoring=\"neg_mean_squared_error\", cv=20, verbose=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tree_grid_search.fit(data_train,y_train)\nprint('the best parameters : ',tree_grid_search.best_params_)\nprint('the best score = ', np.sqrt(-tree_grid_search.best_score_))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3. xgb"},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_tune_pipeline = Pipeline([\n     ('selector',SelectKBest(f_regression)), ('model',xgb.XGBRegressor(random_state=42))])\n\nxgb_grid_search = GridSearchCV( estimator = xgb_tune_pipeline, param_grid = {'selector__k':[70] , \n  'model__learning_rate':[0.05],'model__n_estimators':[3000,5000],'model__max_depth':[10],'model__colsample_bytree':[0.3]},\n                               n_jobs=-1, scoring=\"neg_mean_squared_error\", cv=20, verbose=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_grid_search.fit(data_train,y_train)\nprint('the best parameters : ',xgb_grid_search.best_params_)\nprint('the best score = ', np.sqrt(-xgb_grid_search.best_score_))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4. lgb"},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_tune_pipeline = Pipeline([\n     ('selector',SelectKBest(f_regression)),\n     ('model',lgb.LGBMRegressor(random_state=42,objective='regression',\n                              bagging_fraction = 0.8,\n                              bagging_freq = 5, feature_fraction = 0.2319,\n                              feature_fraction_seed=9, bagging_seed=9,\n                              min_data_in_leaf =6, min_sum_hessian_in_leaf = 11))])\n\nlgb_grid_search = GridSearchCV( estimator = lgb_tune_pipeline, param_grid = {'selector__k':[70] , \n  'model__learning_rate':[0.01],'model__num_iterations':[10000],'model__n_estimators':[500],\n                            'model__max_bin':[100],'model__num_leaves':[40,50]},\n                               n_jobs=-1, scoring=\"neg_mean_squared_error\", cv=20, verbose=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_grid_search.fit(data_train,y_train)\nprint('the best parameters : ',lgb_grid_search.best_params_)\nprint('the best score = ', np.sqrt(-lgb_grid_search.best_score_))\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Summary\nthe most algorithm that worked well is **lgb** then **xgb** then **RandomForestRegressor** then **DecisionTreeRegressor**"},{"metadata":{},"cell_type":"markdown","source":"# Final Model for Test set\n"},{"metadata":{},"cell_type":"markdown","source":"\n### Working on Test data processing\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv')\ntest","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_data():\n    global test\n    n = 0\n    for  i in test.segment_id :\n        file = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/test/{}.csv'.format(i))\n        for x in file.columns:\n            test.loc[n:n+1,x+'_mean'] = file[x].mean()\n            test.loc[n,x+'_std'] = file[x].std()\n            test.loc[n,x+'_min'] = file[x].min()\n            test.loc[n,x+'_20'] = file[x].quantile(0.20)\n            test.loc[n,x+'_40'] = file[x].quantile(0.40)\n            test.loc[n,x+'_50'] = file[x].quantile(0.50)\n            test.loc[n,x+'_60'] = file[x].quantile(0.60)\n            test.loc[n,x+'_80'] = file[x].quantile(0.80)\n            test.loc[n,x+'_max'] = file[x].max()\n        n +=1\n\n    test.fillna(0,inplace=True)\n    return test\nmake_data()    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"segment_id_test = test['segment_id']\ntest.drop(['time_to_eruption','segment_id'],axis = 1 , inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.drop(['sensor_1_50', 'sensor_2_50', 'sensor_3_50','sensor_4_50', 'sensor_5_50', 'sensor_6_50',\n                 'sensor_7_50', 'sensor_8_50', 'sensor_9_50', 'sensor_10_50'],axis = 1 , inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"std_test = std_scaler.transform(test)\ntest = pd.DataFrame(std_test, columns = test.columns)\ntest","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Final Model "},{"metadata":{},"cell_type":"markdown","source":"### We will try *lgb* alone and ensemble *[lgb+xgb]* finally we also will try ensemble *[lgb+xgb+RandomForestRegressor]*"},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_final_model    = lgb_grid_search.best_estimator_\nxgb_final_model    = xgb_grid_search.best_estimator_\nforest_final_model = forest_grid_search.best_estimator_\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_y_pred    = lgb_final_model.predict(test)\nxgb_y_pred    = xgb_final_model.predict(test)\nforest_y_pred = forest_final_model.predict(test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame()\nsub['segment_id'] = segment_id_test\nsub['time_to_eruption'] = lgb_y_pred\nsub.to_csv('submission_lgb.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_ensemble = pd.DataFrame()\nsub_ensemble['segment_id'] = segment_id_test\nsub_ensemble['time_to_eruption'] = lgb_y_pred*0.6 + xgb_y_pred*0.4\nsub_ensemble.to_csv('submission_ensemble_1.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_ensemble_ = pd.DataFrame()\nsub_ensemble_['segment_id'] = segment_id_test\nsub_ensemble_['time_to_eruption'] = lgb_y_pred*0.60 + xgb_y_pred*0.30 + forest_y_pred*0.10\nsub_ensemble_.to_csv('submission_ensemble_2.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}