{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-11-22T06:52:18.245134Z","iopub.status.busy":"2020-11-22T06:52:18.244029Z","iopub.status.idle":"2020-11-22T06:52:18.247439Z","shell.execute_reply":"2020-11-22T06:52:18.246678Z"},"papermill":{"duration":0.056,"end_time":"2020-11-22T06:52:18.247567","exception":false,"start_time":"2020-11-22T06:52:18.191567","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport os\nimport random","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:18.337602Z","iopub.status.busy":"2020-11-22T06:52:18.336560Z","iopub.status.idle":"2020-11-22T06:52:19.494580Z","shell.execute_reply":"2020-11-22T06:52:19.493806Z"},"papermill":{"duration":1.205362,"end_time":"2020-11-22T06:52:19.494716","exception":false,"start_time":"2020-11-22T06:52:18.289354","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error, r2_score\nfrom sklearn.model_selection import KFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from xgboost import XGBRegressor\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:25.339714Z","iopub.status.busy":"2020-11-22T06:52:25.338893Z","iopub.status.idle":"2020-11-22T06:52:25.342035Z","shell.execute_reply":"2020-11-22T06:52:25.341340Z"},"papermill":{"duration":0.052101,"end_time":"2020-11-22T06:52:25.342160","exception":false,"start_time":"2020-11-22T06:52:25.290059","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\nseed_everything(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For data I used a preprocessed data that I uploaded into kaggle. I will share the used notebook for producing the used features. I used classic statistical features (e.g. mean, rms, etc..) on each of the sensor signals, their first and second derivatives, the cummulative sum of the sensor values, and the wavelet transformation. "},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:25.437597Z","iopub.status.busy":"2020-11-22T06:52:25.436769Z","iopub.status.idle":"2020-11-22T06:52:34.937717Z","shell.execute_reply":"2020-11-22T06:52:34.936818Z"},"papermill":{"duration":9.553439,"end_time":"2020-11-22T06:52:34.937869","exception":false,"start_time":"2020-11-22T06:52:25.384430","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/ingv-data/Train.csv')\ntest = pd.read_csv('../input/ingv-data/Test.csv')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.042992,"end_time":"2020-11-22T06:52:35.024185","exception":false,"start_time":"2020-11-22T06:52:34.981193","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 1- Data Processing"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:35.117687Z","iopub.status.busy":"2020-11-22T06:52:35.116893Z","iopub.status.idle":"2020-11-22T06:52:35.129864Z","shell.execute_reply":"2020-11-22T06:52:35.129184Z"},"papermill":{"duration":0.063512,"end_time":"2020-11-22T06:52:35.130005","exception":false,"start_time":"2020-11-22T06:52:35.066493","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"targets_df = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:35.227222Z","iopub.status.busy":"2020-11-22T06:52:35.224818Z","iopub.status.idle":"2020-11-22T06:52:37.574142Z","shell.execute_reply":"2020-11-22T06:52:37.573314Z"},"papermill":{"duration":2.401469,"end_time":"2020-11-22T06:52:37.574295","exception":false,"start_time":"2020-11-22T06:52:35.172826","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = train.merge(targets_df, right_on='segment_id', left_on='id').drop(['segment_id'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:37.696880Z","iopub.status.busy":"2020-11-22T06:52:37.695796Z","iopub.status.idle":"2020-11-22T06:52:37.699341Z","shell.execute_reply":"2020-11-22T06:52:37.698569Z"},"papermill":{"duration":0.082437,"end_time":"2020-11-22T06:52:37.699479","exception":false,"start_time":"2020-11-22T06:52:37.617042","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test_idx = test['id']\ntest.drop(['id'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:37.818780Z","iopub.status.busy":"2020-11-22T06:52:37.817778Z","iopub.status.idle":"2020-11-22T06:52:37.822071Z","shell.execute_reply":"2020-11-22T06:52:37.821343Z"},"papermill":{"duration":0.080246,"end_time":"2020-11-22T06:52:37.822194","exception":false,"start_time":"2020-11-22T06:52:37.741948","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"targets = train['time_to_eruption']\ntrain_idx = train['id']\ntrain = train.drop(['time_to_eruption','id'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:37.914822Z","iopub.status.busy":"2020-11-22T06:52:37.914014Z","iopub.status.idle":"2020-11-22T06:52:40.268426Z","shell.execute_reply":"2020-11-22T06:52:40.267602Z"},"papermill":{"duration":2.403728,"end_time":"2020-11-22T06:52:40.268559","exception":false,"start_time":"2020-11-22T06:52:37.864831","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"c_columns = [c for c in train.columns.tolist() if len(train[c].value_counts())>=40]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:40.363136Z","iopub.status.busy":"2020-11-22T06:52:40.362118Z","iopub.status.idle":"2020-11-22T06:52:40.366513Z","shell.execute_reply":"2020-11-22T06:52:40.367081Z"},"papermill":{"duration":0.05461,"end_time":"2020-11-22T06:52:40.367233","exception":false,"start_time":"2020-11-22T06:52:40.312623","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"len(c_columns)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:40.464195Z","iopub.status.busy":"2020-11-22T06:52:40.463397Z","iopub.status.idle":"2020-11-22T06:52:42.696088Z","shell.execute_reply":"2020-11-22T06:52:42.696697Z"},"papermill":{"duration":2.285911,"end_time":"2020-11-22T06:52:42.696887","exception":false,"start_time":"2020-11-22T06:52:40.410976","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"uni_val_cols = [c for c in train.columns.tolist() if len(train[c].value_counts())==1]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:42.790885Z","iopub.status.busy":"2020-11-22T06:52:42.789687Z","iopub.status.idle":"2020-11-22T06:52:42.794558Z","shell.execute_reply":"2020-11-22T06:52:42.793944Z"},"papermill":{"duration":0.053917,"end_time":"2020-11-22T06:52:42.794700","exception":false,"start_time":"2020-11-22T06:52:42.740783","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"len(uni_val_cols)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:42.892006Z","iopub.status.busy":"2020-11-22T06:52:42.890622Z","iopub.status.idle":"2020-11-22T06:52:45.211843Z","shell.execute_reply":"2020-11-22T06:52:45.211035Z"},"papermill":{"duration":2.37294,"end_time":"2020-11-22T06:52:45.211979","exception":false,"start_time":"2020-11-22T06:52:42.839039","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"cat_cols = [c for c in train.columns.tolist() if (len(train[c].value_counts())<40 and c not in uni_val_cols)]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:45.309773Z","iopub.status.busy":"2020-11-22T06:52:45.308607Z","iopub.status.idle":"2020-11-22T06:52:45.314017Z","shell.execute_reply":"2020-11-22T06:52:45.313374Z"},"papermill":{"duration":0.056061,"end_time":"2020-11-22T06:52:45.314159","exception":false,"start_time":"2020-11-22T06:52:45.258098","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"len(cat_cols)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:45.438797Z","iopub.status.busy":"2020-11-22T06:52:45.411130Z","iopub.status.idle":"2020-11-22T06:52:45.472498Z","shell.execute_reply":"2020-11-22T06:52:45.471673Z"},"papermill":{"duration":0.112496,"end_time":"2020-11-22T06:52:45.472632","exception":false,"start_time":"2020-11-22T06:52:45.360136","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_features = train.drop(uni_val_cols, axis=1)\ntest_features  = test .drop(uni_val_cols, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:45.569663Z","iopub.status.busy":"2020-11-22T06:52:45.568845Z","iopub.status.idle":"2020-11-22T06:52:45.620133Z","shell.execute_reply":"2020-11-22T06:52:45.619412Z"},"papermill":{"duration":0.101991,"end_time":"2020-11-22T06:52:45.620290","exception":false,"start_time":"2020-11-22T06:52:45.518299","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train.isna().sum().sum()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:45.719693Z","iopub.status.busy":"2020-11-22T06:52:45.718877Z","iopub.status.idle":"2020-11-22T06:52:45.766977Z","shell.execute_reply":"2020-11-22T06:52:45.766159Z"},"papermill":{"duration":0.101023,"end_time":"2020-11-22T06:52:45.767117","exception":false,"start_time":"2020-11-22T06:52:45.666094","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_features.fillna(0, inplace=True)\ntest_features.fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:45.865971Z","iopub.status.busy":"2020-11-22T06:52:45.864916Z","iopub.status.idle":"2020-11-22T06:52:45.868462Z","shell.execute_reply":"2020-11-22T06:52:45.867691Z"},"papermill":{"duration":0.055181,"end_time":"2020-11-22T06:52:45.868591","exception":false,"start_time":"2020-11-22T06:52:45.813410","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_features.reset_index(drop=True, inplace=True)\ntest_features.reset_index(drop=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:45.968235Z","iopub.status.busy":"2020-11-22T06:52:45.966933Z","iopub.status.idle":"2020-11-22T06:52:46.010941Z","shell.execute_reply":"2020-11-22T06:52:46.011553Z"},"papermill":{"duration":0.096878,"end_time":"2020-11-22T06:52:46.011732","exception":false,"start_time":"2020-11-22T06:52:45.914854","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_features.isna().sum().sum()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:52:46.113537Z","iopub.status.busy":"2020-11-22T06:52:46.112695Z","iopub.status.idle":"2020-11-22T06:52:46.546470Z","shell.execute_reply":"2020-11-22T06:52:46.545794Z"},"papermill":{"duration":0.486964,"end_time":"2020-11-22T06:52:46.546598","exception":false,"start_time":"2020-11-22T06:52:46.059634","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_features[c_columns[10]].hist(bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:53:10.880179Z","iopub.status.busy":"2020-11-22T06:53:10.879388Z","iopub.status.idle":"2020-11-22T06:53:10.887649Z","shell.execute_reply":"2020-11-22T06:53:10.886872Z"},"papermill":{"duration":0.072362,"end_time":"2020-11-22T06:53:10.887784","exception":false,"start_time":"2020-11-22T06:53:10.815422","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"targets = pd.DataFrame(targets)\ntargets.reset_index(inplace=True)\ntargets.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:53:11.006087Z","iopub.status.busy":"2020-11-22T06:53:11.005148Z","iopub.status.idle":"2020-11-22T06:53:11.010628Z","shell.execute_reply":"2020-11-22T06:53:11.009854Z"},"papermill":{"duration":0.067919,"end_time":"2020-11-22T06:53:11.010754","exception":false,"start_time":"2020-11-22T06:53:10.942835","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"y_train = targets['time_to_eruption']\ny_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2- Modeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_xgb_model(seed_):\n    xgb_meta = XGBRegressor(tree_method='gpu_hist',\n                            colsample_bytree=0.4,\n                             gamma=0,\n                            learning_rate=0.07,\n                            max_depth=3,\n                            min_child_weight=1.5,\n                            n_estimators=1000,\n                            reg_alpha=0.75,\n                            reg_lambda=0.45,\n                            subsample=0.6,\n                            seed=seed_)\n    return xgb_meta","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_xgb(X, y, X_test, fold, seed):\n    \n    seed_everything(seed)\n    \n    \n    train_mask = X['kfold'] != fold\n    valid_idc = X.loc[~train_mask].index\n    \n    X_train = X.loc[train_mask].reset_index(drop=True)\n    y_train = y.loc[train_mask].reset_index(drop=True)\n\n    \n    X_val = X.loc[~train_mask].reset_index(drop=True)\n    y_val = y.loc[~train_mask].reset_index(drop=True)\n    \n    X_train.drop(columns=['kfold'], inplace=True)\n    X_val.drop(columns=['kfold'], inplace=True)\n    \n    oof = np.zeros((X.shape[0], 1))\n    \n    model = build_xgb_model(seed)\n    \n    print(f'============={seed}========={fold}==================')\n    \n    model.fit(X_train, y_train)\n    train_loss = mean_absolute_error(y_train, model.predict(X_train))\n    print(f\"Seed: {seed}, FOLD: {fold}, train_loss: {train_loss}\")\n    valid_preds = model.predict(X_val)\n    oof[valid_idc] = valid_preds.reshape((len(valid_preds),1))\n    valid_loss = mean_absolute_error(y_val, valid_preds)\n    print(f\"Seed: {seed}, FOLD: {fold}, val_loss: {valid_loss}\")\n    #------------------ Predictions -------------------\n\n    predictions = np.zeros((X_test.shape[0], 1))\n    predictions = model.predict(X_test[X_train.columns]).reshape((len(X_test),1))\n    \n    return oof, predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_k_fold_xgb(X, y, X_test, seed):\n    oof = np.zeros((train_features.shape[0], 1))\n    predictions = np.zeros((test_features.shape[0], 1))\n    \n    for fold in range(N_FOLDS):\n        oof_, pred_ = run_xgb(X, y, X_test, fold, seed)\n        \n        predictions += pred_ / N_FOLDS\n        oof += oof_\n        \n    return oof, predictions","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:53:11.127839Z","iopub.status.busy":"2020-11-22T06:53:11.126843Z","iopub.status.idle":"2020-11-22T06:53:11.129389Z","shell.execute_reply":"2020-11-22T06:53:11.130030Z"},"papermill":{"duration":0.064149,"end_time":"2020-11-22T06:53:11.130184","exception":false,"start_time":"2020-11-22T06:53:11.066035","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"N_FOLDS = 10","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T06:53:11.755310Z","iopub.status.busy":"2020-11-22T06:53:11.754019Z","iopub.status.idle":"2020-11-22T07:12:00.357268Z","shell.execute_reply":"2020-11-22T07:12:00.356588Z"},"papermill":{"duration":1128.667847,"end_time":"2020-11-22T07:12:00.357416","exception":false,"start_time":"2020-11-22T06:53:11.689569","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Averaging on multiple SEEDS\n\nseeds = [0, 1, 2, 3, 4, 5, 6]\noof = np.zeros((train_features.shape[0], 1))\npredictions = np.zeros((test_features.shape[0], 1))\n\nfor seed in seeds:\n    folds = train_features.copy()\n    folds['idx'] = train_idx\n    folds['kfold'] = np.zeros(len(folds))\n    kf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\n    for f, (t_idx, v_idx) in enumerate(kf.split(train_features)) :\n        folds.loc[v_idx, 'kfold'] = int(f)\n    folds['kfold'] = folds['kfold'].astype(int)\n    oof_, predictions_ = run_k_fold_xgb(folds.drop(['idx'], axis=1), y_train, test_features, seed)\n    oof += oof_ / len(seeds)\n    predictions += predictions_ / len(seeds)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T07:12:00.845586Z","iopub.status.busy":"2020-11-22T07:12:00.844639Z","iopub.status.idle":"2020-11-22T07:12:00.849966Z","shell.execute_reply":"2020-11-22T07:12:00.849220Z"},"papermill":{"duration":0.252662,"end_time":"2020-11-22T07:12:00.850092","exception":false,"start_time":"2020-11-22T07:12:00.597430","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(\"CV MAE = {}\".format(mean_absolute_error(y_train, oof)))\nprint(\"CV R2 = {}\".format(r2_score(y_train, oof)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3- Predictions"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T07:12:01.341487Z","iopub.status.busy":"2020-11-22T07:12:01.340689Z","iopub.status.idle":"2020-11-22T07:12:01.363452Z","shell.execute_reply":"2020-11-22T07:12:01.362599Z"},"papermill":{"duration":0.271076,"end_time":"2020-11-22T07:12:01.363597","exception":false,"start_time":"2020-11-22T07:12:01.092521","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T07:12:01.858192Z","iopub.status.busy":"2020-11-22T07:12:01.857061Z","iopub.status.idle":"2020-11-22T07:12:01.862242Z","shell.execute_reply":"2020-11-22T07:12:01.861438Z"},"papermill":{"duration":0.258597,"end_time":"2020-11-22T07:12:01.862392","exception":false,"start_time":"2020-11-22T07:12:01.603795","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"preds = pd.DataFrame()\npreds['segment_id'] = test_idx\npreds['time_to_eruption'] = predictions\npreds.head(2)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T07:12:02.358036Z","iopub.status.busy":"2020-11-22T07:12:02.357241Z","iopub.status.idle":"2020-11-22T07:12:02.363944Z","shell.execute_reply":"2020-11-22T07:12:02.363155Z"},"papermill":{"duration":0.258195,"end_time":"2020-11-22T07:12:02.364070","exception":false,"start_time":"2020-11-22T07:12:02.105875","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission = submission.drop(['time_to_eruption'], axis=1).merge(preds, on='segment_id')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-22T07:12:02.855310Z","iopub.status.busy":"2020-11-22T07:12:02.854448Z","iopub.status.idle":"2020-11-22T07:12:03.116545Z","shell.execute_reply":"2020-11-22T07:12:03.115851Z"},"papermill":{"duration":0.510574,"end_time":"2020-11-22T07:12:03.116711","exception":false,"start_time":"2020-11-22T07:12:02.606137","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"submission.to_csv('submission.csv', header=True, index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}