{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# imports\nimport re\nimport pandas as pd\nimport numpy as np\nfrom math import sqrt, ceil\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nfrom lightgbm import LGBMRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error as mse\nfrom sklearn.metrics import mean_absolute_error as mae\nfrom sklearn.feature_selection import VarianceThreshold\nfrom sklearn.preprocessing import MinMaxScaler\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:02.394011Z","iopub.execute_input":"2021-07-04T16:41:02.394424Z","iopub.status.idle":"2021-07-04T16:41:02.402073Z","shell.execute_reply.started":"2021-07-04T16:41:02.394390Z","shell.execute_reply":"2021-07-04T16:41:02.400787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_dataset(train_features_df, labels_df):\n    # this is a function to combine train data and label data and return X, Y\n    _df = train_features_df.merge(labels_df, on='segment_id', how='left')\n    _y = _df['time_to_eruption']\n    _x = _df.drop(['segment_id', 'time_to_eruption'], axis=1)\n    return (_x, _y)","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:02.407712Z","iopub.execute_input":"2021-07-04T16:41:02.408243Z","iopub.status.idle":"2021-07-04T16:41:02.423784Z","shell.execute_reply.started":"2021-07-04T16:41:02.408051Z","shell.execute_reply":"2021-07-04T16:41:02.422327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset_by_range(features_df, labels_df, lower_limit, upper_limit):\n    # this function returns a training dataset of X and Y for a given lower limit and uper limit (on the label)\n    _df = features_df.merge(labels_df, on='segment_id', how='left')\n    _df = _df[_df['time_to_eruption'].notna()]\n    _df = _df[(_df['time_to_eruption'] > lower_limit) & (_df['time_to_eruption'] < upper_limit)]\n    _y = _df['time_to_eruption']\n    _x = _df.drop(['segment_id', 'time_to_eruption'], axis=1)\n    _x = _df.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '_', x))\n    return (_x, _y)","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:02.425857Z","iopub.execute_input":"2021-07-04T16:41:02.426555Z","iopub.status.idle":"2021-07-04T16:41:02.443420Z","shell.execute_reply.started":"2021-07-04T16:41:02.426503Z","shell.execute_reply":"2021-07-04T16:41:02.442471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# read all our datasets\ntrain_df = pd.read_parquet('/kaggle/input/ingv-parquet/train_features.parquet')\ntest_df = pd.read_parquet('/kaggle/input/ingv-parquet/test_features.parquet')\nlabels_df = pd.read_parquet('/kaggle/input/ingv-parquet/labels.parquet')\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:02.445215Z","iopub.execute_input":"2021-07-04T16:41:02.445747Z","iopub.status.idle":"2021-07-04T16:41:07.838828Z","shell.execute_reply.started":"2021-07-04T16:41:02.445687Z","shell.execute_reply":"2021-07-04T16:41:07.837767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# lightGBM doesn't like column names with special characters like -, so we convert them to _\ntrain_df = train_df.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '_', x))\ntest_df = test_df.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '_', x))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:07.841563Z","iopub.execute_input":"2021-07-04T16:41:07.842024Z","iopub.status.idle":"2021-07-04T16:41:08.692313Z","shell.execute_reply.started":"2021-07-04T16:41:07.841975Z","shell.execute_reply":"2021-07-04T16:41:08.691194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# top 501 features\nfeatures = [\n\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_21',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_22',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_23',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_24',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_25',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_26',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_27',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_28',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_29',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_30',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_31',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_32',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_33',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_34',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_35',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_36',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_37',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_38',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_39',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_40',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_41',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_42',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_43',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_44',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_45',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_47',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_50',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_52',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_54',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_55',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_56',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_58',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_59',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_60',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_61',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_62',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_63',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_64',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_69',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_71',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_79',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_80',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_81',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_82',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_83',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_87',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_90',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_92',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_93',\n 'sensor_9__fft_coefficient__attr_\"abs\"__coeff_94',\n 'sensor_9__fft_coefficient__attr_\"angle\"__coeff_23',\n 'sensor_9__fft_coefficient__attr_\"angle\"__coeff_32',\n 'sensor_9__fft_coefficient__attr_\"angle\"__coeff_44',\n 'sensor_9__fft_coefficient__attr_\"angle\"__coeff_75',\n 'sensor_9__fft_coefficient__attr_\"angle\"__coeff_77',\n 'sensor_9__approximate_entropy__m_2__r_0.1',\n 'sensor_9__approximate_entropy__m_2__r_0.3',\n 'sensor_9__agg_linear_trend__attr_\"stderr\"__chunk_len_10__f_agg_\"var\"',\n 'sensor_9__agg_linear_trend__attr_\"stderr\"__chunk_len_50__f_agg_\"max\"',\n 'sensor_9__agg_linear_trend__attr_\"stderr\"__chunk_len_50__f_agg_\"mean\"',\n 'sensor_9__agg_linear_trend__attr_\"stderr\"__chunk_len_50__f_agg_\"var\"',\n 'sensor_9__number_crossing_m__m_0',\n 'sensor_9__ratio_beyond_r_sigma__r_0.5',\n 'sensor_9__permutation_entropy__dimension_3__tau_1',\n 'sensor_9__permutation_entropy__dimension_4__tau_1',\n 'sensor_8__abs_energy',\n 'sensor_8__mean_abs_change',\n 'sensor_8__standard_deviation',\n 'sensor_8__kurtosis',\n 'sensor_8__longest_strike_below_mean',\n 'sensor_8__longest_strike_above_mean',\n 'sensor_8__maximum',\n 'sensor_8__minimum',\n 'sensor_8__cid_ce__normalize_True',\n 'sensor_8__quantile__q_0.2',\n 'sensor_8__quantile__q_0.3',\n 'sensor_8__quantile__q_0.4',\n 'sensor_8__quantile__q_0.6',\n 'sensor_8__quantile__q_0.7',\n 'sensor_8__quantile__q_0.8',\n 'sensor_8__agg_autocorrelation__f_agg_\"mean\"__maxlag_40',\n 'sensor_8__number_cwt_peaks__n_1',\n 'sensor_8__number_cwt_peaks__n_5',\n 'sensor_8__number_peaks__n_1',\n 'sensor_8__number_peaks__n_3',\n 'sensor_8__number_peaks__n_5',\n 'sensor_8__number_peaks__n_10',\n 'sensor_8__number_peaks__n_50',\n 'sensor_8__binned_entropy__max_bins_10',\n 'sensor_8__spkt_welch_density__coeff_2',\n 'sensor_8__spkt_welch_density__coeff_5',\n 'sensor_8__spkt_welch_density__coeff_8',\n 'sensor_8__ar_coefficient__coeff_1__k_10',\n 'sensor_8__ar_coefficient__coeff_2__k_10',\n 'sensor_8__ar_coefficient__coeff_3__k_10',\n 'sensor_8__ar_coefficient__coeff_4__k_10',\n 'sensor_8__ar_coefficient__coeff_5__k_10',\n 'sensor_8__ar_coefficient__coeff_6__k_10',\n 'sensor_8__ar_coefficient__coeff_7__k_10',\n 'sensor_8__ar_coefficient__coeff_8__k_10',\n 'sensor_8__ar_coefficient__coeff_9__k_10',\n 'sensor_8__ar_coefficient__coeff_10__k_10',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_15',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_16',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_17',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_18',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_19',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_20',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_21',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_22',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_23',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_24',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_25',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_26',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_27',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_28',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_29',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_30',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_31',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_32',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_33',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_34',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_35',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_36',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_37',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_38',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_39',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_40',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_41',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_42',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_43',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_44',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_45',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_46',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_47',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_48',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_49',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_50',\n 'sensor_8__fft_coefficient__attr_\"abs\"__coeff_83',\n 'sensor_8__value_count__value_1',\n 'sensor_8__value_count__value_-1',\n 'sensor_8__approximate_entropy__m_2__r_0.1',\n 'sensor_8__approximate_entropy__m_2__r_0.3',\n 'sensor_8__approximate_entropy__m_2__r_0.5',\n 'sensor_8__agg_linear_trend__attr_\"intercept\"__chunk_len_50__f_agg_\"max\"',\n 'sensor_8__agg_linear_trend__attr_\"intercept\"__chunk_len_50__f_agg_\"min\"',\n 'sensor_8__agg_linear_trend__attr_\"intercept\"__chunk_len_50__f_agg_\"var\"',\n 'sensor_8__agg_linear_trend__attr_\"stderr\"__chunk_len_5__f_agg_\"var\"',\n 'sensor_8__agg_linear_trend__attr_\"stderr\"__chunk_len_10__f_agg_\"var\"',\n 'sensor_8__agg_linear_trend__attr_\"stderr\"__chunk_len_50__f_agg_\"max\"',\n 'sensor_8__agg_linear_trend__attr_\"stderr\"__chunk_len_50__f_agg_\"mean\"',\n 'sensor_8__agg_linear_trend__attr_\"stderr\"__chunk_len_50__f_agg_\"var\"',\n 'sensor_8__number_crossing_m__m_0',\n 'sensor_8__ratio_beyond_r_sigma__r_0.5',\n 'sensor_8__permutation_entropy__dimension_3__tau_1',\n 'sensor_3__abs_energy',\n 'sensor_3__standard_deviation',\n 'sensor_3__kurtosis',\n 'sensor_3__longest_strike_below_mean',\n 'sensor_3__longest_strike_above_mean',\n 'sensor_3__maximum',\n 'sensor_3__cid_ce__normalize_True',\n 'sensor_3__quantile__q_0.2',\n 'sensor_3__quantile__q_0.4',\n 'sensor_3__quantile__q_0.6',\n 'sensor_3__quantile__q_0.7',\n 'sensor_3__agg_autocorrelation__f_agg_\"mean\"__maxlag_40',\n 'sensor_3__agg_autocorrelation__f_agg_\"median\"__maxlag_40',\n 'sensor_3__number_cwt_peaks__n_1',\n 'sensor_3__number_cwt_peaks__n_5',\n 'sensor_3__number_peaks__n_1',\n 'sensor_3__number_peaks__n_3',\n 'sensor_3__number_peaks__n_5',\n 'sensor_3__number_peaks__n_10',\n 'sensor_3__number_peaks__n_50',\n 'sensor_3__binned_entropy__max_bins_10',\n 'sensor_3__spkt_welch_density__coeff_2',\n 'sensor_3__spkt_welch_density__coeff_5',\n 'sensor_3__spkt_welch_density__coeff_8',\n 'sensor_3__ar_coefficient__coeff_1__k_10',\n 'sensor_3__ar_coefficient__coeff_2__k_10',\n 'sensor_3__ar_coefficient__coeff_3__k_10',\n 'sensor_3__ar_coefficient__coeff_4__k_10',\n 'sensor_3__ar_coefficient__coeff_5__k_10',\n 'sensor_3__ar_coefficient__coeff_6__k_10',\n 'sensor_3__ar_coefficient__coeff_7__k_10',\n 'sensor_3__ar_coefficient__coeff_8__k_10',\n 'sensor_3__ar_coefficient__coeff_9__k_10',\n 'sensor_3__change_quantiles__f_agg_\"var\"__isabs_False__qh_0.6__ql_0.4',\n 'sensor_3__fft_coefficient__attr_\"real\"__coeff_11',\n 'sensor_3__fft_coefficient__attr_\"real\"__coeff_12',\n 'sensor_3__fft_coefficient__attr_\"real\"__coeff_13',\n 'sensor_3__fft_coefficient__attr_\"real\"__coeff_14',\n 'sensor_3__fft_coefficient__attr_\"imag\"__coeff_1',\n 'sensor_3__fft_coefficient__attr_\"imag\"__coeff_11',\n 'sensor_3__fft_coefficient__attr_\"imag\"__coeff_12',\n 'sensor_3__fft_coefficient__attr_\"imag\"__coeff_13',\n 'sensor_3__fft_coefficient__attr_\"imag\"__coeff_14',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_0',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_1',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_2',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_3',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_4',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_5',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_6',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_7',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_8',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_9',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_10',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_11',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_12',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_13',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_14',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_15',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_16',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_17',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_18',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_19',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_20',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_21',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_22',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_23',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_24',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_25',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_26',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_27',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_28',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_29',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_30',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_31',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_32',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_33',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_34',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_35',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_36',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_37',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_38',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_39',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_40',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_41',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_42',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_43',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_44',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_45',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_46',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_47',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_48',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_49',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_53',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_72',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_77',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_84',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_86',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_88',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_89',\n 'sensor_3__fft_coefficient__attr_\"abs\"__coeff_94',\n 'sensor_3__value_count__value_1',\n 'sensor_3__value_count__value_-1',\n 'sensor_3__approximate_entropy__m_2__r_0.1',\n 'sensor_3__approximate_entropy__m_2__r_0.3',\n 'sensor_3__approximate_entropy__m_2__r_0.5',\n 'sensor_3__agg_linear_trend__attr_\"slope\"__chunk_len_5__f_agg_\"mean\"',\n 'sensor_3__agg_linear_trend__attr_\"stderr\"__chunk_len_5__f_agg_\"var\"',\n 'sensor_3__agg_linear_trend__attr_\"stderr\"__chunk_len_10__f_agg_\"var\"',\n 'sensor_3__agg_linear_trend__attr_\"stderr\"__chunk_len_50__f_agg_\"max\"',\n 'sensor_3__agg_linear_trend__attr_\"stderr\"__chunk_len_50__f_agg_\"mean\"',\n 'sensor_3__agg_linear_trend__attr_\"stderr\"__chunk_len_50__f_agg_\"var\"',\n 'sensor_3__number_crossing_m__m_0',\n 'sensor_3__ratio_beyond_r_sigma__r_0.5',\n 'sensor_3__permutation_entropy__dimension_3__tau_1',\n 'sensor_3__permutation_entropy__dimension_4__tau_1',\n 'sensor_5__mean_abs_change',\n 'sensor_5__longest_strike_below_mean',\n 'sensor_5__longest_strike_above_mean',\n 'sensor_5__cid_ce__normalize_True',\n 'sensor_5__cid_ce__normalize_False',\n 'sensor_5__number_cwt_peaks__n_1',\n 'sensor_5__number_cwt_peaks__n_5',\n 'sensor_5__number_peaks__n_1',\n 'sensor_5__number_peaks__n_3',\n 'sensor_5__number_peaks__n_5',\n 'sensor_5__number_peaks__n_10',\n 'sensor_5__spkt_welch_density__coeff_2',\n 'sensor_5__spkt_welch_density__coeff_5',\n 'sensor_5__spkt_welch_density__coeff_8',\n 'sensor_5__ar_coefficient__coeff_1__k_10',\n 'sensor_5__ar_coefficient__coeff_2__k_10',\n 'sensor_5__ar_coefficient__coeff_3__k_10',\n 'sensor_5__ar_coefficient__coeff_4__k_10',\n 'sensor_5__ar_coefficient__coeff_5__k_10',\n 'sensor_5__ar_coefficient__coeff_6__k_10',\n 'sensor_5__ar_coefficient__coeff_7__k_10',\n 'sensor_5__ar_coefficient__coeff_8__k_10',\n 'sensor_5__ar_coefficient__coeff_9__k_10',\n 'sensor_5__ar_coefficient__coeff_10__k_10',\n 'sensor_5__change_quantiles__f_agg_\"mean\"__isabs_True__qh_0.6__ql_0.0',\n 'sensor_5__change_quantiles__f_agg_\"var\"__isabs_False__qh_1.0__ql_0.0',\n 'sensor_5__change_quantiles__f_agg_\"var\"__isabs_False__qh_0.8__ql_0.2',\n 'sensor_5__change_quantiles__f_agg_\"mean\"__isabs_True__qh_0.8__ql_0.2',\n 'sensor_5__fft_coefficient__attr_\"real\"__coeff_11',\n 'sensor_5__fft_coefficient__attr_\"real\"__coeff_12',\n 'sensor_5__fft_coefficient__attr_\"real\"__coeff_13',\n 'sensor_5__fft_coefficient__attr_\"imag\"__coeff_11',\n 'sensor_5__fft_coefficient__attr_\"imag\"__coeff_12',\n 'sensor_5__fft_coefficient__attr_\"imag\"__coeff_13',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_4',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_5',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_6',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_7',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_8',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_9',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_10',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_11',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_12',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_13',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_14',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_15',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_16',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_17',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_18',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_19',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_20',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_21',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_22',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_23',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_24',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_25',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_26',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_27',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_28',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_29',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_30',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_31',\n 'sensor_5__fft_coefficient__attr_\"abs\"__coeff_32',\n 'sensor_5__value_count__value_1',\n 'sensor_5__value_count__value_-1',\n 'sensor_5__approximate_entropy__m_2__r_0.1',\n 'sensor_5__approximate_entropy__m_2__r_0.3',\n 'sensor_5__approximate_entropy__m_2__r_0.5',\n 'sensor_5__agg_linear_trend__attr_\"stderr\"__chunk_len_50__f_agg_\"mean\"',\n 'sensor_5__number_crossing_m__m_0',\n 'sensor_5__permutation_entropy__dimension_3__tau_1'\n]","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:08.695036Z","iopub.execute_input":"2021-07-04T16:41:08.695484Z","iopub.status.idle":"2021-07-04T16:41:08.729341Z","shell.execute_reply.started":"2021-07-04T16:41:08.695438Z","shell.execute_reply":"2021-07-04T16:41:08.728045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# also renaming features so they match to the current column names in our dataset\nfeatures = [re.sub('[^A-Za-z0-9_]+', '_', feature) for feature in features]\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:08.731133Z","iopub.execute_input":"2021-07-04T16:41:08.731534Z","iopub.status.idle":"2021-07-04T16:41:08.779332Z","shell.execute_reply.started":"2021-07-04T16:41:08.731494Z","shell.execute_reply":"2021-07-04T16:41:08.778098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# get columns only available in test set\nseg_id_test = test_df['segment_id']\nx_test = test_df.drop(['segment_id'], axis=1)[features]\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:08.783684Z","iopub.execute_input":"2021-07-04T16:41:08.784315Z","iopub.status.idle":"2021-07-04T16:41:09.139190Z","shell.execute_reply.started":"2021-07-04T16:41:08.784279Z","shell.execute_reply":"2021-07-04T16:41:09.138186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# get train data\ntrain, label = get_train_dataset(train_df, labels_df)\n# keep only the top features \ntrain = train[features]\ntest_df = test_df[features + ['segment_id']]\n# scaling is a must\nscaler = MinMaxScaler().fit(pd.concat([train, x_test]))\nx_test = pd.DataFrame(scaler.transform(x_test))\ntrain = pd.DataFrame(scaler.transform(train))\n# renaming the columns back after scaling has been done\nx_test.columns = features\ntrain.columns = features\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:09.142121Z","iopub.execute_input":"2021-07-04T16:41:09.142776Z","iopub.status.idle":"2021-07-04T16:41:10.392955Z","shell.execute_reply.started":"2021-07-04T16:41:09.142725Z","shell.execute_reply":"2021-07-04T16:41:10.391804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split the train data into train and val. we will use val for early stopping\nx_train, x_val, y_train, y_val = train_test_split(train, label, random_state=786, test_size=0.2, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:51:23.100551Z","iopub.execute_input":"2021-07-04T16:51:23.101011Z","iopub.status.idle":"2021-07-04T16:51:23.137476Z","shell.execute_reply.started":"2021-07-04T16:51:23.100953Z","shell.execute_reply":"2021-07-04T16:51:23.135973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# hyper params for our LightGBM\nparams = {'application':'regression',\n         'boosting ': 'dart',\n         'num_iterations':10000, \n         'learning_rate':0.03, \n         'num_leaves': 45,\n         'extra_trees': True,\n         'feature_fraction':0.8, \n         'bagging_fraction':0.9,\n         'lambda_l1':0.1, \n         'lambda_l2':0.1, \n         'min_split_gain':0.01, \n         'early_stopping_round':100, \n         'max_depth':6,\n         'min_child_weight':40, \n         'n_estimators': 400,\n         'metric':'mse',\n         'verbosity': -1}\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:51:24.543946Z","iopub.execute_input":"2021-07-04T16:51:24.544365Z","iopub.status.idle":"2021-07-04T16:51:24.552284Z","shell.execute_reply.started":"2021-07-04T16:51:24.544330Z","shell.execute_reply":"2021-07-04T16:51:24.550632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# train, fit, and get prediction for validation dataset\nlgb_first = LGBMRegressor(**params)\nlgb_first.fit(x_train, y_train, eval_set=(x_val, y_val))\nval_preds = lgb_first.predict(x_val)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:51:26.290593Z","iopub.execute_input":"2021-07-04T16:51:26.291263Z","iopub.status.idle":"2021-07-04T16:54:40.177733Z","shell.execute_reply.started":"2021-07-04T16:51:26.291221Z","shell.execute_reply":"2021-07-04T16:54:40.176637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create an output dataframe of values and predictions\noutput = pd.DataFrame(list(zip(y_val, val_preds)))\noutput.columns = ['val', 'pred']\n# we dont have segment id here but since the label (time to eruption) for six rows will be the same, we groupby val\noutput = output.groupby('val').mean().reset_index()\nprint('Simple LGB model rmse: ', sqrt(mse(output['val'].to_numpy(), output['pred'].to_numpy())))\nprint('Simple LGB model mae: ', mae(output['val'].to_numpy(), output['pred'].to_numpy()))","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:49:28.084110Z","iopub.execute_input":"2021-07-04T16:49:28.084842Z","iopub.status.idle":"2021-07-04T16:49:28.106556Z","shell.execute_reply.started":"2021-07-04T16:49:28.084803Z","shell.execute_reply":"2021-07-04T16:49:28.105490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# we will zoom into the performance of the model for these different segments\nsteps = [(0, 2500000), (2500000, 15000000), (10000000, 25000000), \n         (20000000, 35000000), (30000000, 44000000), (44000000, 50000000)]\n\n_mae_list = []\n\n# loop over the range defined in steps and get performance\nfor l, u in steps:\n    _output = output[(output['val'] > l) & (output['val'] < u)]\n    _range = str(l)+'-'+str(u)\n    _mae_list.append((_range, mae(_output['val'].to_numpy(), _output['pred'].to_numpy())))\n    \nmae_df = pd.DataFrame(_mae_list, columns=['range', 'mae'])\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:50:15.262785Z","iopub.execute_input":"2021-07-04T16:50:15.263166Z","iopub.status.idle":"2021-07-04T16:50:15.284123Z","shell.execute_reply.started":"2021-07-04T16:50:15.263134Z","shell.execute_reply":"2021-07-04T16:50:15.282732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\noutput['diff'] = output['pred'] - output['val']\nfig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(20,5))\nfig.suptitle('Figure 1 - Zooming Into 1st Round Model Performance')\nax1.hist([output['val'], output['pred']])\nax2.scatter(output['val'], output['diff'], marker='x')\nax3.tick_params(labelrotation=45)\nax3.bar(mae_df['range'], mae_df['mae'])\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:50:18.875601Z","iopub.execute_input":"2021-07-04T16:50:18.876044Z","iopub.status.idle":"2021-07-04T16:50:19.367247Z","shell.execute_reply.started":"2021-07-04T16:50:18.876009Z","shell.execute_reply":"2021-07-04T16:50:19.366012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# first test prediction set\ntest_preds = lgb_first.predict(x_test)\ntest_first_preds_df = pd.DataFrame(list(zip(seg_id_test, test_preds)))\ntest_first_preds_df.columns = ['segment_id', 'time_to_eruption']\ntest_first_preds_df = test_first_preds_df.groupby('segment_id').mean().reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:12.288999Z","iopub.execute_input":"2021-07-04T16:41:12.289300Z","iopub.status.idle":"2021-07-04T16:41:12.418289Z","shell.execute_reply.started":"2021-07-04T16:41:12.289270Z","shell.execute_reply":"2021-07-04T16:41:12.417141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define steps of ranges for which we want to create smaller models\nsteps = [(0, 2500000), (0, 15000000), (10000000, 25000000), \n         (20000000, 35000000), (30000000, 50000000), (44000000, 50000000)]\n# empty lists to store results\noutput_list, model_list = [], []\ny_val_list, val_preds_list = [], []","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:12.419708Z","iopub.execute_input":"2021-07-04T16:41:12.420099Z","iopub.status.idle":"2021-07-04T16:41:12.473747Z","shell.execute_reply.started":"2021-07-04T16:41:12.420068Z","shell.execute_reply":"2021-07-04T16:41:12.472693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hyper-params for our specialized model. notice how we make these model a bit more complex by increasing number of leaves and max depth\nparams = {'application':'regression',\n         'boosting ': 'dart',\n         'num_iterations':8000, \n         'learning_rate':0.05, \n         'num_leaves': 95,\n         'extra_trees': True,\n         'feature_fraction':0.8, \n         'bagging_fraction':0.9,\n         'lambda_l1':0.1, \n         'lambda_l2':0.1, \n         'min_split_gain':0.01, \n         'early_stopping_round':100, \n         'max_depth': 7,\n         'min_child_weight':40, \n         'n_estimators': 400,\n         'metric':'mae',\n         'verbosity': -1}","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:12.476030Z","iopub.execute_input":"2021-07-04T16:41:12.476373Z","iopub.status.idle":"2021-07-04T16:41:12.483157Z","shell.execute_reply.started":"2021-07-04T16:41:12.476340Z","shell.execute_reply":"2021-07-04T16:41:12.481962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# iterate over the steps and train models\nfor l, u in steps:\n    # get data for range and scale\n    _x, _y = get_dataset_by_range(train_df, labels_df, l, u)\n    _x = _x[features]\n    _x = pd.DataFrame(scaler.transform(_x))\n    _x.columns = features\n    # split the data for this range\n    _x_train, _x_val, _y_train, _y_val = train_test_split(_x, _y, random_state=786, test_size=0.2, shuffle=False)\n    # train model for this range\n    _lgb = LGBMRegressor(**params)\n    _lgb.fit(_x_train, _y_train, eval_set=(_x_val, _y_val))\n    _val_preds = _lgb.predict(_x_val)\n    # create a dataframe to compare actual values vs predictions\n    _output = pd.DataFrame(list(zip(_y_val, _val_preds)))\n    _output.columns = ['val', 'pred']\n    _output = _output.groupby('val').median().reset_index()\n    _output['diff'] = _output['val'] - _output['pred']\n    # save model in the model list\n    model_list.append(_lgb)\n    # save output dataframe containing actual value vs predicted value in a list\n    output_list.append(_output)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:41:12.484711Z","iopub.execute_input":"2021-07-04T16:41:12.485032Z","iopub.status.idle":"2021-07-04T16:43:49.779576Z","shell.execute_reply.started":"2021-07-04T16:41:12.485000Z","shell.execute_reply":"2021-07-04T16:43:49.778577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n_mae_list = []\n\nfor idx, _output in enumerate(output_list):\n    l, h = steps[idx]\n    _mae = mae(_output['val'].to_numpy(), _output['pred'].to_numpy())\n    _range = '%s-%s' % (l, h)\n    _mae_list.append((_range, _mae))\n\nnew_mae_df = pd.DataFrame(_mae_list, columns=['range', 'mae'])\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:43:49.781161Z","iopub.execute_input":"2021-07-04T16:43:49.781946Z","iopub.status.idle":"2021-07-04T16:43:49.793924Z","shell.execute_reply.started":"2021-07-04T16:43:49.781883Z","shell.execute_reply":"2021-07-04T16:43:49.792759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,5))\nfig.suptitle('Figure 2 - Model Performance Comparision For Round One and Two')\nax1.tick_params(labelrotation=45)\nax2.tick_params(labelrotation=45)\nax1.bar(mae_df['range'], mae_df['mae'])\nax2.bar(new_mae_df['range'], new_mae_df['mae'])","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:43:49.795195Z","iopub.execute_input":"2021-07-04T16:43:49.795510Z","iopub.status.idle":"2021-07-04T16:43:50.108727Z","shell.execute_reply.started":"2021-07-04T16:43:49.795478Z","shell.execute_reply":"2021-07-04T16:43:50.107509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nfig, ax = plt.subplots(len(steps), 2, figsize=(15,15))\nfig.suptitle('Figure 3 - Individual Model Performance Round Two')\nfor idx, _output in enumerate(output_list):\n    ax[idx][0].hist([_output['val'], _output['pred']])\n    ax[idx][1].scatter(_output['val'], _output['diff'], marker='x')\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:43:50.110441Z","iopub.execute_input":"2021-07-04T16:43:50.110909Z","iopub.status.idle":"2021-07-04T16:43:52.345197Z","shell.execute_reply.started":"2021-07-04T16:43:50.110860Z","shell.execute_reply":"2021-07-04T16:43:52.343902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a dataframe with test features and their predictions\ntest_pred_df = pd.merge(test_df, test_first_preds_df, on='segment_id')","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:43:52.347173Z","iopub.execute_input":"2021-07-04T16:43:52.347593Z","iopub.status.idle":"2021-07-04T16:43:52.389821Z","shell.execute_reply.started":"2021-07-04T16:43:52.347551Z","shell.execute_reply":"2021-07-04T16:43:52.388648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# empty list for storing results\nsegment_id, time_to_eruption = [], []\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:43:52.391450Z","iopub.execute_input":"2021-07-04T16:43:52.391936Z","iopub.status.idle":"2021-07-04T16:43:52.397764Z","shell.execute_reply.started":"2021-07-04T16:43:52.391890Z","shell.execute_reply":"2021-07-04T16:43:52.396635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loop over each segment\nfor idx, value in enumerate(steps):\n    l, h = value\n    if l == 0:\n        l = -50000000\n    if h == 50000000:\n        h = 100000000\n    # get segments that fall within the range\n    _test_df = test_pred_df[(test_pred_df['time_to_eruption'] >= l) & \n                            (test_pred_df['time_to_eruption'] <= h)]\n    _test_id = _test_df['segment_id']\n    _x_test = _test_df.drop(['segment_id', 'time_to_eruption'], axis=1)\n    _x_test = pd.DataFrame(scaler.transform(_x_test))\n    _x_test.columns = features\n    _preds = model_list[idx].predict(_x_test)\n    segment_id += list(_test_id) \n    time_to_eruption += list(_preds)","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:43:52.399915Z","iopub.execute_input":"2021-07-04T16:43:52.400379Z","iopub.status.idle":"2021-07-04T16:43:52.506643Z","shell.execute_reply.started":"2021-07-04T16:43:52.400329Z","shell.execute_reply":"2021-07-04T16:43:52.504818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nkaggle_submit = pd.DataFrame(list(zip(segment_id, time_to_eruption)))\nkaggle_submit.columns = ['segment_id', 'time_to_eruption']\nkaggle_submit = kaggle_submit.groupby('segment_id').median().reset_index()\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:43:52.508150Z","iopub.status.idle":"2021-07-04T16:43:52.508913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check our submission is good\nassert len(kaggle_submit) == 4520\nassert kaggle_submit['segment_id'].dtypes == 'int64'\nassert kaggle_submit['time_to_eruption'].dtypes == 'float64'","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:43:52.510215Z","iopub.status.idle":"2021-07-04T16:43:52.510707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle_submit.to_csv('./submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-04T16:43:52.511662Z","iopub.status.idle":"2021-07-04T16:43:52.512072Z"},"trusted":true},"execution_count":null,"outputs":[]}]}