\n
Data loading and features extraction
"},{"metadata":{},"cell_type":"markdown","source":"## 1: load the data\n### 1.1: create df with the sequence ID in training set"},{"metadata":{"trusted":true},"cell_type":"code","source":"# create empty list: filename_list \nfilename_list = [] \nfile_path = r'/kaggle/input/predict-volcanic-eruptions-ingv-oe/train'\nall_files = glob.glob(file_path + \"/*.csv\")\n\n# add filename to filename_list\nfilename_list.append(all_files)\n\n# Get the sequence number: remove the path and finaly the file type\nlist_sequence = [] \nfor file in all_files:\n file = file.split(\"/\")[-1]\n file = file.split(\".\")[-2]\n list_sequence.append(int(file))\n\ndf_list_sequence = pd.DataFrame(list_sequence)\ndf_list_sequence.columns=['segment_id']\ndf_list_sequence.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1.2: create df with the sequence ID in testing set"},{"metadata":{"trusted":true},"cell_type":"code","source":"# create empty list: filename_list \nfilename_list_test = [] \nfile_path_test = r'/kaggle/input/predict-volcanic-eruptions-ingv-oe/test'\nall_files_test = glob.glob(file_path_test + \"/*.csv\")\n\n# add filename to filename_list\nfilename_list_test.append(all_files_test)\n\n# add filename to filename_list\nfilename_list_test.append(all_files_test)\n\n# Get the sequence number: remove the path and finaly the file type\nlist_sequence_test = [] \nfor file in all_files_test:\n file = file.split(\"/\")[-1]\n file = file.split(\".\")[-2]\n list_sequence_test.append(int(file))\n\ndf_list_sequence_test = pd.DataFrame(list_sequence_test)\ndf_list_sequence_test.columns=['segment_id']\ndf_list_sequence_test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.3: load file with segment id and time to eruption"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/predict-volcanic-eruptions-ingv-oe/train.csv\")\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_= train['time_to_eruption'].hist(bins=20)\nprint(train['time_to_eruption'].skew())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2: attributes for feature engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Filter requirements.\n# T = 600.0 # Sample Period\n# fs = 100.0 # sample rate, Hz\n# cutoff = 20 # desired cutoff frequency of the filter, Hz ,\n# nyq = 0.5 * fs # Nyquist Frequency\n# normal_cutoff = cutoff / nyq\n# order = 2 # sin wave can be approx represented as quadratic\n# n = int(T * fs) # total number of samples\n\n# # Prepare the STFT\n# HOP_LENGTH = 2**6\n# SIZE_WINDOW = 2**9","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def butter_lowpass_filter(data, cutoff, fs, order):\n# \"\"\"\n# remove frequency lower than cutoff\n# Parameters\n# ----------\n# data : array_like\n# cutoff: int\n# fs: float\n# order: int\n# Returns\n# -------\n# y:numpy.ndarray\n# \"\"\"\n# # Get the filter coefficients \n# b, a = butter(order, normal_cutoff, btype='low', analog=False)\n# y = filtfilt(b, a, data)\n# return y\n\n# def compute_stat(var):\n# var_sum = np.sum(var)\n# var_max = np.max(var)\n# var_mean = np.mean(var)\n# var_std = np.std(var)\n# # var_p01 = np.percentile(var,1)\n# # var_p05 = np.percentile(var,5)\n# # var_p15 = np.percentile(var,15)\n# var_p25 = np.percentile(var,25)\n# # var_p35 = np.percentile(var,35)\n# # var_p45 = np.percentile(var,45)\n# var_p50 = np.percentile(var,50)\n# # var_p55 = np.percentile(var,55)\n# var_p75 = np.percentile(var,75)\n# # var_p85 = np.percentile(var,85)\n# # var_p90 = np.percentile(var,90)\n# var_p95 = np.percentile(var,95)\n# var_p99 = np.percentile(var,99)\n# var_skew = skew(var)\n# var_kurtosis = kurtosis(var)\n# result=[[var_sum,var_max,var_mean,var_std,var_skew,var_kurtosis,\\\n# var_p25,var_p50,var_p75,var_p95,var_p99]]\n# X = pd.DataFrame(result)\n \n# return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def extract_feature(list_files,list_sequence):\n# features_env = pd.DataFrame()\n# features_fft = pd.DataFrame()\n# features_centroids = pd.DataFrame()\n# features_bandwidths = pd.DataFrame()\n\n# for filename in list_files:\n# # load the file\n# sequence = pd.read_csv(filename)\n# # replace NaN value by '0'\n# sequence = sequence.fillna(0)\n\n# # ----------------------------- filter the signals\n# df_filter = pd.DataFrame() \n# for i in range(0,sequence.shape[1]):\n# filter_sequence = pd.DataFrame(butter_lowpass_filter(sequence.iloc[:,i], cutoff, fs, order=2))\n# df_filter = pd.concat([df_filter,filter_sequence],axis=1)\n\n# # ------------------------------ envelop\n# # calculate absolute signal\n# df_sequence_abs = df_filter.apply(np.abs)\n# features_env_1 = pd.DataFrame()\n# for i in range(0,df_sequence_abs.shape[1]):\n# stat = compute_stat(df_sequence_abs.iloc[:,i])\n# features_env_1 = pd.concat([features_env_1,stat],axis=1)\n \n# features_env = pd.concat([features_env,features_env_1]) \n \n# # ------------------------------ fft\n# # # FFT parameters\n# n=len(filter_sequence) # number of point\n# Lx =600 # time period\n# freqs = fftfreq(n) # Creates all the necessary frequencies\n# mask = freqs > 0 # mask array to be used for power spectra\n# ## fft\n# # fft_stat = pd.DataFrame()\n# features_fft_1 = pd.DataFrame()\n# features_bandwidths_1 = pd.DataFrame()\n# features_centroids_1 = pd.DataFrame()\n# for i in range(0,df_filter.shape[1]):\n# #----------------------------- extract features from fft\n# fft_vals = fft(df_filter.iloc[:, i].values)\n# fft_theo = 2.0*np.abs(fft_vals/n)\n# stat_fft = compute_stat(fft_theo[mask])\n# features_fft_1 = pd.concat([features_fft_1,stat_fft],axis=1) \n \n# # ------------------------ extract features from spectogram\n# # convert signal to numpy array float\n# arr = np.array(df_filter.iloc[:, i].values.astype(float))\n# # create the spectogramme\n# spec = stft(arr, hop_length=HOP_LENGTH, n_fft=SIZE_WINDOW)\n# # Convert into decibels\n# spec_db = amplitude_to_db(np.abs(spec))\n# # Calculate the spectral centroid and bandwidth for the spectrogram\n# bandwidths = lr.feature.spectral_bandwidth(S=np.abs(spec_db))[0]\n# centroids = lr.feature.spectral_centroid(S=np.abs(spec_db))[0]\n# # remove high values at the beginning and end\n# centroids = centroids[3:-3]\n# bandwidths = bandwidths[3:-3]\n# # feature extraction \n# stat_bandwidths = compute_stat(bandwidths)\n# stat_centroids = compute_stat(centroids)\n# features_bandwidths_1 = pd.concat([features_bandwidths_1,stat_bandwidths],axis=1)\n# features_centroids_1 = pd.concat([features_centroids_1,stat_centroids],axis=1)\n \n# features_fft = pd.concat([features_fft,features_fft_1])\n# features_bandwidths = pd.concat([features_bandwidths,features_bandwidths_1])\n# features_centroids = pd.concat([features_centroids,features_centroids_1])\n \n# # add column name\n# name_envelop = ['env_s1_sum','env_s1_max','env_s1_mean','env_s1_std','env_s1_skew','env_s1_kurtosis','env_s1_p25','env_s1_p50','env_s1_p75','env_s1_p95','env_s1_p99',\n# 'env_s2_sum','env_s2_max','env_s2_mean','env_s2_std','env_s2_skew','env_s2_kurtosis','env_s2_p25','env_s2_p50','env_s2_p75','env_s2_p95','env_s2_p99',\n# 'env_s3_sum','env_s3_max','env_s3_mean','env_s3_std','env_s3_skew','env_s3_kurtosis','env_s3_p25','env_s3_p50','env_s3_p75','env_s3_p95','env_s3_p99',\n# 'env_s4_sum','env_s4_max','env_s4_mean','env_s4_std','env_s4_skew','env_s4_kurtosis','env_s4_p25','env_s4_p50','env_s4_p75','env_s4_p95','env_s4_p99',\n# 'env_s5_sum','env_s5_max','env_s5_mean','env_s5_std','env_s5_skew','env_s5_kurtosis','env_s5_p25','env_s5_p50','env_s5_p75','env_s5_p95','env_s5_p99',\n# 'env_s6_sum','env_s6_max','env_s6_mean','env_s6_std','env_s6_skew','env_s6_kurtosis','env_s6_p25','env_s6_p50','env_s6_p75','env_s6_p95','env_s6_p99',\n# 'env_s7_sum','env_s7_max','env_s7_mean','env_s7_std','env_s7_skew','env_s7_kurtosis','env_s7_p25','env_s7_p50','env_s7_p75','env_s7_p95','env_s7_p99',\n# 'env_s8_sum','env_s8_max','env_s8_mean','env_s8_std','env_s8_skew','env_s8_kurtosis','env_s8_p25','env_s8_p50','env_s8_p75','env_s8_p95','env_s8_p99',\n# 'env_s9_sum','env_s9_max','env_s9_mean','env_s9_std','env_s9_skew','env_s9_kurtosis','env_s9_p25','env_s9_p50','env_s9_p75','env_s9_p95','env_s9_p99',\n# 'env_s10_sum','env_s10_max','env_s10_mean','env_s10_std','env_s10_skew','env_s10_kurtosis','env_s10_p25','env_s10_p50','env_s10_p75','env_s10_p95','env_s10_p99']\n# features_env.columns = name_envelop\n# # #\n# name_fft = ['fft_s1_sum','fft_s1_max','fft_s1_mean','fft_s1_std','fft_s1_skew','fft_s1_kurtosis','fft_s1_p25','fft_s1_p50','fft_s1_p75','fft_s1_p95','fft_s1_p99',\n# 'fft_s2_sum','fft_s2_max','fft_s2_mean','fft_s2_std','fft_s2_skew','fft_s2_kurtosis','fft_s2_p25','fft_s2_p50','fft_s2_p75','fft_s2_p95','fft_s2_p99',\n# 'fft_s3_sum','fft_s3_max','fft_s3_mean','fft_s3_std','fft_s3_skew','fft_s3_kurtosis','fft_s3_p25','fft_s3_p50','fft_s3_p75','fft_s3_p95','fft_s3_p99',\n# 'fft_s4_sum','fft_s4_max','fft_s4_mean','fft_s4_std','fft_s4_skew','fft_s4_kurtosis','fft_s4_p25','fft_s4_p50','fft_s4_p75','fft_s4_p95','fft_s4_p99',\n# 'fft_s5_sum','fft_s5_max','fft_s5_mean','fft_s5_std','fft_s5_skew','fft_s5_kurtosis','fft_s5_p25','fft_s5_p50','fft_s5_p75','fft_s5_p95','fft_s5_p99',\n# 'fft_s6_sum','fft_s6_max','fft_s6_mean','fft_s6_std','fft_s6_skew','fft_s6_kurtosis','fft_s6_p25','fft_s6_p50','fft_s6_p75','fft_s6_p95','fft_s6_p99',\n# 'fft_s7_sum','fft_s7_max','fft_s7_mean','fft_s7_std','fft_s7_skew','fft_s7_kurtosis','fft_s7_p25','fft_s7_p50','fft_s7_p75','fft_s7_p95','fft_s7_p99',\n# 'fft_s8_sum','fft_s8_max','fft_s8_mean','fft_s8_std','fft_s8_skew','fft_s8_kurtosis','fft_s8_p25','fft_s8_p50','fft_s8_p75','fft_s8_p95','fft_s8_p99',\n# 'fft_s9_sum','fft_s9_max','fft_s9_mean','fft_s9_std','fft_s9_skew','fft_s9_kurtosis','fft_s9_p25','fft_s9_p50','fft_s9_p75','fft_s9_p95','fft_s9_p99',\n# 'fft_s10_sum','fft_s10_max','fft_s10_mean','fft_s10_std','fft_s10_skew','fft_s10_kurtosis','fft_s10_p25','fft_s10_p50','fft_s10_p75','fft_s10_p95','fft_s10_p99']\n# features_fft.columns = name_fft\n\n# name_centroid = ['cent_s1_sum','cent_s1_max','cent_s1_mean','cent_s1_std','cent_s1_skew','cent_s1_kurtosis','cent_s1_p25','cent_s1_p50','cent_s1_p75','cent_s1_p95','cent_s1_p99',\n# 'cent_s2_sum','cent_s2_max','cent_s2_mean','cent_s2_std','cent_s2_skew','cent_s2_kurtosis','cent_s2_p25','cent_s2_p50','cent_s2_p75','cent_s2_p95','cent_s2_p99',\n# 'cent_s3_sum','cent_s3_max','cent_s3_mean','cent_s3_std','cent_s3_skew','cent_s3_kurtosis','cent_s3_p25','cent_s3_p50','cent_s3_p75','cent_s3_p95','cent_s3_p99',\n# 'cent_s4_sum','cent_s4_max','cent_s4_mean','cent_s4_std','cent_s4_skew','cent_s4_kurtosis','cent_s4_p25','cent_s4_p50','cent_s4_p75','cent_s4_p95','cent_s4_p99',\n# 'cent_s5_sum','cent_s5_max','cent_s5_mean','cent_s5_std','cent_s5_skew','cent_s5_kurtosis','cent_s5_p25','cent_s5_p50','cent_s5_p75','cent_s5_p95','cent_s5_p99',\n# 'cent_s6_sum','cent_s6_max','cent_s6_mean','cent_s6_std','cent_s6_skew','cent_s6_kurtosis','cent_s6_p25','cent_s6_p50','cent_s6_p75','cent_s6_p95','cent_s6_p99',\n# 'cent_s7_sum','cent_s7_max','cent_s7_mean','cent_s7_std','cent_s7_skew','cent_s7_kurtosis','cent_s7_p25','cent_s7_p50','cent_s7_p75','cent_s7_p95','cent_s7_p99',\n# 'cent_s8_sum','cent_s8_max','cent_s8_mean','cent_s8_std','cent_s8_skew','cent_s8_kurtosis','cent_s8_p25','cent_s8_p50','cent_s8_p75','cent_s8_p95','cent_s8_p99',\n# 'cent_s9_sum','cent_s9_max','cent_s9_mean','cent_s9_std','cent_s9_skew','cent_s9_kurtosis','cent_s9_p25','cent_s9_p50','cent_s9_p75','cent_s9_p95','cent_s9_p99',\n# 'cent_s10_sum','cent_s10_max','cent_s10_mean','cent_s10_std','cent_s10_skew','cent_s10_kurtosis','cent_s10_p25','cent_s10_p50','cent_s10_p75','cent_s10_p95','cent_s10_p99']\n# features_centroids.columns = name_centroid\n\n# name_bandwidths = ['band_s1_sum','band_s1_max','band_s1_mean','band_s1_std','band_s1_skew','band_s1_kurtosis','band_s1_p25','band_s1_p50','band_s1_p75','band_s1_p95','band_s1_p99',\n# 'band_s2_sum','band_s2_max','band_s2_mean','band_s2_std','band_s2_skew','band_s2_kurtosis','band_s2_p25','band_s2_p50','band_s2_p75','band_s2_p95','band_s2_p99',\n# 'band_s3_sum','band_s3_max','band_s3_mean','band_s3_std','band_s3_skew','band_s3_kurtosis','band_s3_p25','band_s3_p50','band_s3_p75','band_s3_p95','band_s3_p99',\n# 'band_s4_sum','band_s4_max','band_s4_mean','band_s4_std','band_s4_skew','band_s4_kurtosis','band_s4_p25','band_s4_p50','band_s4_p75','band_s4_p95','band_s4_p99',\n# 'band_s5_sum','band_s5_max','band_s5_mean','band_s5_std','band_s5_skew','band_s5_kurtosis','band_s5_p25','band_s5_p50','band_s5_p75','band_s5_p95','band_s5_p99',\n# 'band_s6_sum','band_s6_max','band_s6_mean','band_s6_std','band_s6_skew','band_s6_kurtosis','band_s6_p25','band_s6_p50','band_s6_p75','band_s6_p95','band_s6_p99',\n# 'band_s7_sum','band_s7_max','band_s7_mean','band_s7_std','band_s7_skew','band_s7_kurtosis','band_s7_p25','band_s7_p50','band_s7_p75','band_s7_p95','band_s7_p99',\n# 'band_s8_sum','band_s8_max','band_s8_mean','band_s8_std','band_s8_skew','band_s8_kurtosis','band_s8_p25','band_s8_p50','band_s8_p75','band_s8_p95','band_s8_p99',\n# 'band_s9_sum','band_s9_max','band_s9_mean','band_s9_std','band_s9_skew','band_s9_kurtosis','band_s9_p25','band_s9_p50','band_s9_p75','band_s9_p95','band_s9_p99',\n# 'band_s10_sum','band_s10_max','band_s10_mean','band_s10_std','band_s10_skew','band_s10_kurtosis','band_s10_p25','band_s10_p50','band_s10_p75','band_s10_p95','band_s10_p99']\n# features_bandwidths.columns = name_bandwidths\n\n\n# # add segment_id\n# features = pd.concat([list_sequence,features_env,features_fft,features_centroids,features_bandwidths],axis=1)\n# # features = features_bandwidths\n# return features\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## FEATURES EXTRACTION"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_features = extract_feature(all_files,df_list_sequence)\n# train_features = pd.merge(train_features, train, on=['segment_id', 'segment_id'])\n# train_features.to_csv('train_features2.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_features = extract_feature(all_files_test,df_list_sequence_test)\ntest_features.to_csv('test_features2.csv')\ntest_features.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2: Load the feature data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_features = pd.read_csv(\"../input/features2/train_features2.csv\")\ntarget = train_features[['time_to_eruption']]\ntrain_features = train_features.drop(columns=['Unnamed: 0','time_to_eruption','index'])\ntrain_features.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_features = pd.read_csv(\"../input/features2/test_features2.csv\")\ntest_features = test_features.drop(columns=['Unnamed: 0','index'])\n# test_features = test_features.drop(columns=['segment_id'])\ntest_features.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.1: Scale the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import RobustScaler\nscaler= RobustScaler()\n# transform \"train_features\"\ntrain_features_scaled = scaler.fit_transform(train_features)\n# transform \"test_features\"\ntest_features_scaled = scaler.transform(test_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_features_scaled = pd.DataFrame(train_features_scaled)\ndf_test_features_scaled = pd.DataFrame(test_features_scaled)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.2: calculate correlation between feature data and time to eruption"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_cor = abs(df_train_features_scaled[df_train_features_scaled.columns[1:-1]].apply(lambda x: x.corr(target['time_to_eruption']))).sort_values(ascending=False)\ndf_cor","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.3: select meaningful features"},{"metadata":{"trusted":true},"cell_type":"code","source":"best_corr = pd.DataFrame(df_cor[:100])\nbest_corr.reset_index(inplace = True)\nbest_corr.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col_name_feature_importance = best_corr.iloc[:,0]\ncol_name_feature_importance","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"
\n\n
\n
3: MODEL
"},{"metadata":{},"cell_type":"markdown","source":"## 3.1: function to evaluate the models"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\ndef score_RMSE(y_pred):\n return str(math.sqrt(mean_squared_error(y_test, y_pred)))\n\ndef get_best_score(grid):\n \n best_score = np.sqrt(-grid.best_score_)\n print(best_score) \n print(grid.best_params_)\n print(grid.best_estimator_)\n \n return best_score\n\nfrom statsmodels.graphics.api import abline_plot\ndef model_evaluation(prediction):\n print(\"R2 (explained variance):\", round(metrics.r2_score(y_test, prediction), 2))\n print(\"Mean Absolute Perc Error (Σ(|y-pred|/y)/n):\", np.mean(np.abs((y_test-prediction)/prediction)))\n print(\"Mean Absolute Error (Σ|y-pred|/n):\", \"{:,f}\".format(metrics.mean_absolute_error(y_test, prediction)))\n print(\"Root Mean Squared Error (sqrt(Σ(y-pred)^2/n)):\", \"{:,f}\".format(np.sqrt(metrics.mean_squared_error(y_test, prediction))))\n ## residuals\n prediction = prediction.reshape(len(prediction),1)\n residuals = y_test - prediction\n if abs(max(residuals)) > abs(min(residuals)):\n max_error = max(residuals) \n else:\n max_error = min(residuals) \n max_idx = list(residuals).index(max(residuals)) if abs(max(residuals)) > abs(min(residuals)) else list(residuals).index(min(residuals))\n max_true = y_test[max_idx]\n max_pred = prediction[max_idx]\n print(\"Max Error:\", \"{}\".format(max_error))\n \n ## Plot predicted vs true\n fig, ax = plt.subplots(nrows=1, ncols=2,figsize=(10,5))\n ax[0].scatter(prediction, y_test, color=\"black\")\n abline_plot(intercept=0, slope=1, color=\"red\", ax=ax[0])\n ax[0].vlines(x=max_pred, ymin=max_true, ymax=max_true-max_error, color='red', linestyle='--', alpha=0.7, label=\"max error\")\n ax[0].grid(True)\n ax[0].set(xlabel=\"Predicted\", ylabel=\"True\", title=\"Predicted vs True\")\n ax[0].legend()\n\n ## Plot predicted vs residuals\n ax[1].scatter(prediction, residuals, color=\"red\")\n ax[1].vlines(x=max_pred, ymin=0, ymax=max_error, color='black', linestyle='--', alpha=0.7, label=\"max error\")\n ax[1].grid(True)\n ax[1].set(xlabel=\"Predicted\", ylabel=\"Residuals\", title=\"Predicted vs Residuals\")\n ax[1].hlines(y=0, xmin=np.min(prediction), xmax=np.max(prediction))\n ax[1].legend()\n plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.1: Linear regression"},{"metadata":{},"cell_type":"markdown","source":"### 3.1.1: Find the optimal number of features to use for ML"},{"metadata":{"trusted":true},"cell_type":"code","source":"# linreg_score = []\n# number_feature = []\n# for i in range (1,100):\n# # select diferent \n# columns = col_name_feature_importance[:i]\n# X = df_train_features_scaled[columns].values\n# y = target.values\n# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=12)\n \n# linreg = LinearRegression()\n# parameters = {'fit_intercept':[True,False], 'normalize':[True,False], 'copy_X':[True, False]}\n# grid_linear = GridSearchCV(linreg, parameters, cv=12, verbose=1 , scoring = 'neg_mean_squared_error')\n# grid_linear.fit(X_train, y_train)\n\n# sc_linear = get_best_score(grid_linear)\n# linreg_score.append(sc_linear)\n# number_feature.append(i)\n\n# result = pd.DataFrame(zip(number_feature,linreg_score),columns = ['number_feat', 'best_score'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # PLOT RESULT:\n# result.plot('number_feat', 'best_score')\n\n# # Returns index of minimun best_score\n# index = result[['best_score']].idxmin() \n\n# # get the number of features used to have the best score \n# print(result['number_feat'][index])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3.1.2: Gridsearch"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # get the 15 best correlated columns\n# columns = col_name_feature_importance[:11]\n\n# # slipt the data\n# X = df_train_features_scaled.values\n# y = target.values\n# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=12)\n\n# # linear regression model and gridsearch\n# linreg = LinearRegression()\n# parameters = {'fit_intercept':[True,False], 'normalize':[True,False], 'copy_X':[True, False]}\n# grid_linear = GridSearchCV(linreg, parameters, cv=12, verbose=1 , scoring = 'neg_mean_squared_error')\n# grid_linear.fit(X_train, y_train)\n\n# sc_linear = get_best_score(grid_linear)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3.1.3: run linear regression with best score"},{"metadata":{"trusted":true},"cell_type":"code","source":"# model_LinReg = linear_model.LinearRegression(copy_X= True, fit_intercept= True, normalize= False)\n# model_LinReg.fit(X_train,y_train)\n# prediction = model_LinReg.predict(X_test)\n# model_evaluation(prediction)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.2: XGBRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"from xgboost import XGBRegressor\n\nX = df_train_features_scaled\ny = target\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=12)\n \nxgb = XGBRegressor(n_estimators=1500)\nxgb.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb.feature_importances_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sort it with descending\nsorted_idx = np.argsort(xgb.feature_importances_)[::-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_importance =[]\nfor index in sorted_idx:\n print([X_train.columns[index], xgb.feature_importances_[index]]) \n feature_importance.append(X_train.columns[index])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from xgboost import plot_importance\nplot_importance(xgb, max_num_features = 100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import xgboostxgboost as xgb\nimport xgboost as xgb\n\nbest_feature_importance = feature_importance[:78]\n\nX = df_train_features_scaled[best_feature_importance]\ny = target\n\n# X = train_features[best_feature_importance]\ny = target.values\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=12)\n\n# # Create the DMatrix: housing_dmatrix\ntrain_dmatrix = xgb.DMatrix(data=X_train, label=y_train)\n\n# # Instantiate the regressor: gbm\n# gbm = xgb.XGBRegressor(objective='reg:linear',n_estimators=1500)\n\n# gbm_param_grid = {\n# 'colsample_bytree': [0.1,0.2,0.3,0.4,0.6,0.8],\n# 'max_depth': range(3,10,1), \n# 'eta' : [0.01,0.02,0.03,0.04,0.05,0.1,0.15,0.2,0.3,0.4],\n# 'lambda': [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1],\n# 'alpha': [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1],\n# 'subsample': [0.2,0.3,0.4,0.5,0.6,0.7,0.8]\n# }\n\n# randomized_mse = RandomizedSearchCV(estimator=gbm,\n# param_distributions=gbm_param_grid,\n# scoring=\"neg_mean_squared_error\",\n# n_iter=200,cv=4, verbose=1)\n\n# # Fit randomized_mse to the data\n# grid_xgb = randomized_mse.fit(X_train,y_train)\n\n# # Print the best parameters and lowest RMSE\n# sc_xgb = get_best_score(grid_xgb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Instantiate the XGBRegressor: xg_reg\nimport xgboost as xgb\n\n# Create the DMatrix: housing_dmatrix\ntrain_dmatrix = xgb.DMatrix(data=X_train, label=y_train)\n\nxgb = xgb.XGBRegressor(objective='reg:linear',n_estimators = 1500,subsample = 0.6,\n max_depth = 9, eta = 0.01,colsample_bytree = 0.4,alpha = 0.3,reg_lambda = 0.2)\n\n# subsample': 0.4, 'max_depth': 9, 'lambda': 0.8, 'eta': 0.02, 'colsample_bytree': 0.8, 'alpha': 0.2\n \n# Fit the regressor to the training set\nxgb.fit(X_train,y_train)\n\n## Predicting the target value based on \"Test_x\"\ny_pred = xgb.predict(X_test)\n\nmodel_evaluation(y_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"with feature: 66 \nBest parameters found: {objective='reg:linear',n_estimators = 1500,subsample = 0.5,max_depth = 9, eta = 0.02,colsample_bytree = 0.8,alpha = 0,reg_lambda = 0.3}\nLowest RMSE found: 4,262,837.775371\n\nwith feature: 68 \nBest parameters found: {subsample = 0.6,max_depth = 8, eta = 0.01,colsample_bytree = 1,alpha = 0.5,reg_lambda = 0.2)}\nLowest RMSE found: 4,259,511.986089\n\nwith feature: 41\nBest parameters found: {'subsample = 0.6,max_depth = 9, eta = 0.01,colsample_bytree = 0.4,alpha = 0.3,reg_lambda = 0.2}\nLowest RMSE found: 4,259,582.235790\n\nwith feature: 51\nBest parameters found: {'subsample': 0.7, 'max_depth': 7, 'eta': 0.1, 'colsample_bytree': 0.8}\nLowest RMSE found: 7,005,340.813945933\n\nwith feature: 56\nBest parameters found: {'subsample': 0.8, 'max_depth': 8, 'eta': 0.1, 'colsample_bytree': 0.8}\nLowest RMSE found: 6,986,756.825361339\n\nwith feature: 61\nBest parameters found: {'subsample': 0.8, 'max_depth': 8, 'eta': 0.1, 'colsample_bytree': 0.6}\nLowest RMSE found: 6,796,230.816416972\n\nwith feature: 71\nBest parameters found: {'subsample': 0.7, 'max_depth': 8, 'eta': 0.15, 'colsample_bytree': 0.8}\nLowest RMSE found: 6,862,673.756157105\n\nwith feature: 61\nBest parameters found: {'subsample': 0.4, 'n_estimators': 500, 'max_depth': 9, 'eta': 0.05, 'colsample_bytree': 0.6}\nLowest RMSE found: 6,569,617.031801722\n\nwith feature: 66\nBest parameters found: {'subsample': 0.6, 'n_estimators': 500, 'max_depth': 8, 'eta': 0.03, 'colsample_bytree': 0.4}\nLowest RMSE found: 6,532,065.060200441\n\nwith feature: 66\nBest parameters found: {'subsample': 0.5, 'n_estimators': 900, 'max_depth': 7, 'eta': 0.04, 'colsample_bytree': 0.6}\nLowest RMSE found: 6,410,294.69215607\n\nwith feature: 67\nBest parameters found: {'subsample': 0.5, 'n_estimators': 900, 'max_depth': 7, 'eta': 0.02, 'colsample_bytree': 0.8}\nLowest RMSE found: 6441142.32956806"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"
\n\n
\n
XGBRegressor: xg_reg
"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import xgboostxgboost as xgb\nimport xgboost as xgb\n\nbest_feature_importance = feature_importance[:68]\n\nX = df_train_features_scaled[best_feature_importance]\ny = target.values\n\n# # Create the DMatrix: housing_dmatrix\ntrain_dmatrix = xgb.DMatrix(data=X, label=y)\n\n# # Instantiate the regressor: gbm\n# gbm = xgb.XGBRegressor(objective='reg:linear',n_estimators=1500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Instantiate the XGBRegressor: xg_reg\nimport xgboost as xgb\n\nbest_feature_importance = feature_importance[:68]\n\nX = df_train_features_scaled[best_feature_importance]\ny = target.values\n\n# # Create the DMatrix: housing_dmatrix\ntrain_dmatrix = xgb.DMatrix(data=X, label=y)\n\nxgb = xgb.XGBRegressor(objective='reg:linear',n_estimators = 1500,subsample = 0.6,\n max_depth = 8, eta = 0.01,colsample_bytree = 1,alpha = 0.5,reg_lambda = 0.2)\n \n# Fit the regressor to the training set\nxgb.fit(X,y)\n\ntest_features_scaled = df_test_features_scaled[best_feature_importance]\n\n## Predicting the target value based on \"Test_x\"\ny_pred = xgb.predict(test_features_scaled)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['segment_id'] = df_list_sequence_test[\"segment_id\"]\nsubmission['time_to_eruption'] = y_pred\nsubmission.head()\nsubmission.to_csv('submission5.csv', header=True, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}