{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import numpy as npp\nimport pandas as pd\nimport os\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm_notebook\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.metrics import mean_absolute_error\npd.options.display.precision = 15\n\nimport lightgbm as lgb\nimport xgboost as xgb\nimport time\nimport datetime\nfrom catboost import CatBoostRegressor\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.linear_model import LinearRegression\nimport gc\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom scipy.signal import hilbert\nfrom scipy.signal import hann\nfrom scipy.signal import convolve\nfrom scipy import stats\nfrom sklearn.kernel_ridge import KernelRidge\nfrom itertools import product\n\nfrom tsfresh.feature_extraction import feature_calculators\nfrom joblib import Parallel, delayed","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"# Create a training file with simple derived features\n\ndef add_trend_feature(arr, abs_values=False):\n    idx = np.array(range(len(arr)))\n    if abs_values:\n        arr = np.abs(arr)\n    lr = LinearRegression()\n    lr.fit(idx.reshape(-1, 1), arr)\n    return lr.coef_[0]\n\ndef classic_sta_lta(x, length_sta, length_lta):\n    \n    sta = np.cumsum(x ** 2)\n\n    # Convert to float\n    sta = np.require(sta, dtype=np.float)\n\n    # Copy for LTA\n    lta = sta.copy()\n\n    # Compute the STA and the LTA\n    sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]\n    sta /= length_sta\n    lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]\n    lta /= length_lta\n\n    # Pad zeros\n    sta[:length_lta - 1] = 0\n\n    # Avoid division by zero by setting zero values to tiny float\n    dtiny = np.finfo(0.0).tiny\n    idx = lta < dtiny\n    lta[idx] = dtiny\n\n    return sta / lta\n\ndef calc_change_rate(x):\n    change = (np.diff(x) / x[:-1]).values\n    change = change[np.nonzero(change)[0]]\n    change = change[~np.isnan(change)]\n    change = change[change != -np.inf]\n    change = change[change != np.inf]\n    return np.mean(change)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm_notebook\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.metrics import mean_absolute_error\npd.options.display.precision = 15\n\nimport lightgbm as lgb\nimport xgboost as xgb\nimport time\nimport datetime\nfrom catboost import CatBoostRegressor\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.linear_model import LinearRegression\nimport gc\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom scipy.signal import hilbert\nfrom scipy.signal import hann\nfrom scipy.signal import convolve\nfrom scipy import stats\nfrom sklearn.kernel_ridge import KernelRidge\nfrom itertools import product\n\nfrom tsfresh.feature_extraction import feature_calculators\nfrom joblib import Parallel, delayed\n# Create a training file with simple derived features\n\ndef add_trend_feature(arr, abs_values=False):\n    idx = np.array(range(len(arr)))\n    if abs_values:\n        arr = np.abs(arr)\n    lr = LinearRegression()\n    lr.fit(idx.reshape(-1, 1), arr)\n    return lr.coef_[0]\n\ndef classic_sta_lta(x, length_sta, length_lta):\n    \n    sta = np.cumsum(x ** 2)\n\n    # Convert to float\n    sta = np.require(sta, dtype=np.float)\n\n    # Copy for LTA\n    lta = sta.copy()\n\n    # Compute the STA and the LTA\n    sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]\n    sta /= length_sta\n    lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]\n    lta /= length_lta\n\n    # Pad zeros\n    sta[:length_lta - 1] = 0\n\n    # Avoid division by zero by setting zero values to tiny float\n    dtiny = np.finfo(0.0).tiny\n    idx = lta < dtiny\n    lta[idx] = dtiny\n\n    return sta / lta\n\ndef calc_change_rate(x):\n    change = (np.diff(x) / x[:-1]).values\n    change = change[np.nonzero(change)[0]]\n    change = change[~np.isnan(change)]\n    change = change[change != -np.inf]\n    change = change[change != np.inf]\n    return np.mean(change)\nclass FeatureGenerator(object):\n    def __init__(self, dtype, n_jobs=1, chunk_size=None):\n        self.chunk_size = chunk_size\n        self.dtype = dtype\n        self.filename = None\n        self.n_jobs = n_jobs\n        self.test_files = []\n        if self.dtype == 'train':\n            self.filename = '../input/train.csv'\n            self.total_data = int(629145481 / self.chunk_size)\n        else:\n            submission = pd.read_csv('../input/sample_submission.csv')\n            for seg_id in submission.seg_id.values:\n                self.test_files.append((seg_id, '../input/test/' + seg_id + '.csv'))\n            self.total_data = int(len(submission))\n\n    def read_chunks(self):\n        if self.dtype == 'train':\n            iter_df = pd.read_csv(self.filename, iterator=True, chunksize=self.chunk_size,\n                                  dtype={'acoustic_data': np.float64, 'time_to_failure': np.float64})\n            for counter, df in enumerate(iter_df):\n                x = df.acoustic_data.values\n                y = df.time_to_failure.values[-1]\n                seg_id = 'train_' + str(counter)\n                del df\n                yield seg_id, x, y\n        else:\n            for seg_id, f in self.test_files:\n                df = pd.read_csv(f, dtype={'acoustic_data': np.float64})\n                x = df.acoustic_data.values[-self.chunk_size:]\n                del df\n                yield seg_id, x, -999\n    \n    def get_features(self, x, y, seg_id):\n        \"\"\"\n        Gets three groups of features: from original data and from reald and imaginary parts of FFT.\n        \"\"\"\n        \n        x = pd.Series(x)\n    \n        zc = np.fft.fft(x)\n        realFFT = pd.Series(np.real(zc))\n        imagFFT = pd.Series(np.imag(zc))\n        \n        main_dict = self.features(x, y, seg_id)\n        r_dict = self.features(realFFT, y, seg_id)\n        i_dict = self.features(imagFFT, y, seg_id)\n        \n        for k, v in r_dict.items():\n            if k not in ['target', 'seg_id']:\n                main_dict[f'fftr_{k}'] = v\n                \n        for k, v in i_dict.items():\n            if k not in ['target', 'seg_id']:\n                main_dict[f'ffti_{k}'] = v\n        \n        return main_dict\n        \n    \n    def features(self, x, y, seg_id):\n        feature_dict = dict()\n        feature_dict['target'] = y\n        feature_dict['seg_id'] = seg_id\n\n        # create features here\n\n        # lists with parameters to iterate over them\n        percentiles = [1, 5, 10, 20, 25, 30, 40, 50, 60, 70, 75, 80, 90, 95, 99]\n        percentiles1 = [5,10,20]\n        hann_windows = [50, 150, 1500, 15000]\n        spans = [300, 3000, 30000, 50000]\n        windows = [10, 50, 100, 500, 1000, 10000]\n        borders = list(range(-4000, 4001, 1000))\n        peaks = [10, 20, 50, 100]\n        coefs = [1, 5, 10, 50, 100]\n        lags = [10, 100, 1000, 10000]\n        autocorr_lags = [5, 10, 50, 100, 500, 1000, 5000, 10000]\n\n        # basic stats\n#         feature_dict['mean'] = x.mean()\n#         feature_dict['std'] = x.std()\n#         feature_dict['max'] = x.max()\n#         feature_dict['min'] = x.min()\n\n#         # basic stats on absolute values\n#         feature_dict['mean_change_abs'] = np.mean(np.diff(x))\n#         feature_dict['abs_max'] = np.abs(x).max()\n#         feature_dict['abs_mean'] = np.abs(x).mean()\n#         feature_dict['abs_std'] = np.abs(x).std()\n\n#         # geometric and harminic means\n#         feature_dict['hmean'] = stats.hmean(np.abs(x[np.nonzero(x)[0]]))\n#         feature_dict['gmean'] = stats.gmean(np.abs(x[np.nonzero(x)[0]])) \n\n#         # k-statistic and moments\n#         for i in range(1, 5):\n#             feature_dict[f'kstat_{i}'] = stats.kstat(x, i)\n#             feature_dict[f'moment_{i}'] = stats.moment(x, i)\n\n#         for i in [1, 2]:\n#             feature_dict[f'kstatvar_{i}'] = stats.kstatvar(x, i)\n        # note!\n        feature_dict['std_first_50000'] = x[:50000].agg('std')\n        # aggregations on various slices of data\n#         for agg_type, slice_length, direction in product(['std', 'min', 'max', 'mean'], [1000, 10000, 50000], ['first', 'last']):\n#             if direction == 'first':\n#                 feature_dict[f'{agg_type}_{direction}_{slice_length}'] = x[:slice_length].agg(agg_type)\n#             elif direction == 'last':\n#                 feature_dict[f'{agg_type}_{direction}_{slice_length}'] = x[-slice_length:].agg(agg_type)\n\n#         feature_dict['max_to_min'] = x.max() / np.abs(x.min())\n#         feature_dict['max_to_min_diff'] = x.max() - np.abs(x.min())\n#         feature_dict['count_big'] = len(x[np.abs(x) > 500])\n#         feature_dict['sum'] = x.sum()\n\n#         feature_dict['mean_change_rate'] = calc_change_rate(x)\n        # calc_change_rate on slices of data\n#         for slice_length, direction in product([1000, 10000, 50000], ['first', 'last']):\n#             if direction == 'first':\n#                 feature_dict[f'mean_change_rate_{direction}_{slice_length}'] = calc_change_rate(x[:slice_length])\n#             elif direction == 'last':\n#                 feature_dict[f'mean_change_rate_{direction}_{slice_length}'] = calc_change_rate(x[-slice_length:])\n\n        # percentiles on original and absolute values\n        feature_dict['percentile_25'] = np.percentile(x, 25)\n        \n        for p in percentiles1:\n            feature_dict[f'abs_percentile_{p}'] = np.percentile(np.abs(x), p)\n\n#         feature_dict['trend'] = add_trend_feature(x)\n#         feature_dict['abs_trend'] = add_trend_feature(x, abs_values=True)\n\n#         feature_dict['mad'] = x.mad()\n#         feature_dict['kurt'] = x.kurtosis()\n#         feature_dict['skew'] = x.skew()\n#         feature_dict['med'] = x.median()\n\n#         feature_dict['Hilbert_mean'] = np.abs(hilbert(x)).mean()\n\n#         for hw in hann_windows:\n#             feature_dict[f'Hann_window_mean_{hw}'] = (convolve(x, hann(hw), mode='same') / sum(hann(hw))).mean()\n\n#         feature_dict['classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000).mean()\n#         feature_dict['classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000).mean()\n#         feature_dict['classic_sta_lta3_mean'] = classic_sta_lta(x, 3333, 6666).mean()\n#         feature_dict['classic_sta_lta4_mean'] = classic_sta_lta(x, 10000, 25000).mean()\n#         feature_dict['classic_sta_lta5_mean'] = classic_sta_lta(x, 50, 1000).mean()\n#         feature_dict['classic_sta_lta6_mean'] = classic_sta_lta(x, 100, 5000).mean()\n#         feature_dict['classic_sta_lta7_mean'] = classic_sta_lta(x, 333, 666).mean()\n#         feature_dict['classic_sta_lta8_mean'] = classic_sta_lta(x, 4000, 10000).mean()\n\n        # exponential rolling statistics\n        ewma = pd.Series.ewm\n        for s in spans:\n#             feature_dict[f'exp_Moving_average_{s}_mean'] = (ewma(x, span=s).mean(skipna=True)).mean(skipna=True)\n#             feature_dict[f'exp_Moving_average_{s}_std'] = (ewma(x, span=s).mean(skipna=True)).std(skipna=True)\n#             feature_dict[f'exp_Moving_std_{s}_mean'] = (ewma(x, span=s).std(skipna=True)).mean(skipna=True)\n            feature_dict[f'exp_Moving_std_{s}_std'] = (ewma(x, span=s).std(skipna=True)).std(skipna=True)\n\n        feature_dict['iqr'] = np.subtract(*np.percentile(x, [75, 25]))\n#         feature_dict['iqr1'] = np.subtract(*np.percentile(x, [95, 5]))\n#         feature_dict['ave10'] = stats.trim_mean(x, 0.1)\n        \n        for slice_length, threshold in product([50000, 100000, 150000],\n                                                     [10, 50, 100]):\n            feature_dict[f'count_big_{slice_length}_threshold_{threshold}'] = (np.abs(x[-slice_length:]) > threshold).sum()\n            feature_dict[f'count_big_{slice_length}_less_threshold_{threshold}'] = (np.abs(x[-slice_length:]) < threshold).sum()\n\n        # tfresh features take too long to calculate, so I comment them for now\n\n#         feature_dict['abs_energy'] = feature_calculators.abs_energy(x)\n#         feature_dict['abs_sum_of_changes'] = feature_calculators.absolute_sum_of_changes(x)\n#         feature_dict['count_above_mean'] = feature_calculators.count_above_mean(x)\n#         feature_dict['count_below_mean'] = feature_calculators.count_below_mean(x)\n#         feature_dict['mean_abs_change'] = feature_calculators.mean_abs_change(x)\n#         feature_dict['mean_change'] = feature_calculators.mean_change(x)\n#         feature_dict['var_larger_than_std_dev'] = feature_calculators.variance_larger_than_standard_deviation(x)\n#         feature_dict['range_minf_m4000'] = feature_calculators.range_count(x, -np.inf, -4000)\n#         feature_dict['range_p4000_pinf'] = feature_calculators.range_count(x, 4000, np.inf)\n\n        for i, j in zip(borders, borders[1:]):\n            feature_dict[f'range_{i}_{j}'] = feature_calculators.range_count(x, i, j)\n\n#         feature_dict['ratio_unique_values'] = feature_calculators.ratio_value_number_to_time_series_length(x)\n#         feature_dict['first_loc_min'] = feature_calculators.first_location_of_minimum(x)\n#         feature_dict['first_loc_max'] = feature_calculators.first_location_of_maximum(x)\n#         feature_dict['last_loc_min'] = feature_calculators.last_location_of_minimum(x)\n#         feature_dict['last_loc_max'] = feature_calculators.last_location_of_maximum(x)\n\n#         for lag in lags:\n#             feature_dict[f'time_rev_asym_stat_{lag}'] = feature_calculators.time_reversal_asymmetry_statistic(x, lag)\n#         for autocorr_lag in autocorr_lags:\n#             feature_dict[f'autocorrelation_{autocorr_lag}'] = feature_calculators.autocorrelation(x, autocorr_lag)\n#             feature_dict[f'c3_{autocorr_lag}'] = feature_calculators.c3(x, autocorr_lag)\n\n#         for coeff, attr in product([1, 2, 3, 4, 5], ['real', 'imag', 'angle']):\n#             feature_dict[f'fft_{coeff}_{attr}'] = list(feature_calculators.fft_coefficient(x, [{'coeff': coeff, 'attr': attr}]))[0][1]\n\n#         feature_dict['long_strk_above_mean'] = feature_calculators.longest_strike_above_mean(x)\n#         feature_dict['long_strk_below_mean'] = feature_calculators.longest_strike_below_mean(x)\n#         feature_dict['cid_ce_0'] = feature_calculators.cid_ce(x, 0)\n#         feature_dict['cid_ce_1'] = feature_calculators.cid_ce(x, 1)\n\n#         for p in percentiles:\n#             feature_dict[f'binned_entropy_{p}'] = feature_calculators.binned_entropy(x, p)\n\n#         feature_dict['num_crossing_0'] = feature_calculators.number_crossing_m(x, 0)\n        \n    \n        feature_dict['num_peaks_10'] = feature_calculators.number_peaks(x, 10)\n        \n#         for peak in peaks:\n#             feature_dict[f'num_peaks_{peak}'] = feature_calculators.number_peaks(x, peak)\n\n#         for c in coefs:\n#             feature_dict[f'spkt_welch_density_{c}'] = list(feature_calculators.spkt_welch_density(x, [{'coeff': c}]))[0][1]\n#             feature_dict[f'time_rev_asym_stat_{c}'] = feature_calculators.time_reversal_asymmetry_statistic(x, c)  \n\n        # statistics on rolling windows of various sizes\n        feature_dict['min_roll_std_1000'] = x.rolling(1000).std().dropna().values.min()\n        feature_dict['min_roll_std_10000'] = x.rolling(10000).std().dropna().values.min()\n\n        feature_dict[f'std_roll_std_{10}'] = x.rolling(10).std().dropna().values.std()\n        \n        for w in [1000,10000]:\n            x_roll_std = x.rolling(w).std().dropna().values\n            feature_dict[f'abs_max_roll_std_{w}'] = np.abs(x_roll_std).max()\n        \n        for w in windows:\n            x_roll_std = x.rolling(w).std().dropna().values\n            x_roll_mean = x.rolling(w).mean().dropna().values\n\n#             feature_dict[f'ave_roll_std_{w}'] = x_roll_std.mean()\n#             feature_dict[f'std_roll_std_{w}'] = x_roll_std.std()\n#             feature_dict[f'max_roll_std_{w}'] = x_roll_std.max()\n#             feature_dict[f'min_roll_std_{w}'] = x_roll_std.min()\n\n            for p in percentiles:\n                feature_dict[f'percentile_roll_std_{p}_window_{w}'] = np.percentile(x_roll_std, p)\n\n#             feature_dict[f'av_change_abs_roll_std_{w}'] = np.mean(np.diff(x_roll_std))\n#             feature_dict[f'av_change_rate_roll_std_{w}'] = np.mean(np.nonzero((np.diff(x_roll_std) / x_roll_std[:-1]))[0])\n#             feature_dict[f'abs_max_roll_std_{w}'] = np.abs(x_roll_std).max()\n\n#             feature_dict[f'ave_roll_mean_{w}'] = x_roll_mean.mean()\n#             feature_dict[f'std_roll_mean_{w}'] = x_roll_mean.std()\n#             feature_dict[f'max_roll_mean_{w}'] = x_roll_mean.max()\n#             feature_dict[f'min_roll_mean_{w}'] = x_roll_mean.min()\n\n            for p in [70,99,10]:\n                feature_dict[f'percentile_roll_mean_{p}_window_{w}'] = np.percentile(x_roll_mean, p)\n\n#             feature_dict[f'av_change_abs_roll_mean_{w}'] = np.mean(np.diff(x_roll_mean))\n#             feature_dict[f'av_change_rate_roll_mean_{w}'] = np.mean(np.nonzero((np.diff(x_roll_mean) / x_roll_mean[:-1]))[0])\n#             feature_dict[f'abs_max_roll_mean_{w}'] = np.abs(x_roll_mean).max()       \n\n        return feature_dict\n\n    def generate(self):\n        feature_list = []\n        res = Parallel(n_jobs=self.n_jobs,\n                       backend='threading')(delayed(self.get_features)(x, y, s)\n                                            for s, x, y in tqdm_notebook(self.read_chunks(), total=self.total_data))\n        for r in res:\n            feature_list.append(r)\n        return pd.DataFrame(feature_list)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"training_fg = FeatureGenerator(dtype='train', n_jobs=20, chunk_size=150000)\ntraining_data = training_fg.generate()\n\ntest_fg = FeatureGenerator(dtype='test', n_jobs=20, chunk_size=150000)\ntest_data = test_fg.generate()\n\ntrain_X1 = training_data.drop(['target', 'seg_id'], axis=1)\ntest_X1 = test_data.drop(['target', 'seg_id'], axis=1)\ntest_segs = test_data.seg_id\ntrain_y1 = training_data.target","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_X = train_X1.iloc[:-1,1:]\ntrain_X.head(3)\ntrain_X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test_X = test_X1.iloc[:,1:]\nprint('test_X',test_X.shape)\ntest_X.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(train_y1.shape)\nprint(train_y1.head(3))\ntrain_y = train_y1.iloc[:-1]\nprint(train_y.shape)\ntrain_y.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"means_dict = {}\nfor col in train_X.columns:\n    if train_X[col].isnull().any():\n        print(col)\n        mean_value = train_X.loc[train_X[col] != -np.inf, col].mean()\n        train_X.loc[train_X[col] == -np.inf, col] = mean_value\n        train_X[col] = train_X[col].fillna(mean_value)\n        means_dict[col] = mean_value","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"for col in test_X.columns:\n    if test_X[col].isnull().any():\n        test_X.loc[test_X[col] == -np.inf, col] = means_dict[col]\n        test_X[col] = test_X[col].fillna(means_dict[col])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(train_X)\nscaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns)\nscaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"purchased = ['abs_max_roll_std_10000',\n             'ffti_abs_percentile_20',\n 'ffti_count_big_100000_less_threshold_100',\n 'ffti_count_big_100000_threshold_50',\n 'ffti_count_big_150000_less_threshold_50',\n 'ffti_count_big_150000_threshold_50',\n 'ffti_percentile_roll_std_10_window_10',\n 'ffti_percentile_roll_std_25_window_10',\n 'ffti_percentile_roll_std_50_window_1000',\n 'ffti_percentile_roll_std_50_window_500',\n 'ffti_percentile_roll_std_70_window_500',\n 'ffti_percentile_roll_std_80_window_10000',\n 'ffti_percentile_roll_std_80_window_50',\n 'ffti_percentile_roll_std_80_window_500',\n 'ffti_percentile_roll_std_99_window_500' ,\n             'ffti_range_-4000_-3000',\n 'ffti_range_2000_3000' ,\n             'fftr_count_big_100000_threshold_100',\n 'fftr_exp_Moving_std_30000_std', 'fftr_exp_Moving_std_3000_std',\n 'fftr_exp_Moving_std_300_std', 'fftr_exp_Moving_std_50000_std',\n 'fftr_percentile_roll_mean_10_window_10',\n 'fftr_percentile_roll_mean_99_window_10',\n 'fftr_percentile_roll_std_20_window_10',\n 'fftr_percentile_roll_std_60_window_500',\n 'fftr_percentile_roll_std_70_window_10',\n 'fftr_percentile_roll_std_70_window_1000',\n 'fftr_percentile_roll_std_70_window_50',\n 'fftr_percentile_roll_std_75_window_1000',\n 'fftr_percentile_roll_std_75_window_10000',\n 'fftr_percentile_roll_std_75_window_50',\n 'fftr_percentile_roll_std_99_window_100',\n 'fftr_percentile_roll_std_99_window_500' ,'fftr_range_-3000_-2000',\n 'fftr_range_-4000_-3000' ,'fftr_std_roll_std_10',\n 'percentile_roll_mean_99_window_50',\n             'percentile_roll_std_10_window_1000',\n 'percentile_roll_std_10_window_50', 'percentile_roll_std_10_window_500',\n 'percentile_roll_std_1_window_1000', 'percentile_roll_std_20_window_100',\n 'percentile_roll_std_20_window_1000', 'percentile_roll_std_25_window_10',\n 'percentile_roll_std_25_window_100' ,'percentile_roll_std_25_window_50',\n 'percentile_roll_std_25_window_500', 'percentile_roll_std_40_window_500',\n 'percentile_roll_std_50_window_1000' ,'percentile_roll_std_5_window_1000',\n 'percentile_roll_std_5_window_10000', 'percentile_roll_std_5_window_50',\n 'percentile_roll_std_5_window_500' ,'percentile_roll_std_90_window_10000',\n 'percentile_roll_std_95_window_500', 'std_first_50000']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"candidate = ['count_big_100000_less_threshold_10' \n             ,'ffti_abs_percentile_10',\n 'ffti_abs_percentile_5',\n             'ffti_count_big_100000_threshold_100',\n 'ffti_count_big_50000_less_threshold_50',\n 'ffti_percentile_roll_std_40_window_100',\n 'ffti_percentile_roll_std_40_window_50',\n 'ffti_percentile_roll_std_50_window_10',\n 'ffti_percentile_roll_std_50_window_100',\n 'ffti_percentile_roll_std_50_window_50',\n 'ffti_percentile_roll_std_60_window_10',\n 'ffti_percentile_roll_std_60_window_100',\n 'ffti_percentile_roll_std_60_window_50',\n 'ffti_percentile_roll_std_60_window_500',\n 'ffti_percentile_roll_std_70_window_10000',\n 'ffti_percentile_roll_std_75_window_10000' ,\n             'ffti_range_-3000_-2000',\n 'fftr_abs_max_roll_std_1000' ,\n             'fftr_abs_percentile_10',\n 'fftr_percentile_roll_std_25_window_10',\n 'fftr_percentile_roll_std_30_window_10',\n 'fftr_percentile_roll_std_40_window_100',\n 'fftr_percentile_roll_std_40_window_50',\n 'fftr_percentile_roll_std_50_window_100',\n 'fftr_percentile_roll_std_60_window_1000',\n 'fftr_percentile_roll_std_70_window_100',\n 'fftr_percentile_roll_std_80_window_100',\n 'fftr_percentile_roll_std_80_window_500' ,\n             'iqr' ,\n             'min_roll_std_1000',\n 'min_roll_std_10000',\n             'num_peaks_10',\n             'percentile_25',\n 'percentile_roll_std_10_window_10' ,\n             'percentile_roll_std_10_window_100',\n 'percentile_roll_std_1_window_10000' ,'percentile_roll_std_1_window_50',\n 'percentile_roll_std_1_window_500' ,'percentile_roll_std_20_window_500',\n 'percentile_roll_std_25_window_1000' ,'percentile_roll_std_30_window_100',\n 'percentile_roll_std_30_window_1000' ,'percentile_roll_std_30_window_500',\n 'percentile_roll_std_40_window_100' ,'percentile_roll_std_40_window_1000',\n 'percentile_roll_std_50_window_100' ,'percentile_roll_std_5_window_100',\n 'percentile_roll_std_60_window_10000', 'percentile_roll_std_80_window_100',\n            \n            \n            'fftr_percentile_roll_std_70_window_10000' ,'percentile_roll_std_40_window_50',\n 'ffti_percentile_roll_std_25_window_10' ,'ffti_percentile_roll_std_60_window_500' ,\n'ffti_percentile_roll_std_70_window_10000','fftr_range_2000_3000' ,\n'fftr_percentile_roll_std_60_window_50', 'fftr_percentile_roll_std_70_window_10',\n 'ffti_percentile_roll_mean_70_window_10' ,'percentile_roll_std_95_window_1000'  \n            \n            \n            ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_X_purchased = scaled_train_X[purchased]\ntest_X_purchased = scaled_test_X[purchased]\nprint(train_X_purchased.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_X_candidate = scaled_train_X[candidate]\ntest_X_candidate  = scaled_test_X[candidate]\nprint(train_X_candidate.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"columns_purchased = train_X_purchased.columns\ncolumns_candidate = train_X_candidate.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_X_partial = scaled_train_X[columns_purchased]\ntest_X_partial = scaled_test_X[columns_purchased]\nprint(train_X_partial.shape, test_X_partial.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#import required packages\nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as cb\nimport gc\nfrom hyperopt import hp, tpe, Trials, STATUS_OK\nfrom hyperopt.fmin import fmin\nfrom hyperopt.pyll.stochastic import sample\n#optional but advised\nimport warnings\nwarnings.filterwarnings('ignore')\n\n#GLOBAL HYPEROPT PARAMETERS\nNUM_EVALS = 1000 #number of hyperopt evaluation rounds\nN_FOLDS = 5 #number of cross-validation folds on data in each evaluation round\n\n#LIGHTGBM PARAMETERS\nLGBM_MAX_LEAVES = 2**10 #maximum number of leaves per tree for LightGBM\nLGBM_MAX_DEPTH = 25 #maximum tree depth for LightGBM\nEVAL_METRIC_LGBM_REG = 'mae' #LightGBM regression metric. Note that 'rmse' is more commonly used \nEVAL_METRIC_LGBM_CLASS = 'auc'#LightGBM classification metric\n\n#XGBOOST PARAMETERS\nXGB_MAX_LEAVES = 2**12 #maximum number of leaves when using histogram splitting\nXGB_MAX_DEPTH = 25 #maximum tree depth for XGBoost\nEVAL_METRIC_XGB_REG = 'mae' #XGBoost regression metric\nEVAL_METRIC_XGB_CLASS = 'auc' #XGBoost classification metric\n\n#CATBOOST PARAMETERS\nCB_MAX_DEPTH = 8 #maximum tree depth in CatBoost\nOBJECTIVE_CB_REG = 'MAE' #CatBoost regression metric\nOBJECTIVE_CB_CLASS = 'Logloss' #CatBoost classification metric\n\n#OPTIONAL OUTPUT\nBEST_SCORE = 0\n\n# if classification problem is solved; Class = True;\n# data is the features; each row is a sample;\n# label is the y value for each sample;\ndef quick_hyperopt(data, labels, package='lgbm', num_evals=NUM_EVALS, diagnostic=False, Class=False):\n    \n    #==========\n    #LightGBM\n    #==========\n    \n    if package=='lgbm':\n        \n        print('Running {} rounds of LightGBM parameter optimisation:'.format(num_evals))\n        #clear space\n        gc.collect()\n        \n        integer_params = ['max_depth',\n                         'num_leaves',\n                          'max_bin',\n                         'min_data_in_leaf',\n                         'min_data_in_bin']\n        \n        def objective(space_params):\n            \n            #cast integer params from float to int\n            for param in integer_params:\n                space_params[param] = int(space_params[param])\n            \n            #extract nested conditional parameters\n            if space_params['boosting']['boosting'] == 'goss':\n                top_rate = space_params['boosting'].get('top_rate')\n                other_rate = space_params['boosting'].get('other_rate')\n                #0 <= top_rate + other_rate <= 1\n                top_rate = max(top_rate, 0)\n                top_rate = min(top_rate, 0.5)\n                other_rate = max(other_rate, 0)\n                other_rate = min(other_rate, 0.5)\n                space_params['top_rate'] = top_rate\n                space_params['other_rate'] = other_rate\n            \n            subsample = space_params['boosting'].get('subsample', 1.0)\n            space_params['boosting'] = space_params['boosting']['boosting']\n            space_params['subsample'] = subsample\n            \n            if Class:\n                cv_results = lgb.cv(space_params, train, nfold = N_FOLDS, stratified=True,\n                                    early_stopping_rounds=100, metrics=EVAL_METRIC_LGBM_CLASS, seed=42)\n                best_loss = 1 - cv_results['auc-mean'][-1]\n                \n            else:\n                cv_results = lgb.cv(space_params, train, nfold = N_FOLDS, stratified=False,\n                                    early_stopping_rounds=100, metrics=EVAL_METRIC_LGBM_REG, seed=42)\n                best_loss = cv_results['l1-mean'][-1] #'l2-mean' for rmse\n            \n            return{'loss':best_loss, 'status': STATUS_OK }\n        \n        train = lgb.Dataset(data, labels)\n                \n        #integer and string parameters, used with hp.choice()\n        boosting_list = [{'boosting': 'gbdt',\n                          'subsample': hp.uniform('subsample', 0.5, 1)},\n                         {'boosting': 'goss',\n                          'subsample': 1.0,\n                         'top_rate': hp.uniform('top_rate', 0, 0.5),\n                         'other_rate': hp.uniform('other_rate', 0, 0.5)}] #if including 'dart', make sure to set 'n_estimators'\n        \n        if Class:\n            metric_list = ['auc'] #modify as required for other classification metrics\n            objective_list = ['binary', 'cross_entropy']\n        \n        else:\n            metric_list = ['MAE', 'RMSE'] \n            objective_list = ['huber', 'gamma', 'fair', 'tweedie']\n        \n        \n        space ={'boosting' : hp.choice('boosting', boosting_list),\n                'num_leaves' : hp.quniform('num_leaves', 2, LGBM_MAX_LEAVES, 1),\n                'max_depth': hp.quniform('max_depth', 2, LGBM_MAX_DEPTH, 1),\n                'max_bin': hp.quniform('max_bin', 32, 255, 1),\n                'min_data_in_leaf': hp.quniform('min_data_in_leaf', 1, 256, 1),\n                'min_data_in_bin': hp.quniform('min_data_in_bin', 1, 256, 1),\n                'min_gain_to_split' : hp.quniform('min_gain_to_split', 0.1, 5, 0.01),\n                'lambda_l1' : hp.uniform('lambda_l1', 0, 5),\n                'lambda_l2' : hp.uniform('lambda_l2', 0, 5),\n                'learning_rate' : hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)),\n                'metric' : hp.choice('metric', metric_list),\n                'objective' : hp.choice('objective', objective_list),\n                'feature_fraction' : hp.quniform('feature_fraction', 0.5, 1, 0.01),\n                'bagging_fraction' : hp.quniform('bagging_fraction', 0.5, 1, 0.01)\n            }\n        \n        #optional: activate GPU for LightGBM\n        #follow compilation steps here:\n        #https://www.kaggle.com/vinhnguyen/gpu-acceleration-for-lightgbm/\n        #then uncomment lines below:\n        #space['device'] = 'gpu'\n        #space['gpu_platform_id'] = 0,\n        #space['gpu_device_id'] =  0\n\n        trials = Trials()\n        best = fmin(fn=objective,\n                    space=space,\n                    algo=tpe.suggest,\n                    max_evals=num_evals, \n                    trials=trials)\n                \n        #fmin() will return the index of values chosen from the lists/arrays in 'space'\n        #to obtain actual values, index values are used to subset the original lists/arrays\n        best['boosting'] = boosting_list[best['boosting']]['boosting']#nested dict, index twice\n        best['metric'] = metric_list[best['metric']]\n        best['objective'] = objective_list[best['objective']]\n                \n        #cast floats of integer params to int\n        for param in integer_params:\n            best[param] = int(best[param])\n        \n        print('{' + '\\n'.join('{}: {}'.format(k, v) for k, v in best.items()) + '}')\n        if diagnostic:\n            return(best, trials)\n        else:\n            return(best)\n    \n    #==========\n    #XGBoost\n    #==========\n    \n    if package=='xgb':\n        \n        print('Running {} rounds of XGBoost parameter optimisation:'.format(num_evals))\n        #clear space\n        gc.collect()\n        \n        integer_params = ['max_depth']\n        \n        def objective(space_params):\n            \n            for param in integer_params:\n                space_params[param] = int(space_params[param])\n                \n            #extract multiple nested tree_method conditional parameters\n            #libera te tutemet ex inferis\n            if space_params['tree_method']['tree_method'] == 'hist':\n                max_bin = space_params['tree_method'].get('max_bin')\n                space_params['max_bin'] = int(max_bin)\n                if space_params['tree_method']['grow_policy']['grow_policy']['grow_policy'] == 'depthwise':\n                    grow_policy = space_params['tree_method'].get('grow_policy').get('grow_policy').get('grow_policy')\n                    space_params['grow_policy'] = grow_policy\n                    space_params['tree_method'] = 'hist'\n                else:\n                    max_leaves = space_params['tree_method']['grow_policy']['grow_policy'].get('max_leaves')\n                    space_params['grow_policy'] = 'lossguide'\n                    space_params['max_leaves'] = int(max_leaves)\n                    space_params['tree_method'] = 'hist'\n            else:\n                space_params['tree_method'] = space_params['tree_method'].get('tree_method')\n                \n            #for classification replace EVAL_METRIC_XGB_REG with EVAL_METRIC_XGB_CLASS\n            cv_results = xgb.cv(space_params, train, nfold=N_FOLDS, metrics=[EVAL_METRIC_XGB_REG],\n                             early_stopping_rounds=100, stratified=False, seed=42)\n            \n            best_loss = cv_results['test-mae-mean'].iloc[-1] #or 'test-rmse-mean' if using RMSE\n            #for classification, comment out the line above and uncomment the line below:\n            #best_loss = 1 - cv_results['test-auc-mean'].iloc[-1]\n            #if necessary, replace 'test-auc-mean' with 'test-[your-preferred-metric]-mean'\n            return{'loss':best_loss, 'status': STATUS_OK }\n        \n        train = xgb.DMatrix(data, labels)\n        \n        #integer and string parameters, used with hp.choice()\n        boosting_list = ['gbtree', 'gblinear'] #if including 'dart', make sure to set 'n_estimators'\n        metric_list = ['MAE', 'RMSE'] \n        #for classification comment out the line above and uncomment the line below\n        #metric_list = ['auc']\n        #modify as required for other classification metrics classification\n        \n        tree_method = [{'tree_method' : 'exact'},\n               {'tree_method' : 'approx'},\n               {'tree_method' : 'hist',\n                'max_bin': hp.quniform('max_bin', 2**3, 2**7, 1),\n                'grow_policy' : {'grow_policy': {'grow_policy':'depthwise'},\n                                'grow_policy' : {'grow_policy':'lossguide',\n                                                  'max_leaves': hp.quniform('max_leaves', 32, XGB_MAX_LEAVES, 1)}}}]\n        \n        #if using GPU, replace 'exact' with 'gpu_exact' and 'hist' with\n        #'gpu_hist' in the nested dictionary above\n        \n        objective_list_reg = ['reg:linear', 'reg:gamma', 'reg:tweedie']\n        objective_list_class = ['reg:logistic', 'binary:logistic']\n        #for classification change line below to 'objective_list = objective_list_class'\n        objective_list = objective_list_reg\n        \n        space ={'boosting' : hp.choice('boosting', boosting_list),\n                'tree_method' : hp.choice('tree_method', tree_method),\n                'max_depth': hp.quniform('max_depth', 2, XGB_MAX_DEPTH, 1),\n                'reg_alpha' : hp.uniform('reg_alpha', 0, 5),\n                'reg_lambda' : hp.uniform('reg_lambda', 0, 5),\n                'min_child_weight' : hp.uniform('min_child_weight', 0, 5),\n                'gamma' : hp.uniform('gamma', 0, 5),\n                'learning_rate' : hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)),\n                'eval_metric' : hp.choice('eval_metric', metric_list),\n                'objective' : hp.choice('objective', objective_list),\n                'colsample_bytree' : hp.quniform('colsample_bytree', 0.1, 1, 0.01),\n                'colsample_bynode' : hp.quniform('colsample_bynode', 0.1, 1, 0.01),\n                'colsample_bylevel' : hp.quniform('colsample_bylevel', 0.1, 1, 0.01),\n                'subsample' : hp.quniform('subsample', 0.5, 1, 0.05),\n                'nthread' : -1\n            }\n        \n        trials = Trials()\n        best = fmin(fn=objective,\n                    space=space,\n                    algo=tpe.suggest,\n                    max_evals=num_evals, \n                    trials=trials)\n        \n        best['tree_method'] = tree_method[best['tree_method']]['tree_method']\n        best['boosting'] = boosting_list[best['boosting']]\n        best['eval_metric'] = metric_list[best['eval_metric']]\n        best['objective'] = objective_list[best['objective']]\n        \n        #cast floats of integer params to int\n        for param in integer_params:\n            best[param] = int(best[param])\n        if 'max_leaves' in best:\n            best['max_leaves'] = int(best['max_leaves'])\n        if 'max_bin' in best:\n            best['max_bin'] = int(best['max_bin'])\n        \n        print('{' + '\\n'.join('{}: {}'.format(k, v) for k, v in best.items()) + '}')\n        \n        if diagnostic:\n            return(best, trials)\n        else:\n            return(best)\n    \n    #==========\n    #CatBoost\n    #==========\n    \n    if package=='cb':\n        \n        print('Running {} rounds of CatBoost parameter optimisation:'.format(num_evals))\n        \n        #clear memory \n        gc.collect()\n            \n        integer_params = ['depth',\n                          #'one_hot_max_size', #for categorical data\n                          'min_data_in_leaf',\n                          'max_bin']\n        \n        def objective(space_params):\n                        \n            #cast integer params from float to int\n            for param in integer_params:\n                space_params[param] = int(space_params[param])\n                \n            #extract nested conditional parameters\n            if space_params['bootstrap_type']['bootstrap_type'] == 'Bayesian':\n                bagging_temp = space_params['bootstrap_type'].get('bagging_temperature')\n                space_params['bagging_temperature'] = bagging_temp\n                \n            if space_params['grow_policy']['grow_policy'] == 'LossGuide':\n                max_leaves = space_params['grow_policy'].get('max_leaves')\n                space_params['max_leaves'] = int(max_leaves)\n                \n            space_params['bootstrap_type'] = space_params['bootstrap_type']['bootstrap_type']\n            space_params['grow_policy'] = space_params['grow_policy']['grow_policy']\n                           \n            #random_strength cannot be < 0\n            space_params['random_strength'] = max(space_params['random_strength'], 0)\n            #fold_len_multiplier cannot be < 1\n            space_params['fold_len_multiplier'] = max(space_params['fold_len_multiplier'], 1)\n                       \n            #for classification set stratified=True\n            cv_results = cb.cv(train, space_params, fold_count=N_FOLDS, \n                             early_stopping_rounds=25, stratified=False, partition_random_seed=42)\n           \n            best_loss = cv_results['test-MAE-mean'].iloc[-1] #'test-RMSE-mean' for RMSE\n            #for classification, comment out the line above and uncomment the line below:\n            #best_loss = cv_results['test-Logloss-mean'].iloc[-1]\n            #if necessary, replace 'test-Logloss-mean' with 'test-[your-preferred-metric]-mean'\n            \n            return{'loss':best_loss, 'status': STATUS_OK}\n        \n        train = cb.Pool(data, labels.astype('float32'))\n        \n        #integer and string parameters, used with hp.choice()\n        bootstrap_type = [{'bootstrap_type':'Poisson'}, \n                           {'bootstrap_type':'Bayesian',\n                            'bagging_temperature' : hp.loguniform('bagging_temperature', np.log(1), np.log(50))},\n                          {'bootstrap_type':'Bernoulli'}] \n        #LEB = ['No', 'AnyImprovement', 'Armijo'] #remove 'Armijo' if not using GPU\n        LEB = ['No', 'AnyImprovement'] #remove 'Armijo' if not using GPU\n        #score_function = ['Correlation', 'L2', 'NewtonCorrelation', 'NewtonL2']\n        grow_policy = [{'grow_policy':'SymmetricTree'},\n                       {'grow_policy':'Depthwise'},\n                       {'grow_policy':'Lossguide',\n                        'max_leaves': hp.quniform('max_leaves', 2, 32, 1)}]\n        eval_metric_list_reg = ['MAE', 'RMSE', 'Poisson']\n        eval_metric_list_class = ['Logloss', 'AUC', 'F1']\n        #for classification change line below to 'eval_metric_list = eval_metric_list_class'\n        eval_metric_list = eval_metric_list_reg\n                \n        space ={'depth': hp.quniform('depth', 2, CB_MAX_DEPTH, 1),\n                'max_bin' : hp.quniform('max_bin', 1, 32, 1), #if using CPU just set this to 254\n                'l2_leaf_reg' : hp.uniform('l2_leaf_reg', 0, 5),\n                'min_data_in_leaf' : hp.quniform('min_data_in_leaf', 1, 50, 1),\n                'random_strength' : hp.loguniform('random_strength', np.log(0.005), np.log(5)),\n                #'one_hot_max_size' : hp.quniform('one_hot_max_size', 2, 16, 1), #uncomment if using categorical features\n                'bootstrap_type' : hp.choice('bootstrap_type', bootstrap_type),\n                'learning_rate' : hp.uniform('learning_rate', 0.05, 0.25),\n                'eval_metric' : hp.choice('eval_metric', eval_metric_list),\n                'objective' : OBJECTIVE_CB_REG,\n                #'score_function' : hp.choice('score_function', score_function), #crashes kernel - reason unknown\n                'leaf_estimation_backtracking' : hp.choice('leaf_estimation_backtracking', LEB),\n                'grow_policy': hp.choice('grow_policy', grow_policy),\n                #'colsample_bylevel' : hp.quniform('colsample_bylevel', 0.1, 1, 0.01),# CPU only\n                'fold_len_multiplier' : hp.loguniform('fold_len_multiplier', np.log(1.01), np.log(2.5)),\n                'od_type' : 'Iter',\n                'od_wait' : 25,\n                'task_type' : 'GPU',\n                'verbose' : 0\n            }\n        \n        #optional: run CatBoost without GPU\n        #uncomment line below\n        space['task_type'] = 'CPU'\n            \n        trials = Trials()\n        best = fmin(fn=objective,\n                    space=space,\n                    algo=tpe.suggest,\n                    max_evals=num_evals, \n                    trials=trials)\n        \n        #unpack nested dicts first\n        best['bootstrap_type'] = bootstrap_type[best['bootstrap_type']]['bootstrap_type']\n        best['grow_policy'] = grow_policy[best['grow_policy']]['grow_policy']\n        best['eval_metric'] = eval_metric_list[best['eval_metric']]\n        \n        #best['score_function'] = score_function[best['score_function']] \n        #best['leaf_estimation_method'] = LEM[best['leaf_estimation_method']] #CPU only\n        best['leaf_estimation_backtracking'] = LEB[best['leaf_estimation_backtracking']]        \n        \n        #cast floats of integer params to int\n        for param in integer_params:\n            best[param] = int(best[param])\n        if 'max_leaves' in best:\n            best['max_leaves'] = int(best['max_leaves'])\n        \n        print('{' + '\\n'.join('{}: {}'.format(k, v) for k, v in best.items()) + '}')\n        \n        if diagnostic:\n            return(best, trials)\n        else:\n            return(best)\n    \n    else:\n        print('Package not recognised. Please use \"lgbm\" for LightGBM, \"xgb\" for XGBoost or \"cb\" for CatBoost.')     ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"params = quick_hyperopt(train_X_partial, train_y, 'lgbm', 200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"predictions1 = np.zeros(len(test_X_partial))\nfeature_importance_df = pd.DataFrame()\n#run model\n\nN_aug = 30\n\nn_fold = 5\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=42)\n\nfor i_aug in tqdm_notebook(range(0, N_aug)):\n    a = np.arange(0, train_X_partial.shape[1])\n    #initialise aug dataframe - remember to set dtype!\n    train_aug = pd.DataFrame(index = train_X_partial.index, columns=train_X_partial.columns, dtype='float64')\n\n    # Please note that pandas will set the datatype of its columns as 'object' unless you specify otherwise. \n    # I mention this because the above code, which takes less than 1 minute to process 4194 rows of 100 features,\n    # will take around an hour if dtype isn't set to 'float64'!\n\n    for i in tqdm_notebook(range(0, len(train_X_partial))):\n        # ratio of features to be randomly sampled\n        AUG_FEATURE_RATIO = 0.5\n        # to integer count\n        AUG_FEATURE_COUNT = np.floor(train_X_partial.shape[1]*AUG_FEATURE_RATIO).astype('int16')\n\n        # randomly sample half of columns (features) that will contain random values\n        # indices for features which will be sampled from the same feature\n        aug_feature_index = np.random.choice(train_X_partial.shape[1], AUG_FEATURE_COUNT, replace=False)\n        aug_feature_index.sort()\n\n        # obtain indices for features not in aug_feature_index;\n        # i.e., the indices for features that will be kept for all rows\n        feature_index = np.where(np.logical_not(np.in1d(a, aug_feature_index)))[0]\n\n        # first insert features with real values; i.e., feature_index records the incices for features \n        # that are kept\n        train_aug.iloc[i, feature_index] = train_X_partial.iloc[i, feature_index]\n\n        # random row index to randomly sampled values for each features\n        rand_row_index = np.random.choice(len(train_X_partial), len(aug_feature_index), replace=True)\n\n        # loop over all rows of augmented set; for each row and each sampled feature, \n        # randomly choose a value from other row to fill in this sampled feature\n        for n, j in enumerate(aug_feature_index):\n            train_aug.iloc[i, j] = train_X_partial.iloc[rand_row_index[n], j]\n\n    \n    train_all = pd.concat([train_X_partial, train_aug])\n    y_all = np.append(train_y, train_y)\n    \n    # params = quick_hyperopt(train_all, y_all, 'lgbm', 200)\n    \n    for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_all,y_all)):\n        strLog = \"fold {}\".format(fold_)\n        print(strLog)\n\n        X_tr, X_val = train_all.iloc[trn_idx], train_all.iloc[val_idx]\n        y_tr, y_val = y_all[trn_idx], y_all[val_idx]\n\n        model = lgb.LGBMRegressor(**params, n_estimators = 20000, n_jobs = -1)\n        model.fit(X_tr, \n                  y_tr, \n                  eval_set=[(X_tr, y_tr), (X_val, y_val)], \n                  eval_metric='mae',\n                  verbose=1000, \n                  early_stopping_rounds=500)\n\n        predictions1 += model.predict(test_X_partial, num_iteration=model.best_iteration_) / (folds.n_splits*N_aug)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof = np.zeros(len(train_X_partial))\ntrain_score = []\n\npredictions4 = np.zeros(len(scaled_test_X))\n\nn_fold = 5\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=42)\n\nfeature_importance_df = pd.DataFrame()\n#run model\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_X_partial,train_y)):\n    strLog = \"fold {}\".format(fold_)\n    print(strLog)\n    X_tr, X_val = train_X_partial.iloc[trn_idx], train_X_partial.iloc[val_idx]\n    y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]\n\n    model = CatBoostRegressor(n_estimators=25000, verbose=-1, objective=\"MAE\", loss_function=\"MAE\", boosting_type=\"Ordered\", task_type=\"GPU\")\n    model.fit(X_tr, \n              y_tr, \n              eval_set=[(X_val, y_val)], \n#               eval_metric='mae',\n              verbose=2500, \n              early_stopping_rounds=500)\n    oof[val_idx] = model.predict(X_val)\n\n    train_score.append(model.best_score_['learn'][\"MAE\"])\n    predictions4 += model.predict(test_X_partial) / (folds.n_splits)\ncv_score = mean_absolute_error(train_y, oof)\nprint('cv_score',cv_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"combine = np.concatenate([columns_candidate,columns_purchased])\n\ntrain_X_partial_cat = scaled_train_X[combine]\ntest_X_partial_cat = scaled_test_X[combine]\nprint(train_X_partial_cat.shape, test_X_partial_cat.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"oof = np.zeros(len(train_X_partial_cat))\ntrain_score = []\n\npredictions2 = np.zeros(len(scaled_test_X))\n\nn_fold = 5\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=42)\n\nfeature_importance_df = pd.DataFrame()\n#run model\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_X_partial_cat,train_y)):\n    strLog = \"fold {}\".format(fold_)\n    print(strLog)\n    X_tr, X_val = train_X_partial_cat.iloc[trn_idx], train_X_partial_cat.iloc[val_idx]\n    y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]\n\n    model = CatBoostRegressor(n_estimators=25000, verbose=-1, objective=\"MAE\", loss_function=\"MAE\", boosting_type=\"Ordered\", task_type=\"GPU\")\n    model.fit(X_tr, \n              y_tr, \n              eval_set=[(X_val, y_val)], \n#               eval_metric='mae',\n              verbose=2500, \n              early_stopping_rounds=500)\n    oof[val_idx] = model.predict(X_val)\n\n    train_score.append(model.best_score_['learn'][\"MAE\"])\n    predictions2 += model.predict(test_X_partial_cat) / (folds.n_splits)\ncv_score = mean_absolute_error(train_y, oof)\nprint('cv_score',cv_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\nsubmission.time_to_failure = 0.2*predictions1+predictions2*0.6+predictions4*0.2\nsubmission.to_csv('submission.csv',index=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}