{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport time\nimport warnings\nimport traceback\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nimport scipy.signal as sg\nimport multiprocessing as mp\nfrom scipy.signal import hann\nfrom scipy.signal import hilbert\nfrom scipy.signal import convolve\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.preprocessing import StandardScaler\n\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\n\nfrom tqdm import tqdm\nwarnings.filterwarnings(\"ignore\")","execution_count":1,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DIR = r'd:\\#earthquake\\final_model' # set for local environment\nDATA_DIR = r'd:\\#earthquake\\data' # set for local environment\n\nSIG_LEN = 150000\nNUM_SEG_PER_PROC = 4000\nNUM_THREADS = 6\n\nNY_FREQ_IDX = 75000 # the test signals are 150k samples long, Nyquist is thus 75k.\nCUTOFF = 18000\nMAX_FREQ_IDX = 20000\nFREQ_STEP = 2500","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_raw_data():\n df = pd.read_csv(os.path.join(DATA_DIR, 'train.csv'))\n\n max_start_index = len(df.index) - SIG_LEN\n slice_len = int(max_start_index / 6)\n\n for i in range(NUM_THREADS):\n print('working', i)\n df0 = df.iloc[slice_len * i: (slice_len * (i + 1)) + SIG_LEN]\n df0.to_csv(os.path.join(DATA_DIR, 'raw_data_%d.csv' % i), index=False)\n del df0\n\n del df","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_rnd_idxs():\n rnd_idxs = np.zeros(shape=(NUM_THREADS, NUM_SEG_PER_PROC), dtype=np.int32)\n max_start_idx = 100000000\n\n for i in range(NUM_THREADS):\n np.random.seed(5591 + i)\n start_indices = np.random.randint(0, max_start_idx, size=NUM_SEG_PER_PROC, dtype=np.int32)\n rnd_idxs[i, :] = start_indices\n\n for i in range(NUM_THREADS):\n print(rnd_idxs[i, :8])\n print(rnd_idxs[i, -8:])\n print(min(rnd_idxs[i,:]), max(rnd_idxs[i,:]))\n\n np.savetxt(fname=os.path.join(OUTPUT_DIR, 'start_indices_4k.csv'), X=np.transpose(rnd_idxs), fmt='%d', delimiter=',')","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_trend_feature(arr, abs_values=False):\n idx = np.array(range(len(arr)))\n if abs_values:\n arr = np.abs(arr)\n lr = LinearRegression()\n lr.fit(idx.reshape(-1, 1), arr)\n return lr.coef_[0]\n\ndef classic_sta_lta(x, length_sta, length_lta):\n sta = np.cumsum(x ** 2)\n # Convert to float\n sta = np.require(sta, dtype=np.float)\n # Copy for LTA\n lta = sta.copy()\n # Compute the STA and the LTA\n sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]\n sta /= length_sta\n lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]\n lta /= length_lta\n # Pad zeros\n sta[:length_lta - 1] = 0\n # Avoid division by zero by setting zero values to tiny float\n dtiny = np.finfo(0.0).tiny\n idx = lta < dtiny\n lta[idx] = dtiny\n return sta / lta","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def des_bw_filter_lp(cutoff=CUTOFF): # low pass filter\n b, a = sg.butter(4, Wn=cutoff/NY_FREQ_IDX)\n return b, a\n\ndef des_bw_filter_hp(cutoff=CUTOFF): # high pass filter\n b, a = sg.butter(4, Wn=cutoff/NY_FREQ_IDX, btype='highpass')\n return b, a\n\ndef des_bw_filter_bp(low, high): # band pass filter\n b, a = sg.butter(4, Wn=(low/NY_FREQ_IDX, high/NY_FREQ_IDX), btype='bandpass')\n return b, a","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_features(seg_id, seg, X, st, end):\n try:\n X.loc[seg_id, 'seg_id'] = np.int32(seg_id)\n X.loc[seg_id, 'seg_start'] = np.int32(st)\n X.loc[seg_id, 'seg_end'] = np.int32(end)\n except:\n pass\n\n xc = pd.Series(seg['acoustic_data'].values)\n xcdm = xc - np.mean(xc)\n\n b, a = des_bw_filter_lp(cutoff=18000)\n xcz = sg.lfilter(b, a, xcdm)\n\n zc = np.fft.fft(xcz)\n zc = zc[:MAX_FREQ_IDX]\n\n # FFT transform values\n realFFT = np.real(zc)\n imagFFT = np.imag(zc)\n\n freq_bands = [x for x in range(0, MAX_FREQ_IDX, FREQ_STEP)]\n magFFT = np.sqrt(realFFT ** 2 + imagFFT ** 2)\n phzFFT = np.arctan(imagFFT / realFFT)\n phzFFT[phzFFT == -np.inf] = -np.pi / 2.0\n phzFFT[phzFFT == np.inf] = np.pi / 2.0\n phzFFT = np.nan_to_num(phzFFT)\n\n for freq in freq_bands:\n X.loc[seg_id, 'FFT_Mag_01q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.01)\n X.loc[seg_id, 'FFT_Mag_10q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.1)\n X.loc[seg_id, 'FFT_Mag_90q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.9)\n X.loc[seg_id, 'FFT_Mag_99q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.99)\n X.loc[seg_id, 'FFT_Mag_mean%d' % freq] = np.mean(magFFT[freq: freq + FREQ_STEP])\n X.loc[seg_id, 'FFT_Mag_std%d' % freq] = np.std(magFFT[freq: freq + FREQ_STEP])\n X.loc[seg_id, 'FFT_Mag_max%d' % freq] = np.max(magFFT[freq: freq + FREQ_STEP])\n\n X.loc[seg_id, 'FFT_Phz_mean%d' % freq] = np.mean(phzFFT[freq: freq + FREQ_STEP])\n X.loc[seg_id, 'FFT_Phz_std%d' % freq] = np.std(phzFFT[freq: freq + FREQ_STEP])\n\n X.loc[seg_id, 'FFT_Rmean'] = realFFT.mean()\n X.loc[seg_id, 'FFT_Rstd'] = realFFT.std()\n X.loc[seg_id, 'FFT_Rmax'] = realFFT.max()\n X.loc[seg_id, 'FFT_Rmin'] = realFFT.min()\n X.loc[seg_id, 'FFT_Imean'] = imagFFT.mean()\n X.loc[seg_id, 'FFT_Istd'] = imagFFT.std()\n X.loc[seg_id, 'FFT_Imax'] = imagFFT.max()\n X.loc[seg_id, 'FFT_Imin'] = imagFFT.min()\n\n X.loc[seg_id, 'FFT_Rmean_first_6000'] = realFFT[:6000].mean()\n X.loc[seg_id, 'FFT_Rstd__first_6000'] = realFFT[:6000].std()\n X.loc[seg_id, 'FFT_Rmax_first_6000'] = realFFT[:6000].max()\n X.loc[seg_id, 'FFT_Rmin_first_6000'] = realFFT[:6000].min()\n X.loc[seg_id, 'FFT_Rmean_first_18000'] = realFFT[:18000].mean()\n X.loc[seg_id, 'FFT_Rstd_first_18000'] = realFFT[:18000].std()\n X.loc[seg_id, 'FFT_Rmax_first_18000'] = realFFT[:18000].max()\n X.loc[seg_id, 'FFT_Rmin_first_18000'] = realFFT[:18000].min()\n\n del xcz\n del zc\n\n b, a = des_bw_filter_lp(cutoff=2500)\n xc0 = sg.lfilter(b, a, xcdm)\n\n b, a = des_bw_filter_bp(low=2500, high=5000)\n xc1 = sg.lfilter(b, a, xcdm)\n\n b, a = des_bw_filter_bp(low=5000, high=7500)\n xc2 = sg.lfilter(b, a, xcdm)\n\n b, a = des_bw_filter_bp(low=7500, high=10000)\n xc3 = sg.lfilter(b, a, xcdm)\n\n b, a = des_bw_filter_bp(low=10000, high=12500)\n xc4 = sg.lfilter(b, a, xcdm)\n\n b, a = des_bw_filter_bp(low=12500, high=15000)\n xc5 = sg.lfilter(b, a, xcdm)\n\n b, a = des_bw_filter_bp(low=15000, high=17500)\n xc6 = sg.lfilter(b, a, xcdm)\n\n b, a = des_bw_filter_bp(low=17500, high=20000)\n xc7 = sg.lfilter(b, a, xcdm)\n\n b, a = des_bw_filter_hp(cutoff=20000)\n xc8 = sg.lfilter(b, a, xcdm)\n\n sigs = [xc, pd.Series(xc0), pd.Series(xc1), pd.Series(xc2), pd.Series(xc3),\n pd.Series(xc4), pd.Series(xc5), pd.Series(xc6), pd.Series(xc7), pd.Series(xc8)]\n\n for i, sig in enumerate(sigs):\n X.loc[seg_id, 'mean_%d' % i] = sig.mean()\n X.loc[seg_id, 'std_%d' % i] = sig.std()\n X.loc[seg_id, 'max_%d' % i] = sig.max()\n X.loc[seg_id, 'min_%d' % i] = sig.min()\n\n X.loc[seg_id, 'mean_change_abs_%d' % i] = np.mean(np.diff(sig))\n X.loc[seg_id, 'mean_change_rate_%d' % i] = np.mean(np.nonzero((np.diff(sig) / sig[:-1]))[0])\n X.loc[seg_id, 'abs_max_%d' % i] = np.abs(sig).max()\n X.loc[seg_id, 'abs_min_%d' % i] = np.abs(sig).min()\n\n X.loc[seg_id, 'std_first_50000_%d' % i] = sig[:50000].std()\n X.loc[seg_id, 'std_last_50000_%d' % i] = sig[-50000:].std()\n X.loc[seg_id, 'std_first_10000_%d' % i] = sig[:10000].std()\n X.loc[seg_id, 'std_last_10000_%d' % i] = sig[-10000:].std()\n\n X.loc[seg_id, 'avg_first_50000_%d' % i] = sig[:50000].mean()\n X.loc[seg_id, 'avg_last_50000_%d' % i] = sig[-50000:].mean()\n X.loc[seg_id, 'avg_first_10000_%d' % i] = sig[:10000].mean()\n X.loc[seg_id, 'avg_last_10000_%d' % i] = sig[-10000:].mean()\n\n X.loc[seg_id, 'min_first_50000_%d' % i] = sig[:50000].min()\n X.loc[seg_id, 'min_last_50000_%d' % i] = sig[-50000:].min()\n X.loc[seg_id, 'min_first_10000_%d' % i] = sig[:10000].min()\n X.loc[seg_id, 'min_last_10000_%d' % i] = sig[-10000:].min()\n\n X.loc[seg_id, 'max_first_50000_%d' % i] = sig[:50000].max()\n X.loc[seg_id, 'max_last_50000_%d' % i] = sig[-50000:].max()\n X.loc[seg_id, 'max_first_10000_%d' % i] = sig[:10000].max()\n X.loc[seg_id, 'max_last_10000_%d' % i] = sig[-10000:].max()\n\n X.loc[seg_id, 'max_to_min_%d' % i] = sig.max() / np.abs(sig.min())\n X.loc[seg_id, 'max_to_min_diff_%d' % i] = sig.max() - np.abs(sig.min())\n X.loc[seg_id, 'count_big_%d' % i] = len(sig[np.abs(sig) > 500])\n X.loc[seg_id, 'sum_%d' % i] = sig.sum()\n\n X.loc[seg_id, 'mean_change_rate_first_50000_%d' % i] = np.mean(np.nonzero((np.diff(sig[:50000]) / sig[:50000][:-1]))[0])\n X.loc[seg_id, 'mean_change_rate_last_50000_%d' % i] = np.mean(np.nonzero((np.diff(sig[-50000:]) / sig[-50000:][:-1]))[0])\n X.loc[seg_id, 'mean_change_rate_first_10000_%d' % i] = np.mean(np.nonzero((np.diff(sig[:10000]) / sig[:10000][:-1]))[0])\n X.loc[seg_id, 'mean_change_rate_last_10000_%d' % i] = np.mean(np.nonzero((np.diff(sig[-10000:]) / sig[-10000:][:-1]))[0])\n\n X.loc[seg_id, 'q95_%d' % i] = np.quantile(sig, 0.95)\n X.loc[seg_id, 'q99_%d' % i] = np.quantile(sig, 0.99)\n X.loc[seg_id, 'q05_%d' % i] = np.quantile(sig, 0.05)\n X.loc[seg_id, 'q01_%d' % i] = np.quantile(sig, 0.01)\n\n X.loc[seg_id, 'abs_q95_%d' % i] = np.quantile(np.abs(sig), 0.95)\n X.loc[seg_id, 'abs_q99_%d' % i] = np.quantile(np.abs(sig), 0.99)\n X.loc[seg_id, 'abs_q05_%d' % i] = np.quantile(np.abs(sig), 0.05)\n X.loc[seg_id, 'abs_q01_%d' % i] = np.quantile(np.abs(sig), 0.01)\n\n X.loc[seg_id, 'trend_%d' % i] = add_trend_feature(sig)\n X.loc[seg_id, 'abs_trend_%d' % i] = add_trend_feature(sig, abs_values=True)\n X.loc[seg_id, 'abs_mean_%d' % i] = np.abs(sig).mean()\n X.loc[seg_id, 'abs_std_%d' % i] = np.abs(sig).std()\n\n X.loc[seg_id, 'mad_%d' % i] = sig.mad()\n X.loc[seg_id, 'kurt_%d' % i] = sig.kurtosis()\n X.loc[seg_id, 'skew_%d' % i] = sig.skew()\n X.loc[seg_id, 'med_%d' % i] = sig.median()\n\n X.loc[seg_id, 'Hilbert_mean_%d' % i] = np.abs(hilbert(sig)).mean()\n X.loc[seg_id, 'Hann_window_mean'] = (convolve(xc, hann(150), mode='same') / sum(hann(150))).mean()\n\n X.loc[seg_id, 'classic_sta_lta1_mean_%d' % i] = classic_sta_lta(sig, 500, 10000).mean()\n X.loc[seg_id, 'classic_sta_lta2_mean_%d' % i] = classic_sta_lta(sig, 5000, 100000).mean()\n X.loc[seg_id, 'classic_sta_lta3_mean_%d' % i] = classic_sta_lta(sig, 3333, 6666).mean()\n X.loc[seg_id, 'classic_sta_lta4_mean_%d' % i] = classic_sta_lta(sig, 10000, 25000).mean()\n\n X.loc[seg_id, 'Moving_average_700_mean_%d' % i] = sig.rolling(window=700).mean().mean(skipna=True)\n X.loc[seg_id, 'Moving_average_1500_mean_%d' % i] = sig.rolling(window=1500).mean().mean(skipna=True)\n X.loc[seg_id, 'Moving_average_3000_mean_%d' % i] = sig.rolling(window=3000).mean().mean(skipna=True)\n X.loc[seg_id, 'Moving_average_6000_mean_%d' % i] = sig.rolling(window=6000).mean().mean(skipna=True)\n\n ewma = pd.Series.ewm\n X.loc[seg_id, 'exp_Moving_average_300_mean_%d' % i] = ewma(sig, span=300).mean().mean(skipna=True)\n X.loc[seg_id, 'exp_Moving_average_3000_mean_%d' % i] = ewma(sig, span=3000).mean().mean(skipna=True)\n X.loc[seg_id, 'exp_Moving_average_30000_mean_%d' % i] = ewma(sig, span=6000).mean().mean(skipna=True)\n\n no_of_std = 2\n X.loc[seg_id, 'MA_700MA_std_mean_%d' % i] = sig.rolling(window=700).std().mean()\n X.loc[seg_id, 'MA_700MA_BB_high_mean_%d' % i] = (\n X.loc[seg_id, 'Moving_average_700_mean_%d' % i] + no_of_std * X.loc[seg_id, 'MA_700MA_std_mean_%d' % i]).mean()\n X.loc[seg_id, 'MA_700MA_BB_low_mean_%d' % i] = (\n X.loc[seg_id, 'Moving_average_700_mean_%d' % i] - no_of_std * X.loc[seg_id, 'MA_700MA_std_mean_%d' % i]).mean()\n X.loc[seg_id, 'MA_400MA_std_mean_%d' % i] = sig.rolling(window=400).std().mean()\n X.loc[seg_id, 'MA_400MA_BB_high_mean_%d' % i] = (\n X.loc[seg_id, 'Moving_average_700_mean_%d' % i] + no_of_std * X.loc[seg_id, 'MA_400MA_std_mean_%d' % i]).mean()\n X.loc[seg_id, 'MA_400MA_BB_low_mean_%d' % i] = (\n X.loc[seg_id, 'Moving_average_700_mean_%d' % i] - no_of_std * X.loc[seg_id, 'MA_400MA_std_mean_%d' % i]).mean()\n X.loc[seg_id, 'MA_1000MA_std_mean_%d' % i] = sig.rolling(window=1000).std().mean()\n\n X.loc[seg_id, 'iqr_%d' % i] = np.subtract(*np.percentile(sig, [75, 25]))\n X.loc[seg_id, 'q999_%d' % i] = np.quantile(sig, 0.999)\n X.loc[seg_id, 'q001_%d' % i] = np.quantile(sig, 0.001)\n X.loc[seg_id, 'ave10_%d' % i] = stats.trim_mean(sig, 0.1)\n\n for windows in [10, 100, 1000]:\n x_roll_std = xc.rolling(windows).std().dropna().values\n x_roll_mean = xc.rolling(windows).mean().dropna().values\n\n X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()\n X.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()\n X.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()\n X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()\n X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)\n X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)\n X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)\n X.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)\n X.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))\n X.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(\n np.nonzero((np.diff(x_roll_std) / x_roll_std[:-1]))[0])\n X.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std).max()\n\n X.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()\n X.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()\n X.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()\n X.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()\n X.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)\n X.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)\n X.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)\n X.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)\n X.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))\n X.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(\n np.nonzero((np.diff(x_roll_mean) / x_roll_mean[:-1]))[0])\n X.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean).max()\n\n return X","execution_count":8,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_fields(proc_id):\n success = 1\n count = 0\n try:\n seg_st = int(NUM_SEG_PER_PROC * proc_id)\n train_df = pd.read_csv(os.path.join(DATA_DIR, 'raw_data_%d.csv' % proc_id), dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})\n len_df = len(train_df.index)\n start_indices = (np.loadtxt(fname=os.path.join(OUTPUT_DIR, 'start_indices_4k.csv'), dtype=np.int32, delimiter=','))[:, proc_id]\n train_X = pd.DataFrame(dtype=np.float64)\n train_y = pd.DataFrame(dtype=np.float64, columns=['time_to_failure'])\n t0 = time.time()\n\n for seg_id, start_idx in zip(range(seg_st, seg_st + NUM_SEG_PER_PROC), start_indices):\n end_idx = np.int32(start_idx + 150000)\n print('working: %d, %d, %d to %d of %d' % (proc_id, seg_id, start_idx, end_idx, len_df))\n seg = train_df.iloc[start_idx: end_idx]\n # train_X = create_features_pk_det(seg_id, seg, train_X, start_idx, end_idx)\n train_X = create_features(seg_id, seg, train_X, start_idx, end_idx)\n train_y.loc[seg_id, 'time_to_failure'] = seg['time_to_failure'].values[-1]\n\n if count == 10: \n print('saving: %d, %d to %d' % (seg_id, start_idx, end_idx))\n train_X.to_csv('train_x_%d.csv' % proc_id, index=False)\n train_y.to_csv('train_y_%d.csv' % proc_id, index=False)\n\n count += 1\n\n print('final_save, process id: %d, loop time: %.2f for %d iterations' % (proc_id, time.time() - t0, count))\n train_X.to_csv(os.path.join(OUTPUT_DIR, 'train_x_%d.csv' % proc_id), index=False)\n train_y.to_csv(os.path.join(OUTPUT_DIR, 'train_y_%d.csv' % proc_id), index=False)\n\n except:\n print(traceback.format_exc())\n success = 0\n\n return success","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_mp_build():\n t0 = time.time()\n num_proc = NUM_THREADS\n pool = mp.Pool(processes=num_proc)\n results = [pool.apply_async(build_fields, args=(pid, )) for pid in range(NUM_THREADS)]\n output = [p.get() for p in results]\n num_built = sum(output)\n pool.close()\n pool.join()\n print(num_built)\n print('Run time: %.2f' % (time.time() - t0))","execution_count":10,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def join_mp_build():\n df0 = pd.read_csv(os.path.join(OUTPUT_DIR, 'train_x_%d.csv' % 0))\n df1 = pd.read_csv(os.path.join(OUTPUT_DIR, 'train_y_%d.csv' % 0))\n\n for i in range(1, NUM_THREADS):\n print('working %d' % i)\n temp = pd.read_csv(os.path.join(OUTPUT_DIR, 'train_x_%d.csv' % i))\n df0 = df0.append(temp)\n\n temp = pd.read_csv(os.path.join(OUTPUT_DIR, 'train_y_%d.csv' % i))\n df1 = df1.append(temp)\n\n df0.to_csv(os.path.join(OUTPUT_DIR, 'train_x.csv'), index=False)\n df1.to_csv(os.path.join(OUTPUT_DIR, 'train_y.csv'), index=False)","execution_count":11,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_test_fields():\n train_X = pd.read_csv(os.path.join(OUTPUT_DIR, 'train_x.csv'))\n try:\n train_X.drop(labels=['seg_id', 'seg_start', 'seg_end'], axis=1, inplace=True)\n except:\n pass\n\n submission = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'), index_col='seg_id')\n test_X = pd.DataFrame(columns=train_X.columns, dtype=np.float64, index=submission.index)\n\n print('start for loop')\n count = 0\n for seg_id in tqdm_notebook(test_X.index): # just tqdm in IDE\n seg = pd.read_csv(os.path.join(DATA_DIR, 'test', str(seg_id) + '.csv'))\n # train_X = create_features_pk_det(seg_id, seg, train_X, start_idx, end_idx)\n test_X = create_features(seg_id, seg, test_X, 0, 0)\n\n if count % 100 == 0:\n print('working', seg_id)\n count += 1\n\n test_X.to_csv(os.path.join(OUTPUT_DIR, 'test_x.csv'), index=False)","execution_count":12,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def scale_fields(fn_train='train_x.csv', fn_test='test_x.csv', \n fn_out_train='scaled_train_X.csv' , fn_out_test='scaled_test_X.csv'):\n train_X = pd.read_csv(os.path.join(OUTPUT_DIR, fn_train))\n try:\n train_X.drop(labels=['seg_id', 'seg_start', 'seg_end'], axis=1, inplace=True)\n except:\n pass\n test_X = pd.read_csv(os.path.join(OUTPUT_DIR, fn_test))\n\n print('start scaler')\n scaler = StandardScaler()\n scaler.fit(train_X)\n scaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns)\n scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns)\n\n scaled_train_X.to_csv(os.path.join(OUTPUT_DIR, fn_out_train), index=False)\n scaled_test_X.to_csv(os.path.join(OUTPUT_DIR, fn_out_test), index=False)","execution_count":13,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"split_raw_data()\nbuild_rnd_idxs()\nrun_mp_build()\njoin_mp_build()\nbuild_test_fields()\nscale_fields()","execution_count":14,"outputs":[{"output_type":"error","ename":"FileNotFoundError","evalue":"File b'd:\\\\#earthquake\\\\data/train.csv' does not exist","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msplit_raw_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mbuild_rnd_idxs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mrun_mp_build\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mjoin_mp_build\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mbuild_test_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m\u001b[0m in \u001b[0;36msplit_raw_data\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msplit_raw_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDATA_DIR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'train.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mmax_start_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mSIG_LEN\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mslice_len\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_start_index\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m6\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m 676\u001b[0m skip_blank_lines=skip_blank_lines)\n\u001b[1;32m 677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 678\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 679\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 680\u001b[0m \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 439\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 440\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 441\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 442\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 785\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 786\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 787\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 788\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 789\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m 1012\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1014\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1015\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1016\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m 1706\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'usecols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1708\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1710\u001b[0m \u001b[0mpassed_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnames\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n","\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n","\u001b[0;31mFileNotFoundError\u001b[0m: File b'd:\\\\#earthquake\\\\data/train.csv' does not exist"]}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_features_pk_det(seg_id, seg, X, st, end):\n X.loc[seg_id, 'seg_id'] = np.int32(seg_id)\n X.loc[seg_id, 'seg_start'] = np.int32(st)\n X.loc[seg_id, 'seg_end'] = np.int32(end)\n\n sig = pd.Series(seg['acoustic_data'].values)\n b, a = des_bw_filter_lp(cutoff=18000)\n sig = sg.lfilter(b, a, sig)\n\n peakind = []\n noise_pct = .001\n count = 0\n\n while len(peakind) < 12 and count < 24:\n peakind = sg.find_peaks_cwt(sig, np.arange(1, 16), noise_perc=noise_pct, min_snr=4.0)\n noise_pct *= 2.0\n count += 1\n\n if len(peakind) < 12:\n print('Warning: Failed to find 12 peaks for %d' % seg_id)\n\n while len(peakind) < 12:\n peakind.append(149999)\n\n df_pk = pd.DataFrame(data={'pk': sig[peakind], 'idx': peakind}, columns=['pk', 'idx'])\n df_pk.sort_values(by='pk', ascending=False, inplace=True)\n\n for i in range(0, 12):\n X.loc[seg_id, 'pk_idx_%d' % i] = df_pk['idx'].iloc[i]\n X.loc[seg_id, 'pk_val_%d' % i] = df_pk['pk'].iloc[i]\n\n return X","execution_count":15,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\n\ndf = pd.read_csv('test_x_8pk.csv')\ndf_out = None\n\nfor pks in df.itertuples():\n data = {'pk_idxs': [pks.pk_idx_0, pks.pk_idx_1, pks.pk_idx_2, pks.pk_idx_3, pks.pk_idx_4, pks.pk_idx_5, pks.pk_idx_6, pks.pk_idx_7, pks.pk_idx_8, pks.pk_idx_9, pks.pk_idx_10, pks.pk_idx_11],\n 'pk_vals': [pks.pk_val_0, pks.pk_val_1, pks.pk_val_2, pks.pk_val_3, pks.pk_val_4, pks.pk_val_5, pks.pk_val_6, pks.pk_val_7, pks.pk_val_8, pks.pk_val_9, pks.pk_val_10, pks.pk_val_11]}\n pdf = pd.DataFrame(data=data)\n pdf.sort_values(by='pk_idxs', axis=0, inplace=True)\n\n data = {'pk_idx_0': pdf['pk_idxs'].iloc[0], 'pk_val_0': pdf['pk_vals'].iloc[0],\n 'pk_idx_1': pdf['pk_idxs'].iloc[1], 'pk_val_1': pdf['pk_vals'].iloc[1],\n 'pk_idx_2': pdf['pk_idxs'].iloc[2], 'pk_val_2': pdf['pk_vals'].iloc[2],\n 'pk_idx_3': pdf['pk_idxs'].iloc[3], 'pk_val_3': pdf['pk_vals'].iloc[3],\n 'pk_idx_4': pdf['pk_idxs'].iloc[4], 'pk_val_4': pdf['pk_vals'].iloc[4],\n 'pk_idx_5': pdf['pk_idxs'].iloc[5], 'pk_val_5': pdf['pk_vals'].iloc[5],\n 'pk_idx_6': pdf['pk_idxs'].iloc[6], 'pk_val_6': pdf['pk_vals'].iloc[6],\n 'pk_idx_7': pdf['pk_idxs'].iloc[7], 'pk_val_7': pdf['pk_vals'].iloc[7],\n 'pk_idx_8': pdf['pk_idxs'].iloc[8], 'pk_val_8': pdf['pk_vals'].iloc[8],\n 'pk_idx_9': pdf['pk_idxs'].iloc[9], 'pk_val_9': pdf['pk_vals'].iloc[9],\n 'pk_idx_10': pdf['pk_idxs'].iloc[10], 'pk_val_10': pdf['pk_vals'].iloc[10],\n 'pk_idx_11': pdf['pk_idxs'].iloc[11], 'pk_val_11': pdf['pk_vals'].iloc[11]}\n\n if df_out is None:\n df_out = pd.DataFrame(data=data, index=[0])\n else:\n temp = pd.DataFrame(data=data, index=[0])\n df_out = df_out.append(temp, ignore_index=True)\n\ndf_out = df_out[['pk_idx_0', 'pk_val_0',\n 'pk_idx_1', 'pk_val_1',\n 'pk_idx_2', 'pk_val_2',\n 'pk_idx_3', 'pk_val_3',\n 'pk_idx_4', 'pk_val_4',\n 'pk_idx_5', 'pk_val_5',\n 'pk_idx_6', 'pk_val_6',\n 'pk_idx_7', 'pk_val_7',\n 'pk_idx_8', 'pk_val_8',\n 'pk_idx_9', 'pk_val_9',\n 'pk_idx_10', 'pk_val_10',\n 'pk_idx_11', 'pk_val_11']]\nprint(df_out.head())\nprint(df_out.tail())\ndf_out.to_csv('test_x_8pk_by_idx.csv')","execution_count":16,"outputs":[{"output_type":"error","ename":"FileNotFoundError","evalue":"File b'test_x_8pk.csv' does not exist","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'test_x_8pk.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mdf_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m 676\u001b[0m skip_blank_lines=skip_blank_lines)\n\u001b[1;32m 677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 678\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 679\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 680\u001b[0m \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 439\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 440\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 441\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 442\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 785\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 786\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 787\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 788\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 789\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m 1012\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1014\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1015\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1016\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m 1706\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'usecols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1708\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1710\u001b[0m \u001b[0mpassed_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnames\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n","\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n","\u001b[0;31mFileNotFoundError\u001b[0m: File b'test_x_8pk.csv' does not exist"]}]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\npk_idx_base = 'pk_idx_'\npk_val_base = 'pk_val_'\n\nprint('do train')\ndf = pd.read_csv(r'pk8/train_x_8pk.csv')\nslopes = np.zeros((len(df.index), 12))\n\nfor i in df.index:\n for j in range(12):\n pk_idx = pk_idx_base + str(j)\n pk_val = pk_val_base + str(j)\n slopes[i, j] = df[pk_val].iloc[i] / (150000 - df[pk_idx].iloc[i])\n\nfor j in range(12):\n df['slope_' + str(j)] = slopes[:, j]\n\nprint(df.head())\ndf.to_csv(r'pk8/train_x_8_slope.csv', index=False)\n\ndf = pd.read_csv(r'pk8/test_x_8pk.csv')\nslopes = np.zeros((len(df.index), 12))\n\nprint('do test')\nfor i in df.index:\n for j in range(12):\n pk_idx = pk_idx_base + str(j)\n pk_val = pk_val_base + str(j)\n slopes[i, j] = df[pk_val].iloc[i] / (150000 - df[pk_idx].iloc[i])\n\nfor j in range(12):\n df['slope_' + str(j)] = slopes[:, j]\n\nprint(df.head())\ndf.to_csv(r'pk8/test_x_8_slope.csv', index=False)\n\nprint('!DONE!')","execution_count":17,"outputs":[{"output_type":"stream","text":"do train\n","name":"stdout"},{"output_type":"error","ename":"FileNotFoundError","evalue":"File b'pk8/train_x_8pk.csv' does not exist","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'do train'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr'pk8/train_x_8pk.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0mslopes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m12\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m 676\u001b[0m skip_blank_lines=skip_blank_lines)\n\u001b[1;32m 677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 678\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 679\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 680\u001b[0m \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 439\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 440\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 441\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 442\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 785\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 786\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 787\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 788\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 789\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m 1012\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1014\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1015\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1016\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m 1706\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'usecols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1708\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1710\u001b[0m \u001b[0mpassed_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnames\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n","\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n","\u001b[0;31mFileNotFoundError\u001b[0m: File b'pk8/train_x_8pk.csv' does not exist"]}]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 21,\n 'min_data_in_leaf': 20,\n 'objective':'regression',\n 'learning_rate': 0.001,\n 'max_depth': 108,\n \"boosting\": \"gbdt\",\n \"feature_fraction\": 0.91,\n \"bagging_freq\": 1,\n \"bagging_fraction\": 0.91,\n \"bagging_seed\": 42,\n \"metric\": 'mae',\n \"lambda_l1\": 0.1,\n \"verbosity\": -1,\n \"random_state\": 42}\n\n\ndef lgb_base_model():\n maes = []\n rmses = []\n submission = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'), index_col='seg_id')\n scaled_train_X = pd.read_csv(r'train_8_and_9\\scaled_train_X_8.csv')\n scaled_test_X = pd.read_csv(r'train_8_and_9\\scaled_test_X_8.csv')\n train_y = pd.read_csv(r'train_8_and_9\\train_y_8.csv')\n predictions = np.zeros(len(scaled_test_X))\n\n n_fold = 8\n folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)\n\n fold_importance_df = pd.DataFrame()\n fold_importance_df[\"Feature\"] = scaled_train_X.columns\n\n for fold_, (trn_idx, val_idx) in enumerate(folds.split(scaled_train_X, train_y.values)):\n print('working fold %d' % fold_)\n strLog = \"fold {}\".format(fold_)\n print(strLog)\n\n X_tr, X_val = scaled_train_X.iloc[trn_idx], scaled_train_X.iloc[val_idx]\n y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]\n\n model = lgb.LGBMRegressor(**params, n_estimators=80000, n_jobs=-1)\n model.fit(X_tr, y_tr,\n eval_set=[(X_tr, y_tr), (X_val, y_val)], eval_metric='mae',\n verbose=1000, early_stopping_rounds=200)\n\n # predictions\n preds = model.predict(scaled_test_X, num_iteration=model.best_iteration_)\n predictions += preds / folds.n_splits\n preds = model.predict(X_val, num_iteration=model.best_iteration_)\n\n # mean absolute error\n mae = mean_absolute_error(y_val, preds)\n print('MAE: %.6f' % mae)\n maes.append(mae)\n\n # root mean squared error\n rmse = mean_squared_error(y_val, preds)\n print('RMSE: %.6f' % rmse)\n rmses.append(rmse)\n\n fold_importance_df['importance_%d' % fold_] = model.feature_importances_[:len(scaled_train_X.columns)]\n\n print('MAEs', maes)\n print('MAE mean: %.6f' % np.mean(maes))\n print('RMSEs', rmses)\n print('RMSE mean: %.6f' % np.mean(rmses))\n\n submission.time_to_failure = predictions\n submission.to_csv('submission_lgb_8_80k_108dp.csv', index=False)\n fold_importance_df.to_csv('fold_imp_lgb_8_80k_108dp.csv')","execution_count":18,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 21,\n 'min_data_in_leaf': 20,\n 'objective':'regression',\n 'max_depth': 108,\n 'learning_rate': 0.001,\n \"boosting\": \"gbdt\",\n \"feature_fraction\": 0.91,\n \"bagging_freq\": 1,\n \"bagging_fraction\": 0.91,\n \"bagging_seed\": 42,\n \"metric\": 'mae',\n \"lambda_l1\": 0.1,\n \"verbosity\": -1,\n \"random_state\": 42}\n\n\ndef lgb_trimmed_model():\n maes = []\n rmses = []\n tr_maes = []\n tr_rmses = []\n submission = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'), index_col='seg_id')\n\n scaled_train_X = pd.read_csv(r'pk8/scaled_train_X_8.csv')\n df = pd.read_csv(r'pk8/scaled_train_X_8_slope.csv')\n scaled_train_X = scaled_train_X.join(df)\n\n scaled_test_X = pd.read_csv(r'pk8/scaled_test_X_8.csv')\n df = pd.read_csv(r'pk8/scaled_test_X_8_slope.csv')\n scaled_test_X = scaled_test_X.join(df)\n\n pcol = []\n pcor = []\n pval = []\n y = pd.read_csv(r'pk8/train_y_8.csv')['time_to_failure'].values\n\n for col in scaled_train_X.columns:\n pcol.append(col)\n pcor.append(abs(pearsonr(scaled_train_X[col], y)[0]))\n pval.append(abs(pearsonr(scaled_train_X[col], y)[1]))\n\n df = pd.DataFrame(data={'col': pcol, 'cor': pcor, 'pval': pval}, index=range(len(pcol)))\n df.sort_values(by=['cor', 'pval'], inplace=True)\n df.dropna(inplace=True)\n df = df.loc[df['pval'] <= 0.05]\n\n drop_cols = []\n\n for col in scaled_train_X.columns:\n if col not in df['col'].tolist():\n drop_cols.append(col)\n\n scaled_train_X.drop(labels=drop_cols, axis=1, inplace=True)\n scaled_test_X.drop(labels=drop_cols, axis=1, inplace=True)\n\n train_y = pd.read_csv(r'pk8/train_y_8.csv')\n predictions = np.zeros(len(scaled_test_X))\n preds_train = np.zeros(len(scaled_train_X))\n\n print('shapes of train and test:', scaled_train_X.shape, scaled_test_X.shape)\n\n n_fold = 6\n folds = KFold(n_splits=n_fold, shuffle=False, random_state=42)\n\n for fold_, (trn_idx, val_idx) in enumerate(folds.split(scaled_train_X, train_y.values)):\n print('working fold %d' % fold_)\n strLog = \"fold {}\".format(fold_)\n print(strLog)\n\n X_tr, X_val = scaled_train_X.iloc[trn_idx], scaled_train_X.iloc[val_idx]\n y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]\n\n model = lgb.LGBMRegressor(**params, n_estimators=60000, n_jobs=-1)\n model.fit(X_tr, y_tr,\n eval_set=[(X_tr, y_tr), (X_val, y_val)], eval_metric='mae',\n verbose=1000, early_stopping_rounds=200)\n\n # model = xgb.XGBRegressor(n_estimators=1000,\n # learning_rate=0.1,\n # max_depth=6,\n # subsample=0.9,\n # colsample_bytree=0.67,\n # reg_lambda=1.0, # seems best within 0.5 of 2.0\n # # gamma=1,\n # random_state=777+fold_,\n # n_jobs=12,\n # verbosity=2)\n # model.fit(X_tr, y_tr)\n\n # predictions\n preds = model.predict(scaled_test_X) #, num_iteration=model.best_iteration_)\n predictions += preds / folds.n_splits\n preds = model.predict(scaled_train_X) #, num_iteration=model.best_iteration_)\n preds_train += preds / folds.n_splits\n\n preds = model.predict(X_val) #, num_iteration=model.best_iteration_)\n\n # mean absolute error\n mae = mean_absolute_error(y_val, preds)\n print('MAE: %.6f' % mae)\n maes.append(mae)\n\n # root mean squared error\n rmse = mean_squared_error(y_val, preds)\n print('RMSE: %.6f' % rmse)\n rmses.append(rmse)\n\n # training for over fit\n preds = model.predict(X_tr) #, num_iteration=model.best_iteration_)\n\n mae = mean_absolute_error(y_tr, preds)\n print('Tr MAE: %.6f' % mae)\n tr_maes.append(mae)\n\n rmse = mean_squared_error(y_tr, preds)\n print('Tr RMSE: %.6f' % rmse)\n tr_rmses.append(rmse)\n\n print('MAEs', maes)\n print('MAE mean: %.6f' % np.mean(maes))\n print('RMSEs', rmses)\n print('RMSE mean: %.6f' % np.mean(rmses))\n\n print('Tr MAEs', tr_maes)\n print('Tr MAE mean: %.6f' % np.mean(tr_maes))\n print('Tr RMSEs', rmses)\n print('Tr RMSE mean: %.6f' % np.mean(tr_rmses))\n\n submission.time_to_failure = predictions\n submission.to_csv('submission_xgb_slope_pearson_6fold.csv') # index needed, it is seg id\n\n pr_tr = pd.DataFrame(data=preds_train, columns=['time_to_failure'], index=range(0, preds_train.shape[0]))\n pr_tr.to_csv(r'preds_tr_xgb_slope_pearson_6fold.csv', index=False)\n print('Train shape: {}, Test shape: {}, Y shape: {}'.format(scaled_train_X.shape, scaled_test_X.shape, train_y.shape))","execution_count":19,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nimport scipy\nimport sklearn\nprint(sys.version)\nprint('pandas:', pd.__version__)\nprint('numpy:', np.__version__)\nprint('scipy:', scipy.__version__)\nprint('sklearn:', sklearn.__version__)\nprint('light gbm:', lgb.__version__)\nprint('xgboost:', xgb.__version__)","execution_count":20,"outputs":[{"output_type":"stream","text":"3.6.6 |Anaconda, Inc.| (default, Oct 9 2018, 12:34:16) \n[GCC 7.3.0]\npandas: 0.23.4\nnumpy: 1.16.3\nscipy: 1.1.0\nsklearn: 0.20.3\nlight gbm: 2.2.3\nxgboost: 0.82\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}