{"cells":[{"metadata":{},"cell_type":"markdown","source":" # Load libraries and simple functions"},{"metadata":{},"cell_type":"markdown","source":"# AKNOWLEGEMENTS\n\nLGBM parameters were taken from here https://www.kaggle.com/amanjain1008/erupting-volcano-all-in-one-different-eda"},{"metadata":{"trusted":true},"cell_type":"code","source":"ver = 35","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport zipfile as zf\nimport os\nimport datetime as dt\nfrom tqdm import tqdm, tqdm_notebook\nimport re\nimport string\nimport warnings\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import StandardScaler\n\nplt.style.use('ggplot')\n\n\ndef get_df_name(df):\n    name =[x for x in globals() if globals()[x] is df][0]\n    return name\ndef tstats(t, doplot = False):\n    print('#'*20, get_df_name(t), t.shape)\n    for _,i in enumerate(t):\n        nulls = t[i].isna().sum()\n        if nulls > 0:\n            print(i,'=', t[i].nunique(), ',NULLS = ', nulls, ',% of nulls = ',round(100*nulls/t.shape[0]))\n        else:\n            print(i,'=', t[i].nunique())\n    if doplot:\n        print()\n        print(t.sample(10))\n        print()\n        \nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" # Code to load files. Some logic is displayed and explained later"},{"metadata":{"trusted":true},"cell_type":"code","source":"# '/Users/user/Downloads/'\n\nimport os\nfrom tqdm import tqdm_notebook\nimport scipy.fftpack\nfrom scipy.signal import chirp, find_peaks, peak_widths, peak_prominences\n\nhomedir = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/'\n\ndef read_n_files(dirname, nfiles):\n    tmp = pd.DataFrame()\n    cnt = 0\n    for dirname, _, filenames in os.walk(homedir + dirname):\n        for filename in filenames:\n            cnt += 1\n            if cnt > nfiles: break\n            t = pd.read_csv(os.path.join(dirname, filename))\n            t['segment_id'] = filename[:-4]\n            tmp = pd.concat([tmp, t], ignore_index=True)\n        \n    return tmp\n\nt10 = read_n_files('train', 10).fillna(0)\nfloat_cols = [c for c in t10 if t10[c].dtype == \"float64\"]\nfloat32_cols = {c: np.float32 for c in float_cols}\n\n\nnparts = 6 # number of variables = number of sensors * 4 variables for one part * nparts. if nparts == 10 => nvars = 10 parts * 10 sensors * 4 = 400\nr = [int(j*30000/nparts) for j in range(nparts+1)]\n\ndef get_peaks(tab, col):\n    \n    # Number of samplepoints\n    N = tab.shape[0]\n    # sample spacing\n    T = 1.0 / 800.0\n    x = np.linspace(0.0, N*T, N)\n    y = tab[col].values\n    yf = scipy.fftpack.fft(y)\n    xf = np.linspace(0.0, int(1.0/(2.0*T)), int(N/2))\n\n    x = np.log(2.0/N * np.abs(yf[:N//2]))\n    peaks, _ = find_peaks(x, distance=500)\n    \n    return x, peaks\n\ndef read_n_files32(dirname, nfiles):\n    tmp_res = pd.DataFrame()\n    cnt = 0\n    for dirname, _, filenames in os.walk(homedir + dirname):\n        for filename in tqdm_notebook(filenames):\n            cnt += 1\n            if cnt > nfiles: break\n            t = pd.read_csv(os.path.join(dirname, filename), engine='c', dtype=float32_cols)\n            #################################\n            \n\n            res = []\n            for i in [i for i in range(1,11)]:\n\n                x, peaks = get_peaks(t, 'sensor_'+str(i))\n\n                for j in range(len(r)-1):\n\n                    xpeaks = peaks[(peaks > r[j])&(peaks <= r[j+1])]\n                    ypeaks = x[xpeaks]\n\n                    mx = round(np.median(xpeaks))\n                    sx = round(np.std(xpeaks))\n\n                    my = round(np.median(ypeaks), 2)\n                    sy = round(np.std(ypeaks), 2)\n                    \n                    r0, r1 = r[j], r[j+1]\n\n                    res.append([f's{i:02d}_r{r0:05d}_{r1}_mx', mx])\n                    res.append([f's{i:02d}_r{r0:05d}_{r1}_sx', sx])\n                    res.append([f's{i:02d}_r{r0:05d}_{r1}_my', my])\n                    res.append([f's{i:02d}_r{r0:05d}_{r1}_sy', sy])\n                    \n                    \n                #################################\n\n                df1 = pd.DataFrame({'ind':peaks, 'x':x[peaks]})\n                df2 = pd.DataFrame({'ind':np.linspace(0,30000,30001).astype(int)})\n                df = df2.merge(df1, how='left', on='ind') # we need to get nans in order to interpolate them\n\n                df = df.interpolate(method='linear', limit_direction='both', axis=0)\n\n                data = df['x'].values\n                data[0] = data[:5000].min()# first value is often the largest, so it isn't a peak. we need it to be not the largest if we want a first peak\n\n                peaks, _ = find_peaks(data)\n\n                prominences = peak_prominences(data, peaks)[0]\n                contour_heights = data[peaks] - prominences\n\n                results_half = peak_widths(data, peaks, rel_height=0.5)\n                results_half[0]  # widths\n\n                results_full = peak_widths(data, peaks, rel_height=1)\n                results_full[0]  # widths\n\n                peak_data = pd.DataFrame({\n                    'peak_x':peaks,\n                    'peak_y':data[peaks],\n                    'width':results_full[0],\n                    'height':prominences\n                }).sort_values(by='height', ascending=False)\n\n                tmp = peak_data.reset_index(drop=True).reset_index().head(5)\n                tmp['index'] = tmp['index'].astype(str).apply(lambda x: x.zfill(2))\n\n                tmp = tmp.melt(id_vars=['index'], value_vars=['peak_x', 'peak_y', 'width', 'height'])\n\n                tmp['variable'] = tmp['variable'] + tmp['index']\n\n                tmp = np.array(tmp[['variable', 'value']])\n                for titem in tmp:\n                    res.append([f's{i:02d}p_' + titem[0], titem[1]])\n\n                #################################\n            t = pd.DataFrame(np.array(res), columns=['col', 'val'])\n            t['segment_id'] = filename[:-4]\n            t['val'] = t['val'].astype(float)\n            t = t.fillna(0)\n            t = t.pivot_table(\n                index='segment_id',\n                columns='col',\n                values='val'\n            ).reset_index()\n            tmp_res = pd.concat([tmp_res, t], ignore_index=True)\n        \n    return tmp_res.fillna(0)\n\ndef read_all_files(dirname):\n    tmp = pd.DataFrame()\n    cnt = 0\n    for dirname, _, filenames in os.walk(homedir + dirname):\n        for filename in filenames:\n            cnt += 1\n#             if cnt > nfiles: break\n            t = pd.read_csv(os.path.join(dirname, filename)).fillna(0)\n            cols = t.columns\n            t['segment_id'] = filename[:-4]\n            t = t.groupby('segment_id', as_index=False).median()\n            tmp = pd.concat([tmp, t], ignore_index=True)\n        \n    return tmp","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" # Load first 10 files and explore some of them"},{"metadata":{"trusted":true},"cell_type":"code","source":"t10 = read_n_files('train', 10).fillna(0)\nt10['segment_id'].unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" # Plot sensors info"},{"metadata":{"trusted":true},"cell_type":"code","source":"t = t10.loc[t10['segment_id'] == '800654756'].copy()\nplt.figure(figsize=(15,15))\n\nfor i in [i for i in range(1,11)]:\n    \n    if i > 4: k = i - 4 \n    else: k = i\n    plt.subplot(3,4,i)\n\n    plt.plot(t.index, t['sensor_'+str(i)])\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" # Plot fast Fourier of sensors signals and point peaks on them"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,15))\n\nfor i in [i for i in range(1,11)]:\n    \n    plt.subplot(3,4,i)\n\n    # Number of samplepoints\n    N = t.shape[0]\n    # sample spacing\n    T = 1.0 / 800.0\n    x = np.linspace(0.0, N*T, N)\n    y = t['sensor_'+str(i)].values\n    yf = scipy.fftpack.fft(y)\n    xf = np.linspace(0.0, int(1.0/(2.0*T)), int(N/2))\n    \n    x = 2.0/N * np.abs(yf[:N//2])\n    peaks, _ = find_peaks(x, distance=500, height=0.5)\n    \n    plt.plot(x)\n    plt.plot(peaks, x[peaks], \"x\")\n    plt.plot(np.zeros_like(x), \"--\", color=\"gray\")\n    plt.title(str(np.median(x[peaks])))\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" # Transform FFT to logscale and find peaks.\n Break peaks to intervals, and get summary in each interval"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,15))\n\nfor i in [i for i in range(1,11)]:\n    \n    plt.subplot(3,4,i)\n\n    # Number of samplepoints\n    N = t.shape[0]\n    # sample spacing\n    T = 1.0 / 800.0\n    x = np.linspace(0.0, N*T, N)\n    y = t['sensor_'+str(i)].values\n    yf = scipy.fftpack.fft(y)\n    xf = np.linspace(0.0, int(1.0/(2.0*T)), int(N/2))\n    \n    x = np.log(2.0/N * np.abs(yf[:N//2]))\n    peaks, _ = find_peaks(x, distance=500)\n    \n    plt.plot(x)\n    plt.plot(peaks, x[peaks], \"x\")\n    plt.plot(np.zeros_like(x), \"--\", color=\"gray\")\n\n#     r = [j*5000 for j in range(7)]\n    for j in range(len(r)-1):\n\n        xpeaks = peaks[(peaks > r[j])&(peaks <= r[j+1])]\n        ypeaks = x[xpeaks]\n\n        mx = round(np.median(xpeaks))\n        sx = round(np.std(xpeaks))\n\n        my = round(np.median(ypeaks), 2)\n        sy = round(np.std(ypeaks), 2)\n        print('plot', i, 'range =', r[j], r[j+1], f': x med = {mx}, x sd = {sx}, y med = {my}, y sd = {sy}')\n    \n    print()\n#     plt.title(str(np.median(x[peaks])))\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explore single sensor"},{"metadata":{"trusted":true},"cell_type":"code","source":"i=2\n\nplt.figure(figsize=(15,5))\n\nx, peaks = get_peaks(t, 'sensor_'+str(i))\n\nplt.plot(x)\nplt.plot(peaks, x[peaks], \"x\")\nplt.show()\n\ndf1 = pd.DataFrame({'ind':peaks, 'x':x[peaks]})\ndf2 = pd.DataFrame({'ind':np.linspace(0,30000,30001).astype(int)})\ndf = df2.merge(df1, how='left', on='ind') # we need to get nans in order to interpolate them\n\ndf = df.interpolate(method='linear', limit_direction='both', axis=0)\n\ndata = df['x'].values\ndata[0] = data[:5000].min()# first value is often the largest, so it isn't a peak. we need it to be not the largest if we want a first peak\n\npeaks, _ = find_peaks(data)\n\nprominences = peak_prominences(data, peaks)[0]\ncontour_heights = data[peaks] - prominences\n\nresults_half = peak_widths(data, peaks, rel_height=0.5)\nresults_half[0]  # widths\n\nresults_full = peak_widths(data, peaks, rel_height=1)\nresults_full[0]  # widths\n\nplt.figure(figsize=(15,5))\n\nplt.plot(data)\nplt.plot(peaks, data[peaks], \"x\")\n\nplt.hlines(*results_half[1:], color=\"C2\")\nplt.hlines(*results_full[1:], color=\"C3\")\n\nplt.vlines(x=peaks, ymin=contour_heights, ymax=data[peaks])\n\nplt.show()\n\npeak_data = pd.DataFrame({\n    'peak_x':peaks,\n    'peak_y':data[peaks],\n    'width':results_full[0],\n    'height':prominences\n}).sort_values(by='height', ascending=False)\n\n# peak_data\n\ntmp = peak_data.reset_index(drop=True).reset_index().head(5)\ntmp['index'] = tmp['index'].astype(str).apply(lambda x: x.zfill(2))\n\ntmp = tmp.melt(id_vars=['index'], value_vars=['peak_x', 'peak_y', 'width', 'height'])\n\ntmp['variable'] = tmp['variable'] + tmp['index']\n\n# tmp = tmp.drop(columns='index').set_index('variable').T\n\ntmp","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transform peaks to line and find peaks on it\nthen state x,y,width and height for each of top 5 tall peaks"},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.signal import find_peaks_cwt\n\nplt.figure(figsize=(15,15))\n\nres = []\n\nfor i in [i for i in range(1,11)]:\n    \n    plt.subplot(3,4,i)\n\n    x, peaks = get_peaks(t, 'sensor_'+str(i))\n    \n    for j in range(len(r)-1):\n\n        xpeaks = peaks[(peaks > r[j])&(peaks <= r[j+1])]\n        ypeaks = x[xpeaks]\n\n        mx = round(np.median(xpeaks))\n        sx = round(np.std(xpeaks))\n\n        my = round(np.median(ypeaks), 2)\n        sy = round(np.std(ypeaks), 2)\n\n        r0, r1 = r[j], r[j+1]\n        \n        res.append([f's{i:02d}_r{r0:05d}_{r1}_mx', mx])\n        res.append([f's{i:02d}_r{r0:05d}_{r1}_sx', sx])\n        res.append([f's{i:02d}_r{r0:05d}_{r1}_my', my])\n        res.append([f's{i:02d}_r{r0:05d}_{r1}_sy', sy])\n\n    df1 = pd.DataFrame({'ind':peaks, 'x':x[peaks]})\n    df2 = pd.DataFrame({'ind':np.linspace(0,30000,30001).astype(int)})\n    df = df2.merge(df1, how='left', on='ind') # we need to get nans in order to interpolate them\n\n    df = df.interpolate(method='linear', limit_direction='both', axis=0)\n\n    data = df['x'].values\n    data[0] = data[:5000].min()# first value is often the largest, so it isn't a peak. we need it to be not the largest if we want a first peak\n\n    peaks, _ = find_peaks(data)\n\n    prominences = peak_prominences(data, peaks)[0]\n    contour_heights = data[peaks] - prominences\n\n    results_half = peak_widths(data, peaks, rel_height=0.5)\n    results_half[0]  # widths\n\n    results_full = peak_widths(data, peaks, rel_height=1)\n    results_full[0]  # widths\n\n    plt.plot(data)\n    plt.plot(peaks, data[peaks], \"x\")\n\n    plt.hlines(*results_half[1:], color=\"C2\")\n    plt.hlines(*results_full[1:], color=\"C3\")\n\n    plt.vlines(x=peaks, ymin=contour_heights, ymax=data[peaks])\n\n    peak_data = pd.DataFrame({\n        'peak_x':peaks,\n        'peak_y':data[peaks],\n        'width':results_full[0],\n        'height':prominences\n    }).sort_values(by='height', ascending=False)\n\n    # peak_data\n\n    tmp = peak_data.reset_index(drop=True).reset_index().head(5) # top5. some sensors go flat, some show 100 peaks. let's count most visible ones\n    tmp['index'] = tmp['index'].astype(str).apply(lambda x: x.zfill(2)) # just for fancy grouping of vars\n\n    tmp = tmp.melt(id_vars=['index'], value_vars=['peak_x', 'peak_y', 'width', 'height'])\n\n    tmp['variable'] = tmp['variable'] + tmp['index']\n\n#     tmp = tmp.drop(columns='index').set_index('variable').T\n    tmp = np.array(tmp[['variable', 'value']])\n    for titem in tmp:\n        res.append([f's{i:02d}_' + titem[0], titem[1]])\n        \n        \nplt.show()\n\ntmp = pd.DataFrame(np.array(res), columns=['col', 'val'])\ntmp['segment_id'] = t['segment_id'][0]\ntmp['val'] = tmp['val'].astype(float)\ntmp = tmp.fillna(0)\ntmp = tmp.pivot_table(\n    index='segment_id',\n    columns='col',\n    values='val'\n).reset_index()\n# res = pd.concat([tmp, t], ignore_index=True)\ntmp\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" # Read all data using transformations mentioned earlier\n \n i've uploaded the result of this two inputs as a dataset, because it takes long to create the files"},{"metadata":{"trusted":true},"cell_type":"code","source":"# t4m = read_n_files32('train', 4431)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# t4p = read_n_files32('test', 4520)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" # Load training labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"# label = pd.read_csv(homedir+'train.csv')\n# tstats(label)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" # Add label to dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# label['segment_id'] = label['segment_id'].astype(str)\n# t4m = t4m.merge(label, how='left', on='segment_id')\n# tstats(t4m)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Import ML staff"},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nimport pandas as pd\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nfrom sklearn.metrics import f1_score,recall_score,precision_score,roc_auc_score\nfrom sklearn.model_selection import GridSearchCV, train_test_split\nfrom sklearn.ensemble import RandomForestRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import KFold\nimport gc\n\ndef plot_history(history, metricname):\n    recall = history.history[metricname]\n    val_recall = history.history['val_' + metricname]\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n    x = range(1, len(recall) + 1)\n\n    plt.figure(figsize=(12, 5))\n    plt.subplot(1, 2, 1)\n    plt.plot(x, recall, 'b', label='Training')\n    plt.plot(x, val_recall, 'r', label='Validation')\n    plt.title('Training and validation')\n    plt.legend()\n    plt.subplot(1, 2, 2)\n    plt.plot(x, loss, 'b', label='Training loss')\n    plt.plot(x, val_loss, 'r', label='Validation loss')\n    plt.title('Training and validation loss')\n    plt.legend()\n    plt.show()\n    \nfrom keras import regularizers as kreg\nfrom keras.layers import Dense\nfrom keras.models import Sequential","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define x and y\n\nand also create correlation matrix and make features as a sum and multiplication of features, \ncorrelated with target\n\nthen do a quantile transformation in order to get Gaussian distribution of target"},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_data(scale=False):\n    \n    t4m = pd.read_csv('/kaggle/input/volcanic-eruptions/t4m_10.csv')\n    t4p = pd.read_csv('/kaggle/input/volcanic-eruptions/t4p_10.csv')\n    features = t4m.columns[1:-1]\n    target = 'time_to_eruption'\n\n    df = t4m[t4m.columns[1:]].copy()\n    t = df.corr()[target].reset_index()\n    t[target] = abs(t[target])\n    t.sort_values(by=target, ascending = False)\n\n    t = t.loc[(t[target] > 0.05)&(t[target] < 1)].sort_values(by=target, ascending=False).reset_index()\n\n    good_features = t['index'].values\n\n    scaler = StandardScaler()\n\n    # for train\n    t = pd.DataFrame(scaler.fit_transform(t4m[good_features]), columns = good_features)\n\n    t['sum_feat'] = t[good_features].sum(axis=1)\n    t['mul_feat'] = t[good_features].prod(axis=1)\n    t['s_m_feat'] = t['sum_feat'] * t['mul_feat']\n\n    t4m['sum_feat'] = t['sum_feat'].values\n    t4m['mul_feat'] = t['mul_feat'].values\n    t4m['s_m_feat'] = t['s_m_feat'].values\n\n\n    # for test\n    t = pd.DataFrame(scaler.fit_transform(t4p[good_features]), columns = good_features)\n\n    t['sum_feat'] = t[good_features].sum(axis=1)\n    t['mul_feat'] = t[good_features].prod(axis=1)\n    t['s_m_feat'] = t['sum_feat'] * t['mul_feat']\n\n    t4p['sum_feat'] = t['sum_feat'].values\n    t4p['mul_feat'] = t['mul_feat'].values\n    t4p['s_m_feat'] = t['s_m_feat'].values\n\n    feat = list(features)\n    feat.append('sum_feat')\n    feat.append('mul_feat')\n    feat.append('s_m_feat')\n    \n\n    if scale:\n        scaler = StandardScaler()\n        t4m[feat] = scaler.fit_transform(t4m[feat])\n        t4p[feat] = scaler.transform(t4p[feat])\n    \n    X, y = t4m[feat], t4m[target]\n\n    data = y.values.reshape(-1, 1)\n\n    from sklearn.preprocessing import QuantileTransformer\n\n    rng = np.random.RandomState(304)\n    qt = QuantileTransformer(n_quantiles=1000, output_distribution='normal',\n                             random_state=rng)\n\n    z = qt.fit_transform(data)\n    z_back = qt.inverse_transform(z)\n\n#     plt.figure(figsize=(15,5))\n#     plt.subplot(131)\n#     plt.hist(data)\n#     plt.title('y')\n\n#     plt.subplot(132)\n#     plt.hist(z, bins = 50)\n#     plt.title('z')\n\n#     plt.subplot(133)\n#     plt.hist(z_back, bins = 50)\n#     plt.title('inverse')\n\n#     plt.show()\n    \n    return t4m, t4p, X, y, z, feat, target, qt\n\nt4m, t4p, X, y, z, feat, target, qt = prepare_data()\n\ndef de_qt(val):\n    return qt.inverse_transform(val.reshape(-1, 1)).ravel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_model_performance(y_test_plot, y_pred_plot):\n\n    converter2days = 1/60/60/24/1000\n    #     res = pd.DataFrame({'y':inv_boxcox1p(y_test_plot, bclambda)*converter2mos,'preds':inv_boxcox1p(y_pred_plot, bclambda)*converter2mos})\n    res = pd.DataFrame({\n        'y':y_test_plot.reshape(-1, 1).ravel()*converter2days,\n        'preds':y_pred_plot.reshape(-1, 1).ravel()*converter2days})\n    res['diff'] = abs(res['preds'] - res['y'])\n    res['err'] = res['diff']/res['y']\n    res.sort_values(by = 'preds', inplace = True)\n    res.reset_index(inplace = True)\n    res['index'] = res.index\n\n    lmetric = int(mean_absolute_error(y_test_plot, y_pred_plot))\n    print('The MAE of prediction is:', lmetric)\n\n    fig = plt.figure(figsize=(19, 4))\n    ax1 = fig.add_subplot(1, 3, 1)\n    ax2 = fig.add_subplot(1, 3, 2)\n    ax3 = fig.add_subplot(1, 3, 3)\n    ax1.hist(y_train, bins=25)\n    ax1.set_title('train labels')\n\n    ax2.hist(y_test, bins=25)\n    ax2.set_title('test labels')\n\n    ax3.hist(y_pred, bins=25)\n    ax3.set_title('Predictions')\n\n    plt.show()\n\n\n    plt.figure(figsize = (15,5))\n    plt.plot(res['index'], res['y'], 'o-b', label = 'true labels')\n    plt.plot(res['index'], res['preds'], 'o-g', label = 'predictions')\n\n    plt.title('Prediction Power')\n    plt.xlabel('Measurement')\n    plt.ylabel('Delta Magnitude')\n    plt.legend(['true','preds'])\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CONCLUSIONS:\n\n    LGBM is the winner\n    k-fold is must-have\n    optimal number of folds = 5\n    LGBM works better with scaling\n    \n    Accuracy on hold-out:\n    \n    keras - 4.8 (4.8 w/o k-fold)\n    catboost - 3.3 (3.9 w/o k-fold)\n    lgbm - 2.91\n\n    SumFeat does cool!\n    Both peak and interval features are important"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nt4m, t4p, X, y, z, feat, target, qt = prepare_data(scale=True)\n\nn_fold = 7\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=101)\n\nparams = {\n    \"n_estimators\": 5000,\n    \"boosting_type\": \"gbdt\",\n    \"metric\": \"mae\",\n    \"num_leaves\": 66,\n    \"learning_rate\": 0.005,\n    \"feature_fraction\": 0.9,\n    \"bagging_fraction\": 0.8,\n    \"agging_freq\": 3,\n    \"max_bins\": 2048,\n    \"verbose\": 0,\n    \"random_state\": 101,\n    \"nthread\": -1,\n    \"device\": \"cpu\",\n}\n\ny_pred = np.zeros(t4p.shape[0])\n\nt_train = t4m.copy()\n\nfor n_fold, (trn_idx, val_idx) in enumerate(folds.split(t_train)):\n    print(f\"Fold {n_fold}:\")\n    trn_x, trn_y = t_train[feat].iloc[trn_idx], t_train[target].iloc[trn_idx]\n    val_x, val_y = t_train[feat].iloc[val_idx], t_train[target].iloc[val_idx]\n    \n    model = lgb.LGBMRegressor(**params)\n    \n    model.fit(trn_x, trn_y, \n            eval_set= [(trn_x, trn_y), (val_x, val_y)], \n            eval_metric=\"mae\", verbose=200, early_stopping_rounds=50\n           )\n\n    y_pred += model.predict(t4p[feat], num_iteration=model.best_iteration_) / folds.n_splits\n    \nplt.figure()\nplt.hist(y_pred, bins=50)\nplt.title('y pred distribution')\nplt.show()\n\nmy_submission = pd.DataFrame({\n    'segment_id':t4p['segment_id'].values,\n    'time_to_eruption':y_pred\n})\n\n# my_submission.loc[my_submission[target] < 60000, target] = 60000 # in case of below zero predictions, correct them according to training data\n# my_submission.loc[my_submission[target] > 4.8e+07, target] = 4.8e+07 # correct preds according to train limits\n\nmy_submission[target] = abs(my_submission[target])\n\nmy_submission.to_csv(f'submission_{ver}.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}