{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-11T13:02:11.809353Z","iopub.execute_input":"2022-12-11T13:02:11.809694Z","iopub.status.idle":"2022-12-11T13:02:11.817639Z","shell.execute_reply.started":"2022-12-11T13:02:11.809626Z","shell.execute_reply":"2022-12-11T13:02:11.816783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR\nfrom sklearn.metrics import mean_absolute_error\nfrom tqdm import tqdm_notebook\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:02:11.822775Z","iopub.execute_input":"2022-12-11T13:02:11.823190Z","iopub.status.idle":"2022-12-11T13:02:12.729228Z","shell.execute_reply.started":"2022-12-11T13:02:11.823131Z","shell.execute_reply":"2022-12-11T13:02:12.728109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2022-12-11T13:02:12.730547Z","iopub.execute_input":"2022-12-11T13:02:12.730830Z","iopub.status.idle":"2022-12-11T13:06:41.334611Z","shell.execute_reply.started":"2022-12-11T13:02:12.730778Z","shell.execute_reply":"2022-12-11T13:06:41.332531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:06:41.336691Z","iopub.execute_input":"2022-12-11T13:06:41.336990Z","iopub.status.idle":"2022-12-11T13:06:41.373997Z","shell.execute_reply.started":"2022-12-11T13:06:41.336913Z","shell.execute_reply":"2022-12-11T13:06:41.372968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.tail()","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:06:41.375350Z","iopub.execute_input":"2022-12-11T13:06:41.375808Z","iopub.status.idle":"2022-12-11T13:06:41.393155Z","shell.execute_reply.started":"2022-12-11T13:06:41.375751Z","shell.execute_reply":"2022-12-11T13:06:41.391124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pandas doesn't show us all the decimals\npd.options.display.precision = 15","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:06:41.395170Z","iopub.execute_input":"2022-12-11T13:06:41.395591Z","iopub.status.idle":"2022-12-11T13:06:41.403019Z","shell.execute_reply.started":"2022-12-11T13:06:41.395526Z","shell.execute_reply":"2022-12-11T13:06:41.402147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# much better!\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:06:41.404087Z","iopub.execute_input":"2022-12-11T13:06:41.404748Z","iopub.status.idle":"2022-12-11T13:06:41.427178Z","shell.execute_reply.started":"2022-12-11T13:06:41.404674Z","shell.execute_reply":"2022-12-11T13:06:41.425412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:06:41.428484Z","iopub.execute_input":"2022-12-11T13:06:41.428961Z","iopub.status.idle":"2022-12-11T13:06:41.437844Z","shell.execute_reply.started":"2022-12-11T13:06:41.428888Z","shell.execute_reply":"2022-12-11T13:06:41.436368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(2,1, figsize=(20,12))\nax[0].plot(train.index.values[:10000000], train.time_to_failure.values[:10000000], c=\"darkred\")\nax[0].set_title(\"Quaketime of 10 Mio rows\")\nax[0].set_xlabel(\"Index\")\nax[0].set_ylabel(\"Quaketime in ms\");\nax[1].plot(train.index.values[:10000000], train.acoustic_data.values[:10000000], c=\"mediumseagreen\")\nax[1].set_title(\"Signal of 10 Mio rows\")\nax[1].set_xlabel(\"Index\")\nax[1].set_ylabel(\"Acoustic Signal\");","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:06:41.439228Z","iopub.execute_input":"2022-12-11T13:06:41.439460Z","iopub.status.idle":"2022-12-11T13:06:48.597773Z","shell.execute_reply.started":"2022-12-11T13:06:41.439417Z","shell.execute_reply":"2022-12-11T13:06:48.596574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"100 마다 표본 추출 시각화","metadata":{}},{"cell_type":"code","source":"train_ad_sample_df = train['acoustic_data'].values[::100]\ntrain_ttf_sample_df = train['time_to_failure'].values[::100]\n\ndef plot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df, title=\"Acoustic data and time to failure: 1% sampled data\"):\n    fig, ax1 = plt.subplots(figsize=(12, 8))\n    plt.title(title)\n    plt.plot(train_ad_sample_df, color='r')\n    ax1.set_ylabel('acoustic data', color='r')\n    plt.legend(['acoustic data'], loc=(0.01, 0.95))\n    ax2 = ax1.twinx()\n    plt.plot(train_ttf_sample_df, color='b')\n    ax2.set_ylabel('time to failure', color='b')\n    plt.legend(['time to failure'], loc=(0.01, 0.9))\n    plt.grid(True)\n\nplot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df)\ndel train_ad_sample_df\ndel train_ttf_sample_df","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:06:48.600657Z","iopub.execute_input":"2022-12-11T13:06:48.600898Z","iopub.status.idle":"2022-12-11T13:06:52.477213Z","shell.execute_reply.started":"2022-12-11T13:06:48.600851Z","shell.execute_reply":"2022-12-11T13:06:52.475598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ad_sample_df = train['acoustic_data'].values[::100]\ntrain_ttf_sample_df = train['time_to_failure'].values[::100]","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:06:52.478585Z","iopub.execute_input":"2022-12-11T13:06:52.478924Z","iopub.status.idle":"2022-12-11T13:06:52.485228Z","shell.execute_reply.started":"2022-12-11T13:06:52.478852Z","shell.execute_reply":"2022-12-11T13:06:52.483461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"=> train data에서 총 16번의 지진이 발생.\n\n=> 음향 신호가 크게 변동하고 나서 일정 시간 이후 지진 발생.\n\n=> 지진이 발생하기 직전에는 음향 신호 작아짐.","metadata":{}},{"cell_type":"code","source":"# plot -150,000 to +30000 samples right around the earthquake\nfailure_idxs = [5656574,  50085878, 104677356, 138772453, 187641820, 218652630,\n                245829585, 307838917, 338276287, 375377848, 419368880, 461811623,\n                495800225, 528777115, 585568144, 621985673]\nf, axes = plt.subplots(4, 4, figsize=(14, 8))\ni = 0\nj = 0\n\nfor idx in failure_idxs:\n    ad = train['acoustic_data'].values[idx - 150000: idx + 30000]\n    ttf = train['time_to_failure'].values[idx - 150000: idx + 30000]\n    \n    axes[j][i].plot(ad, color='tab:orange')\n    \n    s = axes[j][i].twinx()\n    s.plot(ttf, color='tab:blue')\n    \n    i += 1\n    if i >= 4:\n        i = 0\n        j += 1\n\nplt.tight_layout()\nplt.show()\ndel ad, ttf","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:06:52.487027Z","iopub.execute_input":"2022-12-11T13:06:52.487396Z","iopub.status.idle":"2022-12-11T13:06:57.029859Z","shell.execute_reply.started":"2022-12-11T13:06:52.487324Z","shell.execute_reply":"2022-12-11T13:06:57.028731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, axes = plt.subplots(4, 4, figsize=(14, 8))\ni = 0\nj = 0\n\nfor idx in failure_idxs:\n    ad = train['acoustic_data'].values[idx - 2000000: idx + 30000]\n    ttf = train['time_to_failure'].values[idx - 2000000: idx + 30000]\n    \n    axes[j][i].plot(ad, color='tab:orange')\n    axes[j][i].set_xticklabels([])\n    \n    s = axes[j][i].twinx()\n    s.plot(ttf, color='tab:blue')\n    \n    i += 1\n    if i >= 4:\n        i = 0\n        j += 1\n\nplt.tight_layout()\nplt.show()\ndel ad, ttf\n","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:06:57.031324Z","iopub.execute_input":"2022-12-11T13:06:57.031623Z","iopub.status.idle":"2022-12-11T13:07:13.896804Z","shell.execute_reply.started":"2022-12-11T13:06:57.031550Z","shell.execute_reply":"2022-12-11T13:07:13.896083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, axes = plt.subplots(4, 4, figsize=(14, 8))\ni = 0\nj = 0\n\nfor idx in failure_idxs:\n    ad = train['acoustic_data'].values[idx - 2500000: idx + 1000000]\n    ttf = train['time_to_failure'].values[idx - 2500000: idx + 1000000]\n    \n    axes[j][i].plot(ad, color='tab:orange')\n    axes[j][i].set_xticklabels([])\n    \n    s = axes[j][i].twinx()\n    s.plot(ttf, color='tab:blue')\n    \n    i += 1\n    if i >= 4:\n        i = 0\n        j += 1\n\nplt.tight_layout()\nplt.show()\ndel ad, ttf\n","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:07:13.897712Z","iopub.execute_input":"2022-12-11T13:07:13.898063Z","iopub.status.idle":"2022-12-11T13:07:38.376305Z","shell.execute_reply.started":"2022-12-11T13:07:13.898016Z","shell.execute_reply":"2022-12-11T13:07:38.375085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"# Create a training file with simple derived features\n\nrows = 150_000\nsegments = int(np.floor(train.shape[0] / rows))\n\nX_train = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['ave', 'std', 'max', 'min'])\ny_train = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['time_to_failure'])\n\nfor segment in tqdm(range(segments)):\n    seg = train.iloc[segment*rows:segment*rows+rows]\n    x = seg['acoustic_data'].values\n    y = seg['time_to_failure'].values[-1]\n    \n    y_train.loc[segment, 'time_to_failure'] = y\n    \n    X_train.loc[segment, 'ave'] = x.mean()\n    X_train.loc[segment, 'std'] = x.std()\n    X_train.loc[segment, 'max'] = x.max()\n    X_train.loc[segment, 'min'] = x.min()","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:07:38.378197Z","iopub.execute_input":"2022-12-11T13:07:38.378452Z","iopub.status.idle":"2022-12-11T13:07:46.453225Z","shell.execute_reply.started":"2022-12-11T13:07:38.378406Z","shell.execute_reply":"2022-12-11T13:07:46.451883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:07:46.454884Z","iopub.execute_input":"2022-12-11T13:07:46.455183Z","iopub.status.idle":"2022-12-11T13:07:46.471234Z","shell.execute_reply.started":"2022-12-11T13:07:46.455136Z","shell.execute_reply":"2022-12-11T13:07:46.470193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(X_train)\nX_train_scaled = scaler.transform(X_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:07:46.472640Z","iopub.execute_input":"2022-12-11T13:07:46.472865Z","iopub.status.idle":"2022-12-11T13:07:46.485294Z","shell.execute_reply.started":"2022-12-11T13:07:46.472821Z","shell.execute_reply":"2022-12-11T13:07:46.483866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"svm = NuSVR()\nsvm.fit(X_train_scaled, y_train.values.flatten())\ny_pred = svm.predict(X_train_scaled)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:07:46.486908Z","iopub.execute_input":"2022-12-11T13:07:46.487472Z","iopub.status.idle":"2022-12-11T13:07:47.338668Z","shell.execute_reply.started":"2022-12-11T13:07:46.487409Z","shell.execute_reply":"2022-12-11T13:07:47.337371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(6, 6))\nplt.scatter(y_train.values.flatten(), y_pred)\nplt.xlim(0, 20)\nplt.ylim(0, 20)\nplt.xlabel('actual', fontsize=12)\nplt.ylabel('predicted', fontsize=12)\nplt.plot([(0, 0), (20, 20)], [(0, 0), (20, 20)])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:07:47.340147Z","iopub.execute_input":"2022-12-11T13:07:47.340462Z","iopub.status.idle":"2022-12-11T13:07:47.597598Z","shell.execute_reply.started":"2022-12-11T13:07:47.340401Z","shell.execute_reply":"2022-12-11T13:07:47.596176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = mean_absolute_error(y_train.values.flatten(), y_pred)\nprint(f'Score: {score:0.3f}')","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:07:47.598953Z","iopub.execute_input":"2022-12-11T13:07:47.599251Z","iopub.status.idle":"2022-12-11T13:07:47.609297Z","shell.execute_reply.started":"2022-12-11T13:07:47.599201Z","shell.execute_reply":"2022-12-11T13:07:47.605761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:07:47.610525Z","iopub.execute_input":"2022-12-11T13:07:47.610836Z","iopub.status.idle":"2022-12-11T13:07:47.634430Z","shell.execute_reply.started":"2022-12-11T13:07:47.610773Z","shell.execute_reply":"2022-12-11T13:07:47.633200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:07:47.636970Z","iopub.execute_input":"2022-12-11T13:07:47.637634Z","iopub.status.idle":"2022-12-11T13:07:47.645787Z","shell.execute_reply.started":"2022-12-11T13:07:47.637573Z","shell.execute_reply":"2022-12-11T13:07:47.644582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for seg_id in X_test.index:\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    \n    x = seg['acoustic_data'].values\n    \n    X_test.loc[seg_id, 'ave'] = x.mean()\n    X_test.loc[seg_id, 'std'] = x.std()\n    X_test.loc[seg_id, 'max'] = x.max()\n    X_test.loc[seg_id, 'min'] = x.min()\n    \n    \n    #FFT transform values\n    xc = pd.Series(seg['acoustic_data'].values)\n    zc = np.fft.fft(xc)\n    \n    realFFT = np.real(zc)\n    imagFFT = np.imag(zc)\n    \n    \n    #real and imag\n    X_test.loc[seg_id, 'Rmean'] = realFFT.mean()\n    X_test.loc[seg_id, 'Rstd'] = realFFT.std()\n    X_test.loc[seg_id, 'Rmax'] = realFFT.max()\n    X_test.loc[seg_id, 'Rmin'] = realFFT.min()\n    X_test.loc[seg_id, 'Imean'] = imagFFT.mean()\n    X_test.loc[seg_id, 'Istd'] = imagFFT.std()\n    X_test.loc[seg_id, 'Imax'] = imagFFT.max()\n    X_test.loc[seg_id, 'Imin'] = imagFFT.min()\n    X_test.loc[seg_id, 'Rmean_last_5000'] = realFFT[-5000:].mean()\n    X_test.loc[seg_id, 'Rstd__last_5000'] = realFFT[-5000:].std()\n    X_test.loc[seg_id, 'Rmax_last_5000'] = realFFT[-5000:].max()\n    X_test.loc[seg_id, 'Rmin_last_5000'] = realFFT[-5000:].min()\n    X_test.loc[seg_id, 'Rmean_last_15000'] = realFFT[-15000:].mean()\n    X_test.loc[seg_id, 'Rstd_last_15000'] = realFFT[-15000:].std()\n    X_test.loc[seg_id, 'Rmax_last_15000'] = realFFT[-15000:].max()\n    X_test.loc[seg_id, 'Rmin_last_15000'] = realFFT[-15000:].min()\n    \n    \n    #basic fft \n    #X_test.loc[seg_id, 'basic_fft_mean'] = zc.mean()\n    #X_test.loc[seg_id, 'basic_fft_std'] = zc.std()\n    #X_test.loc[seg_id, 'basic_fft_max'] = zc.max()\n    #X_test.loc[seg_id, 'basic_fft_min'] = zc.min()\n\n    #X_test.loc[seg_id, 'basic_fft_last_5000'] = zc[-5000:].mean()\n    #X_test.loc[seg_id, 'basic_fft_last_5000'] = zc[-5000:].std()\n    #X_test.loc[seg_id, 'basic_fft_last_5000'] = zc[-5000:].max()\n    #X_test.loc[seg_id, 'basic_fft_last_5000'] = zc[-5000:].min()\n    #X_test.loc[seg_id, 'basic_fft_last_15000'] = zc[-15000:].mean()\n    #X_test.loc[seg_id, 'basic_fft_last_15000'] = zc[-15000:].std()\n    #X_test.loc[seg_id, 'basic_fft_last_15000'] = zc[-15000:].max()\n    #X_test.loc[seg_id, 'basic_fft_last_15000'] = zc[-15000:].min()\n\n\n    #sampling basic std, mean, min, max \n    X_test.loc[seg_id, 'std_first_50000'] = xc[:50000].std()\n    X_test.loc[seg_id, 'std_last_50000'] = xc[-50000:].std()\n    X_test.loc[seg_id, 'std_first_10000'] = xc[:10000].std()\n    X_test.loc[seg_id, 'std_last_10000'] = xc[-10000:].std()\n\n    X_test.loc[seg_id, 'avg_first_50000'] = xc[:50000].mean()\n    X_test.loc[seg_id, 'avg_last_50000'] = xc[-50000:].mean()\n    X_test.loc[seg_id, 'avg_first_10000'] = xc[:10000].mean()\n    X_test.loc[seg_id, 'avg_last_10000'] = xc[-10000:].mean()\n\n    X_test.loc[seg_id, 'min_first_50000'] = xc[:50000].min()\n    X_test.loc[seg_id, 'min_last_50000'] = xc[-50000:].min()\n    X_test.loc[seg_id, 'min_first_10000'] = xc[:10000].min()\n    X_test.loc[seg_id, 'min_last_10000'] = xc[-10000:].min()\n\n    X_test.loc[seg_id, 'max_first_50000'] = xc[:50000].max()\n    X_test.loc[seg_id, 'max_last_50000'] = xc[-50000:].max()\n    X_test.loc[seg_id, 'max_first_10000'] = xc[:10000].max()\n    X_test.loc[seg_id, 'max_last_10000'] = xc[-10000:].max()\n\n    #mean_change_rate\n    X_test.loc[seg_id, 'mean_change_rate_first_50000'] = np.mean(np.nonzero((np.diff(xc[:50000]) / xc[:50000][:-1]))[0])\n    X_test.loc[seg_id, 'mean_change_rate_last_50000'] = np.mean(np.nonzero((np.diff(xc[-50000:]) / xc[-50000:][:-1]))[0])\n    X_test.loc[seg_id, 'mean_change_rate_first_10000'] = np.mean(np.nonzero((np.diff(xc[:10000]) / xc[:10000][:-1]))[0])\n    X_test.loc[seg_id, 'mean_change_rate_last_10000'] = np.mean(np.nonzero((np.diff(xc[-10000:]) / xc[-10000:][:-1]))[0])\n\n    #rolling \n    for windows in [10, 100, 1000]:\n        x_roll_std = xc.rolling(windows).std().dropna().values\n        x_roll_mean = xc.rolling(windows).mean().dropna().values\n\n        X_test.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()\n        X_test.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()\n        X_test.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()\n        X_test.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()\n        X_test.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)\n        X_test.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)\n        X_test.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)\n        X_test.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)\n\n        X_test.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()\n        X_test.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()\n        X_test.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()\n        X_test.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()\n        X_test.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)\n        X_test.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)\n        X_test.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)\n        X_test.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T14:37:15.856457Z","iopub.execute_input":"2022-12-11T14:37:15.856821Z","iopub.status.idle":"2022-12-11T14:42:11.453963Z","shell.execute_reply.started":"2022-12-11T14:37:15.856762Z","shell.execute_reply":"2022-12-11T14:42:11.452719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"X_test_scaled = scaler.transform(X_test)\nsubmission['time_to_failure'] = svm.predict(X_test_scaled)\nsubmission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:31:42.804182Z","iopub.execute_input":"2022-12-11T13:31:42.804520Z","iopub.status.idle":"2022-12-11T13:31:43.052776Z","shell.execute_reply.started":"2022-12-11T13:31:42.804455Z","shell.execute_reply":"2022-12-11T13:31:42.992232Z"}}},{"cell_type":"markdown","source":"submission","metadata":{"execution":{"iopub.status.busy":"2022-12-11T13:31:42.993226Z","iopub.status.idle":"2022-12-11T13:31:42.993584Z"}}},{"cell_type":"code","source":"from tsfresh.feature_extraction import feature_calculators\nimport librosa\nimport pywt\n\n\nnp.random.seed(1337)\nnoise = np.random.normal(0, 0.5, 150_000)\n\n\ndef denoise_signal_simple(x, wavelet='db4', level=1):\n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    #univeral threshold\n    uthresh = 10\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n    # Reconstruct the signal using the thresholded coefficients\n    return pywt.waverec(coeff, wavelet, mode='per')\n\n\ndef feature_gen(z):\n    X = pd.DataFrame(index=[0], dtype=np.float64)\n    \n    z = z + noise\n    z = z - np.median(z)\n\n    den_sample_simple = denoise_signal_simple(z)\n    mfcc = librosa.feature.mfcc(z)\n    mfcc_mean = mfcc.mean(axis=1)\n    percentile_roll50_std_20 = np.percentile(pd.Series(z).rolling(50).std().dropna().values, 20)\n    \n    X['var_num_peaks_2_denoise_simple'] = feature_calculators.number_peaks(den_sample_simple, 2)\n    X['var_percentile_roll50_std_20'] = percentile_roll50_std_20\n    X['var_mfcc_mean18'] = mfcc_mean[18]\n    X['var_mfcc_mean4'] = mfcc_mean[4]\n    \n    return X","metadata":{"execution":{"iopub.status.busy":"2022-12-11T14:42:11.455599Z","iopub.execute_input":"2022-12-11T14:42:11.455887Z","iopub.status.idle":"2022-12-11T14:42:11.470843Z","shell.execute_reply.started":"2022-12-11T14:42:11.455831Z","shell.execute_reply":"2022-12-11T14:42:11.468872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from joblib import Parallel, delayed\nimport scipy as sp\nimport itertools\nimport gc\n\ndef parse_sample(sample, start):\n    delta = feature_gen(sample['acoustic_data'].values)\n    delta['start'] = start\n    delta['target'] = sample['time_to_failure'].values[-1]\n    return delta\n    \ndef sample_train_gen(df, segment_size=150_000, indices_to_calculate=[0]):\n    result = Parallel(n_jobs=4, temp_folder=\"/tmp\", max_nbytes=None, backend=\"multiprocessing\")(delayed(parse_sample)(df[int(i) : int(i) + segment_size], int(i)) \n                                                                                                for i in tqdm(indices_to_calculate))\n    data = [r.values for r in result]\n    data = np.vstack(data)\n    X = pd.DataFrame(data, columns=result[0].columns)\n    X = X.sort_values(\"start\")\n    return X\n\ndef parse_sample_test(seg_id):\n    sample = pd.read_csv('../input/test/' + seg_id + '.csv', dtype={'acoustic_data': np.int32})\n    delta = feature_gen(sample['acoustic_data'].values)\n    delta['seg_id'] = seg_id\n    return delta\n\ndef sample_test_gen():\n    X = pd.DataFrame()\n    submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\n    result = Parallel(n_jobs=4, temp_folder=\"/tmp\", max_nbytes=None, backend=\"multiprocessing\")(delayed(parse_sample_test)(seg_id) for seg_id in tqdm(submission.index))\n    data = [r.values for r in result]\n    data = np.vstack(data)\n    X = pd.DataFrame(data, columns=result[0].columns)\n    return X\n\nindices_to_calculate = train.index.values[::150_000][:-1]\n\ntrain = sample_train_gen(train, indices_to_calculate=indices_to_calculate)\ngc.collect()\ntest = sample_test_gen()","metadata":{"execution":{"iopub.status.busy":"2022-12-11T14:42:11.472121Z","iopub.execute_input":"2022-12-11T14:42:11.472321Z","iopub.status.idle":"2022-12-11T14:42:12.337510Z","shell.execute_reply.started":"2022-12-11T14:42:11.472289Z","shell.execute_reply":"2022-12-11T14:42:12.280920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"etq_meta = [\n{\"start\":0,         \"end\":5656574},\n{\"start\":5656574,   \"end\":50085878},\n{\"start\":50085878,  \"end\":104677356},\n{\"start\":104677356, \"end\":138772453},\n{\"start\":138772453, \"end\":187641820},\n{\"start\":187641820, \"end\":218652630},\n{\"start\":218652630, \"end\":245829585},\n{\"start\":245829585, \"end\":307838917},\n{\"start\":307838917, \"end\":338276287},\n{\"start\":338276287, \"end\":375377848},\n{\"start\":375377848, \"end\":419368880},\n{\"start\":419368880, \"end\":461811623},\n{\"start\":461811623, \"end\":495800225},\n{\"start\":495800225, \"end\":528777115},\n{\"start\":528777115, \"end\":585568144},\n{\"start\":585568144, \"end\":621985673},\n{\"start\":621985673, \"end\":629145480},\n]\n\nfor i, etq in enumerate(etq_meta):\n    train.loc[(train['start'] + 150_000 >= etq[\"start\"]) & (train['start'] <= etq[\"end\"] - 150_000), \"eq\"] = i\n\ntrain_sample = train[train[\"eq\"].isin([2, 7, 0, 4, 11, 13, 9, 1, 14, 10])]","metadata":{"execution":{"iopub.status.busy":"2022-12-11T14:49:44.864293Z","iopub.execute_input":"2022-12-11T14:49:44.864674Z","iopub.status.idle":"2022-12-11T14:49:44.943346Z","shell.execute_reply.started":"2022-12-11T14:49:44.864621Z","shell.execute_reply":"2022-12-11T14:49:44.940927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Mean:   {train_sample['target'].mean():.4}\")\nprint(f\"Median: {train_sample['target'].median():.4}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-11T14:49:45.081823Z","iopub.execute_input":"2022-12-11T14:49:45.082244Z","iopub.status.idle":"2022-12-11T14:49:45.090077Z","shell.execute_reply.started":"2022-12-11T14:49:45.082182Z","shell.execute_reply":"2022-12-11T14:49:45.088787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold\nfrom numpy import random\nimport lightgbm as lgb\n\nrandom.seed(1234)\n\nfeatures = ['var_num_peaks_2_denoise_simple','var_percentile_roll50_std_20','var_mfcc_mean4',  'var_mfcc_mean18']\ntarget = train_sample[\"target\"].values\n\ntrain_X = train_sample[features].values\ntest_X = test[features].values\n\nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\noof = np.zeros(len(train_X))\nprediction = np.zeros(len(submission))\n\nn_fold = 3\n\nkf = KFold(n_splits=n_fold, shuffle=True, random_state=1337)\nkf = list(kf.split(np.arange(len(train_sample))))\n\nfor fold_n, (train_index, valid_index) in enumerate(kf):\n    print('Fold', fold_n)\n\n    trn_data = lgb.Dataset(train_X[train_index], label=target[train_index])\n    val_data = lgb.Dataset(train_X[valid_index], label=target[valid_index])\n    \n    params = {'num_leaves': 4,\n      'min_data_in_leaf': 5,\n      'objective':'fair',\n      'max_depth': -1,\n      'learning_rate': 0.02,\n      \"boosting\": \"gbdt\",\n      'boost_from_average': True,\n      \"feature_fraction\": 0.9,\n      \"bagging_freq\": 1,\n      \"bagging_fraction\": 0.5,\n      \"bagging_seed\": 0,\n      \"metric\": 'mae',\n      \"verbosity\": -1,\n      'max_bin': 500,\n      'reg_alpha': 0,\n      'reg_lambda': 0,\n      'seed': 0,\n      'n_jobs': 1\n      }\n\n    clf = lgb.train(params, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 1000)\n\n    oof[valid_index] += clf.predict(train_X[valid_index], num_iteration=clf.best_iteration)\n    prediction += clf.predict(test_X, num_iteration=clf.best_iteration)\n\nprediction /= n_fold\n\nprint('\\nMAE: ', mean_absolute_error(target, oof))\n","metadata":{"execution":{"iopub.status.busy":"2022-12-11T14:49:45.274736Z","iopub.execute_input":"2022-12-11T14:49:45.275151Z","iopub.status.idle":"2022-12-11T14:49:46.510908Z","shell.execute_reply.started":"2022-12-11T14:49:45.275080Z","shell.execute_reply":"2022-12-11T14:49:46.509883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['time_to_failure'] = prediction \nprint(submission.head())\nsubmission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-11T14:49:46.512822Z","iopub.execute_input":"2022-12-11T14:49:46.513149Z","iopub.status.idle":"2022-12-11T14:49:46.536703Z","shell.execute_reply.started":"2022-12-11T14:49:46.513098Z","shell.execute_reply":"2022-12-11T14:49:46.535249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}