{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-22T20:36:18.890888Z","iopub.execute_input":"2021-08-22T20:36:18.891512Z","iopub.status.idle":"2021-08-22T20:36:18.895739Z","shell.execute_reply.started":"2021-08-22T20:36:18.891474Z","shell.execute_reply":"2021-08-22T20:36:18.894754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\nfrom scipy import fftpack  # Fourier\nfrom scipy.ndimage import maximum_filter1d\nfrom librosa.feature import mfcc, spectral_contrast, zero_crossing_rate\nfrom tqdm import tqdm\nfrom tsfresh.feature_extraction import feature_calculators as fc","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:41:10.573600Z","iopub.execute_input":"2021-08-22T20:41:10.574308Z","iopub.status.idle":"2021-08-22T20:41:10.580456Z","shell.execute_reply.started":"2021-08-22T20:41:10.574250Z","shell.execute_reply":"2021-08-22T20:41:10.579449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_segment(segment_id):\n    \"\"\"Returns the data about a specific segment_id\"\"\"\n    try:\n        return pd.read_csv(f\"{PATH_DATA}train/{segment_id}.csv\")\n    except FileNotFoundError:\n        return pd.read_csv(f\"{PATH_DATA}test/{segment_id}.csv\")\n\n    \ndef get_index(target=\"train\"):\n    \"\"\"Returns the list of segments of a set (train or test)\"\"\"\n    file = \"train\" if target == \"train\" else \"sample_submission\"\n    data = pd.read_csv(f\"{PATH_DATA}{file}.csv\")\n    return data[\"segment_id\"].values\n\n    \ndef get_features(sig, sensor_id):\n    \"\"\"Analysis of a signal. Grabs temporal and frequential features.\n    Returns a pandas dataframe\"\"\"\n\n    fourier = fftpack.fft(sig.values)\n    real, imag = np.real(fourier), np.imag(fourier)\n\n    # Temporal data\n    features = {}\n    features[f\"{sensor_id}_mean\"] = [sig.mean()]\n    features[f\"{sensor_id}_var\"] = [sig.var()]\n    features[f\"{sensor_id}_skew\"] = [sig.skew()]\n    features[f\"{sensor_id}_delta\"] = [sig.max() - sig.min()]\n    features[f\"{sensor_id}_mad\"] = [sig.mad()]\n    features[f\"{sensor_id}_kurtosis\"] = [sig.kurtosis()]\n    features[f\"{sensor_id}_sem\"] = [sig.sem()]\n    features[f\"{sensor_id}_q5\"] = [np.quantile(sig, 0.05)]\n    features[f\"{sensor_id}_q25\"] = [np.quantile(sig, 0.25)]\n    features[f\"{sensor_id}_q75\"] = [np.quantile(sig, 0.75)]\n    features[f\"{sensor_id}_q95\"] = [np.quantile(sig, 0.95)]\n    grad_rol_max = [maximum_filter1d(np.gradient(np.abs(sig.values)), 50)]\n    delta = np.max(grad_rol_max) - np.min(grad_rol_max)\n    features[f\"{sensor_id}_grmax_delta\"] = delta\n\n    # Frequencial\n    features[f\"{sensor_id}_real_mean\"] = [real.mean()]\n    features[f\"{sensor_id}_real_var\"] = [real.var()]\n    features[f\"{sensor_id}_real_delta\"] = [real.max() - real.min()]\n\n    features[f\"{sensor_id}_imag_mean\"] = [imag.mean()]\n    features[f\"{sensor_id}_imag_var\"] = [imag.var()]\n    features[f\"{sensor_id}_imag_delta\"] = [imag.max() - imag.min()]\n\n    features[f\"{sensor_id}_nb_peak\"] = fc.number_peaks(sig.values, 2)\n    features[f\"{sensor_id}_median_roll_std\"] = np.median(\n        pd.Series(sig).rolling(50).std().dropna().values)\n    features[f\"{sensor_id}_autocorr5\"] = fc.autocorrelation(sig, 5)\n\n    # Added 16\n    features[f\"{sensor_id}_nb_peak_3\"] = fc.number_peaks(sig.values, 3)\n    features[f\"{sensor_id}_absquant95\"] = np.quantile(np.abs(sig), 0.95)\n\n    try:\n        # Mel-frequency cepstral coefficients\n        mfcc_mean = mfcc(sig.values).mean(axis=1)\n        for i in range(20):\n            features[f\"{sensor_id}_mfcc_mean_{i}\"] = mfcc_mean[i]\n        # Contrast spectral\n        spec_contrast = spectral_contrast(sig.values).mean(axis=1)\n        for i in range(7):\n            features[f\"{sensor_id}_lib_spec_cont_{i}\"] = spec_contrast[i]\n        features[f\"{sensor_id}_zero_cross\"] = zero_crossing_rate(sig)[0].mean()\n        # Added 16\n        features[f\"{sensor_id}_percentile_roll20_std_50\"] = np.percentile(\n            sig.rolling(20).std().dropna().values, 50)\n\n    except:\n        pass\n\n    return pd.DataFrame.from_dict(features)\n\n\ndef preprocess_data(target=\"train\"):\n    \"\"\"Generates a dataframe containing all the features of a\n    dataset (train or test)\"\"\"\n    data_set = []\n    for seg in tqdm(get_index(target)):\n        train_row = [pd.DataFrame.from_dict({\"segment_id\": [seg]})]\n        data = load_segment(seg)\n        for i in range(10):\n            sensor_id = f\"sensor_{i+1}\"\n            train_row.append(get_features(data[sensor_id], sensor_id))\n        train_row = pd.concat(train_row, axis=1)\n        data_set.append(train_row)\n\n    data_set = pd.concat(data_set).reset_index()\n    data_set.fillna(-1, inplace=True)\n    data_set.drop(['index'], axis=1, inplace=True)\n    return data_set","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:41:11.162441Z","iopub.execute_input":"2021-08-22T20:41:11.162850Z","iopub.status.idle":"2021-08-22T20:41:11.198507Z","shell.execute_reply.started":"2021-08-22T20:41:11.162812Z","shell.execute_reply":"2021-08-22T20:41:11.197235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_DATA = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/'\nPATH_PREPRO = '/kaggle/working/interim/preprocessing/'\n\nif not os.path.exists(PATH_PREPRO):\n    os.makedirs(PATH_PREPRO)\ntrain_set = preprocess_data()\ntrain_set.to_csv(PATH_PREPRO + \"train_set.csv\", index=False)\ntest_set = preprocess_data(\"test\")\ntest_set.to_csv(PATH_PREPRO + \"test_set.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-21T13:12:34.534079Z","iopub.execute_input":"2021-08-21T13:12:34.534436Z","iopub.status.idle":"2021-08-21T14:46:18.986272Z","shell.execute_reply.started":"2021-08-21T13:12:34.534409Z","shell.execute_reply":"2021-08-21T14:46:18.985171Z"},"trusted":true},"execution_count":null,"outputs":[]}]}