{"cells":[{"metadata":{},"cell_type":"markdown","source":"The datasets are saved here: <a href=\"https://www.kaggle.com/damoonshahhosseini/volcano-pca\">Dataset</a>"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n# Paths to the training and testing datasets\ntrain_path = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/train/'\ntest_path = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/test/'\n\n# Training data\ntrain = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train.csv')\nsample_submission = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv')\n\n# Getting the ids\ntrain_ids = train['segment_id']\ntest_ids = sample_submission['segment_id']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preprocessing:\n- Data is normalized, dimension reduction will not be usefull sicne the data is a series of continous signlas.\n- Missing values should be imputed with zero since it just means that there are no signals detected.\n- Features for each signal:\n    - min / max\n    - std / mean / mad / skew / Kurtosis\n    - quantiles: 1, 5, 10, 20, 25, 40, 50, 65, 75, 80, 85, 90, 95, 99\n    - IQR, OMax: Q1  - 1.5 IQR, OMin :Q3 + 1.5IQR "},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_features(col):\n    \"\"\"\n        Returns a one row DataFrame of features for \n        a singel signal.\n        \n        input:\n            col: the column (signal) used for feature\n                extraction.\n            \n        return:\n            pd.DataFrame object containing the data of \n            the signal.\n    \"\"\"\n    features = {} # Features dictionary\n    # Measure of central tendency\n    features['max'] = col.max()\n    features['min'] = col.min()\n    features['std'] = col.std()\n    features['mean'] = col.mean()\n    features['mad'] = col.mad()\n    features['skew'] = col.skew()\n    features['kurtosis'] = col.kurtosis()\n    \n    # Quantiles\n    features['q-01'] = np.quantile(col, 0.01)\n    features['q-05'] = np.quantile(col, 0.05)\n    features['q-10'] = np.quantile(col, 0.1)\n    features['q-25'] = np.quantile(col, 0.25)\n    features['q-50'] = np.quantile(col, 0.5)\n    features['q-75'] = np.quantile(col, 0.75)\n    features['q-90'] = np.quantile(col, 0.90)\n    features['q-95'] = np.quantile(col, 0.95)\n    features['q-99'] = np.quantile(col, 0.99)\n    \n    return pd.DataFrame(features, index=range(1))\n\ndef reduce_data(data):\n    \"\"\"\n        Reduce the data by only getting the measures of central\n        tendency.\n        \n        input: Gets a dataframe\n        return: Returns a dataframe containing the measures of\n            central tendency needed for the function\n    \"\"\"\n    dataframe = None\n    \n    for s in data.columns:\n        if dataframe is None:\n            dataframe = extract_features(data[s])\n        else:\n            dataframe = pd.concat([dataframe, extract_features(data[s])])\n            \n    return dataframe\n\ndef process_data(ids, file_name, base_path):\n    data = np.empty(shape=(len(ids),10,16))\n\n    for index, Id in enumerate(ids):\n        df = pd.read_csv(f'{base_path}{Id}.csv').fillna(0)\n        data[index] = reduce_data(df)\n    \n    # save the data\n    np.save(f'{file_name}.npy', data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Process data and save the train and test dataset\nprocess_data(test_ids, 'test', test_path)\nprocess_data(train_ids, 'train', train_path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Rolling Dataset\n- Breaking data into smaller chunks and record the change of its measures (mean, std, etc.) can help to make accurate prediction."},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_rolling_features(rolling_data, rolling_type):\n    \"\"\"\n        Gets the measures of central tendency for the rolling data.\n        \n        input:\n            rolling_data\n            rolling_type: mean / std / skew / residual of a rolling data\n        \n        return: \n            dataframe conatining the measures of central tendency\n    \"\"\"\n    return pd.concat(\n        {\n            f'{rolling_type}_mean': rolling_data.mean(), \n            f'{rolling_type}_std': rolling_data.std(), \n            f'{rolling_type}_skew': rolling_data.skew(), \n            f'{rolling_type}_residual': rolling_data.max() - rolling_data.min()\n        }, axis=1)\n    \ndef rolling_analysis(data):\n    \"\"\"\n        Gets the measures of central tendency on the rolling data\n        \n        input:\n            data: Given data\n        \n        return:\n            dataframe containing the features for the measures of central \n            tendency for the rolling data\n    \"\"\"\n    # Getting the rolling data\n    indexer = pd.api.indexers.FixedForwardWindowIndexer(window_size=300) # Instantiating the indexer object\n    rolling_data = data.rolling(window=indexer, min_periods=1)\n    \n    # measures of central tendesncy for the rolling data\n    rolling_mean = rolling_data.mean() # Rolling averages\n    rolling_std = rolling_data.std() # Rolling std\n    rolling_skew = rolling_data.skew() # Rolling skew\n    rolling_residual = rolling_data.max() - rolling_data.min() # Rolling residual values\n    \n    # Extracting their features\n    mean_feats = extract_rolling_features(rolling_mean, 'rmean')\n    std_feats = extract_rolling_features(rolling_std, 'rstd')\n    skew_feats = extract_rolling_features(rolling_skew, 'rskew')\n    residual_feats = extract_rolling_features(rolling_residual, 'rres')\n    \n    \n    return pd.concat([mean_feats, std_feats, skew_feats, residual_feats], axis=1).fillna(0)\n\ndef process_rolling_data(ids, file_name, base_path):\n    data = np.empty(shape=(len(ids),10,16))\n\n    for index, Id in enumerate(ids):\n        df = pd.read_csv(f'{base_path}{Id}.csv').fillna(0)\n        data[index] = rolling_analysis(df)\n    \n    # save the data\n    np.save(f'{file_name}.npy', data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Process the rolling data\nprocess_rolling_data(train_ids, 'rolling_train', train_path)\nprocess_rolling_data(test_ids, 'rolling_test', test_path)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}