{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Time Series Transformations\n\nTaking inspiration from the great notebook that [@pavelvod](https://www.kaggle.com/pavelvod) shared on [AMEX EDA: Revealing time patterns of features](https://www.kaggle.com/code/pavelvod/amex-eda-revealing-time-patterns-of-features/notebook).\n\nAiming to continue the data analysis to review time series features from the data.\n\n### Time series decompose\nWe are able to decompose the time series to understand four key components. \n1. Level: The average value in the series.\n2. Trend: The increasing or decreasing value in the series.\n3. Seasonality: The repeating short-term cycle in the series.\n4. Noise: The random variation in the series.\n\nWith this notebook we will aim to review each of these to understand the temporal nature of the features.\n\n### Stationarity\nAiming to assess if the data points show stationary or non-stationary characteristics","metadata":{}},{"cell_type":"code","source":"# Import libraries\nimport numpy as np\nimport pandas as pd\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom statsmodels.tsa.stattools import acf, pacf\nfrom statsmodels.graphics.tsaplots import plot_acf, plot_pacf\nimport gc\n\n# Decompose time series\nfrom statsmodels.tsa.seasonal import seasonal_decompose\n\n# Stationarity\nfrom statsmodels.tsa.stattools import adfuller, kpss\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-19T10:46:11.101703Z","iopub.execute_input":"2022-06-19T10:46:11.102348Z","iopub.status.idle":"2022-06-19T10:46:11.401278Z","shell.execute_reply.started":"2022-06-19T10:46:11.102308Z","shell.execute_reply":"2022-06-19T10:46:11.400210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_agg = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet').assign(S_2=lambda dx: pd.to_datetime(dx.S_2)).groupby('S_2').mean()\nend_of_train = pd.to_datetime(train_agg.index).max()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:31:42.854589Z","iopub.execute_input":"2022-06-19T09:31:42.855452Z","iopub.status.idle":"2022-06-19T09:32:18.781585Z","shell.execute_reply.started":"2022-06-19T09:31:42.855401Z","shell.execute_reply":"2022-06-19T09:32:18.780601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set number of features to review\nN = 47\ntest_agg = []\nfor cols2use in train_agg.columns.values.reshape(-1, N):\n    test_agg.append(pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet', columns=cols2use.tolist() + ['S_2']).assign(S_2=lambda dx: pd.to_datetime(dx.S_2)).groupby('S_2').mean())\n\n\ntest_agg = pd.concat(test_agg, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:32:18.783143Z","iopub.execute_input":"2022-06-19T09:32:18.783517Z","iopub.status.idle":"2022-06-19T09:33:27.569865Z","shell.execute_reply.started":"2022-06-19T09:32:18.783485Z","shell.execute_reply":"2022-06-19T09:33:27.568587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_data = pd.concat([train_agg, test_agg])\nagg_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:33:27.571446Z","iopub.execute_input":"2022-06-19T09:33:27.571951Z","iopub.status.idle":"2022-06-19T09:33:27.598709Z","shell.execute_reply.started":"2022-06-19T09:33:27.571901Z","shell.execute_reply":"2022-06-19T09:33:27.597578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Review a small list of features to understand the outputs\nfor first_letter in list(set([col.split('_')[0] for col in agg_data.columns])):\n    print(first_letter)\n    for feature in agg_data.columns[:10]:\n        if feature[0] != first_letter:\n            continue\n        print(feature)","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:33:27.600138Z","iopub.execute_input":"2022-06-19T09:33:27.600638Z","iopub.status.idle":"2022-06-19T09:33:27.608990Z","shell.execute_reply.started":"2022-06-19T09:33:27.600591Z","shell.execute_reply":"2022-06-19T09:33:27.607843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Partial Autocorrelation and Autocorrelation graphs\nfor first_letter in list(set([col.split('_')[0] for col in agg_data.columns])):\n    for feature_name in agg_data.columns[:10]:\n        if feature_name[0] != first_letter:\n            continue\n        s = agg_data.loc[:, feature_name]\n        print(feature_name) # for Ctrl + F\n        fig = plt.figure(figsize=(16, 6))\n        sub_pacf = fig.add_subplot(2,2,4)\n        sub_acf = fig.add_subplot(2,2,3) \n        mn = fig.add_subplot(2,2,(1,2)) \n        max_lags = 150\n        plot_pacf(s, lags=max_lags, ax=sub_acf)\n        plot_acf(s, lags=max_lags, ax=sub_pacf)\n        s.plot(color='green', ax=mn)\n        mn.axvline(end_of_train, color='red', linestyle='--')\n        mn.set_title(feature_name)\n        plt.subplots_adjust(wspace= 0.25, hspace= 0.25)\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:33:27.610330Z","iopub.execute_input":"2022-06-19T09:33:27.611334Z","iopub.status.idle":"2022-06-19T09:33:35.954797Z","shell.execute_reply.started":"2022-06-19T09:33:27.611281Z","shell.execute_reply":"2022-06-19T09:33:35.953640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decompose the time series\nfor first_letter in list(set([col.split('_')[0] for col in agg_data.columns])):\n    for feature_name in agg_data.columns[:10]:\n        if feature_name[0] != first_letter:\n            continue\n        s = agg_data.loc[:, feature_name]\n        print(feature_name) # for Ctrl + F\n        fig = plt.figure(figsize=(16, 6))\n#         ax_add = fig.add_subplot(2,2,4)\n#         ax_mult = fig.add_subplot(2,2,3) \n        mn = fig.add_subplot(2,2,(1,2)) \n        s.plot(color='green', ax=mn)\n        mn.axvline(end_of_train, color='red', linestyle='--')\n        mn.set_title(feature_name)\n        \n        res_add = seasonal_decompose(s, extrapolate_trend='freq').plot().suptitle('Additive')\n        res_mult = seasonal_decompose(s, extrapolate_trend='freq', model='multiplicative').plot().suptitle('Multiplicative')\n                \n        plt.subplots_adjust(wspace= 0.25, hspace= 0.25)\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:33:35.956587Z","iopub.execute_input":"2022-06-19T09:33:35.956951Z","iopub.status.idle":"2022-06-19T09:33:55.747472Z","shell.execute_reply.started":"2022-06-19T09:33:35.956919Z","shell.execute_reply":"2022-06-19T09:33:55.746414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Stationarity","metadata":{}},{"cell_type":"code","source":"# Test feature\ntest_feature = 'P_2'\n\n# ADF Test\nresult = adfuller(agg_data.loc[:, test_feature], autolag='AIC')\nprint(f'ADF Statistic: {result[0]}')\nprint(f'p-value: {result[1]}')\n\nif result[1] > 0.05:\n    print('Series is not Stationary')\nelse:\n    print('Series is Stationary')\n\n\n# KPSS Test\nstats, p, lags, critical_values = kpss(agg_data.loc[:, test_feature])\nprint(f'KPSS Test Statistics: {stats}')\nprint(f'p-value: {p}')\n\nif p<0.05:\n      print('Series is not Stationary')\nelse:\n      print('Series is Stationary')","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:33:55.749261Z","iopub.execute_input":"2022-06-19T09:33:55.749644Z","iopub.status.idle":"2022-06-19T09:33:55.801119Z","shell.execute_reply.started":"2022-06-19T09:33:55.749612Z","shell.execute_reply":"2022-06-19T09:33:55.800105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the class to be used on all features\nclass Stationarity:\n    '''\n    Performs a stationarity review of each of the columns that are contained within the DataFrame being reviewed.\n    df: transformed macro-economic variables for review\n    '''\n    \n    # Constructor\n    #def __init__(self):\n        \n        \n    # Method - Create UDF for the basic transformations\n    def stationarity_test(self, df) -> pd.DataFrame:\n        # Construct empty DataFrame with required output columns\n        tmp = pd.DataFrame(columns=['variable', 'obs', 'adfstat',\n                                    'adfpvalue', 'kpssstat', 'kpsspvalue',\n                                    'adf_stat', 'kpss_stat'\n                                   ])\n        for col in df.columns:\n            # Keep only the not null values for review - method requires a series input\n            result = adfuller(df.loc[df[col].notnull(),col], autolag='AIC')\n            stat = result[0]\n            pval = result[1]\n\n            # KPSS test. \n            # The option 'ct' for the regression parameter, means that the 'deterministic trend' is reviewed instead of the mean value \n            stats, p, _, _ = kpss(df.loc[df[col].notnull(),col], nlags='auto')\n\n            # Populate the tmp DataFrame\n            tmp = tmp.append({'variable': col\n                              ,'obs': len(df.loc[df[col].notnull(),col])\n                              ,'adfstat': stat\n                              ,'adfpvalue': pval\n                              ,'kpssstat': stats\n                              ,'kpsspvalue': p\n                              ,'adf_stat': str(np.where(pval > 0.05, \n                                                    'Non-Stationary',\n                                                    'Stationary'))\n                              ,'kpss_stat': str(np.where(p < 0.05,\n                                                    'Non-Stationary',\n                                                    'Stationary'))\n                             }\n                            ,ignore_index=True)\n        return tmp","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:33:55.802949Z","iopub.execute_input":"2022-06-19T09:33:55.803606Z","iopub.status.idle":"2022-06-19T09:33:55.820422Z","shell.execute_reply.started":"2022-06-19T09:33:55.803557Z","shell.execute_reply":"2022-06-19T09:33:55.819187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Review the stationarity\nstat = Stationarity()\nstat_review = stat.stationarity_test(agg_data)\nstat_review.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:33:55.821874Z","iopub.execute_input":"2022-06-19T09:33:55.822666Z","iopub.status.idle":"2022-06-19T09:34:06.112190Z","shell.execute_reply.started":"2022-06-19T09:33:55.822611Z","shell.execute_reply":"2022-06-19T09:34:06.111181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Understand the quantum of stationarity features\nstat_review.groupby(['adf_stat','kpss_stat'])['variable'].count()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:34:06.114006Z","iopub.execute_input":"2022-06-19T09:34:06.114731Z","iopub.status.idle":"2022-06-19T09:34:06.128600Z","shell.execute_reply.started":"2022-06-19T09:34:06.114684Z","shell.execute_reply":"2022-06-19T09:34:06.127530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Many of the features are displaying non-stationary characteristics. Will have to review variable transformations to understand if after removing the time series element the features become stationary.","metadata":{}},{"cell_type":"markdown","source":"### Data transformations","metadata":{}},{"cell_type":"code","source":"from scipy.stats import norm\n\nclass Transformations:\n    '''\n    Examples of using the methods\n    ---\n    Applying basic transformations:\n    df_ind1 = df_ind.copy()\n    utl.transformations().basic_transformations(df_ind1, 1)\n    # Drop initial columns after transformation\n    orig_cols = list(df_ind.columns)\n    df_ind1.drop(columns=orig_cols, inplace=True)\n    ---\n    Applying lags to the basic transformations:\n    df_ind2 = df_ind1.copy()\n    utl.transformations().create_lags(df_ind2)\n    # Drop original columns from the final transformed list\n    orig_cols = list(df_ind1.columns)\n    df_ind2.drop(columns=orig_cols, inplace=True)\n    ---\n    Test for missing and/or infinity value's for a variable after the transformations\n    df_miss_inf = utl.transformations().missing_infinity_values(df_ind2)\n    '''  \n        \n    # Method - Create UDF for the basic transformations\n    def basic_transformations(self, df, addTrans=None):\n        \"\"\"\n        Basic Transformations\n        > - _R Raw series\n        > - _Y y/y% changes\n        > - _Q q/q% changes\n        > - _D 1st difference\n        > - _S 4th difference (seasonal difference)\n        Additional Transformations (if required)\n        > - _L logit transformation\n        > - _X 1st difference of logit\n        ***\n        - ToDo\n        > - _M MA smoothing and then y/y% changes\n        > - _G 1st difference of y/y% changes\n        > - _J 1st difference of q/q% changes\n        > - _B value at time t / value at time 2 years previous\n        > - _P probit transformation\n        > - _Z 1st difference of probit (only if probit requested)\n        \"\"\"\n        for col in df.columns:\n            # Raw Series\n            df[col+'_R'] = df[col]\n            # y/y% changes\n            df[col+'_Y'] = df[col].pct_change(4)\n            # q/q% changes\n            df[col+'_Q'] = df[col].pct_change()\n            # 1st difference\n            df[col+'_D'] = df[col].diff()\n            # 4th difference\n            df[col+'_S'] = df[col].diff(4)\n            # Completing the additional transformations\n            if addTrans == 1:\n                #pass\n                # Logit transformations\n                df[col+'_L'] = np.log(df[col] / (1 - df[col]))\n                df[col+'_X'] = df[col+'_L'].diff()\n                # Probit transformations\n                df[col+'_P'] = norm.ppf(df[col])\n                df[col+'_Z'] = df[col+'_P'].diff()\n        return df\n    \n    # Method - Create UDF for the creation of lags\n    def create_lags(self, df, maxLag=4):\n        for col in df.columns:\n            for lag in np.arange(0, maxLag + 1):\n                df[col+'_L'+str(lag)] = df[col].shift(lag)\n        return df\n    \n    # Method - Test for the number of missing values in a column transformation\n    def missing_infinity_values(self, df):\n        df_out = pd.DataFrame(columns=['Column', 'MissingVals', 'InfinityVals'])\n        for col in df.columns:\n            # Check for missing values\n            df_out = df_out.append(\n                {\"Column\" : col,\n                 \"MissingVals\" : df[col].isnull().sum(),\n                 \"InfinityVals\" : np.isinf(df[col]).values.sum()\n                }\n            ,ignore_index=True)\n        return df_out","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:42:06.954869Z","iopub.execute_input":"2022-06-19T09:42:06.955559Z","iopub.status.idle":"2022-06-19T09:42:06.973400Z","shell.execute_reply.started":"2022-06-19T09:42:06.955517Z","shell.execute_reply":"2022-06-19T09:42:06.971789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Applying basic transformations\ntrans = Transformations()\nagg_data1 = agg_data.copy()\ntrans.basic_transformations(agg_data1)\n# Drop initial columns after transformation\norig_cols = list(agg_data.columns)\nagg_data1.drop(columns=orig_cols, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-19T10:10:29.412883Z","iopub.execute_input":"2022-06-19T10:10:29.413366Z","iopub.status.idle":"2022-06-19T10:10:30.287654Z","shell.execute_reply.started":"2022-06-19T10:10:29.413327Z","shell.execute_reply":"2022-06-19T10:10:30.286582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_data1.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T10:10:31.728423Z","iopub.execute_input":"2022-06-19T10:10:31.728842Z","iopub.status.idle":"2022-06-19T10:10:31.757731Z","shell.execute_reply.started":"2022-06-19T10:10:31.728802Z","shell.execute_reply":"2022-06-19T10:10:31.756746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Applying lags to the basic transformations:\nagg_data2 = agg_data1.copy()\ntrans.create_lags(agg_data2)\n# Drop original columns from the final transformed list\norig_cols = list(agg_data1.columns)\nagg_data2.drop(columns=orig_cols, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-19T10:11:45.651692Z","iopub.execute_input":"2022-06-19T10:11:45.652148Z","iopub.status.idle":"2022-06-19T10:11:50.714140Z","shell.execute_reply.started":"2022-06-19T10:11:45.652116Z","shell.execute_reply":"2022-06-19T10:11:50.713092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_data2.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T10:11:58.387268Z","iopub.execute_input":"2022-06-19T10:11:58.387691Z","iopub.status.idle":"2022-06-19T10:11:58.421525Z","shell.execute_reply.started":"2022-06-19T10:11:58.387659Z","shell.execute_reply":"2022-06-19T10:11:58.420063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For each feature there are 25 variations. Selecting first 10\nfeature_list = [col for col in agg_data2.columns[:250]]\nagg_data3 = agg_data2[feature_list]\nagg_data3.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T10:31:38.493333Z","iopub.execute_input":"2022-06-19T10:31:38.493763Z","iopub.status.idle":"2022-06-19T10:31:38.526464Z","shell.execute_reply.started":"2022-06-19T10:31:38.493731Z","shell.execute_reply":"2022-06-19T10:31:38.525638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Perform the stationarity assessment\n# Review the stationarity for first 10 features\nstat_review1 = stat.stationarity_test(agg_data3)\nstat_review1.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T10:31:46.800871Z","iopub.execute_input":"2022-06-19T10:31:46.801966Z","iopub.status.idle":"2022-06-19T10:32:00.726493Z","shell.execute_reply.started":"2022-06-19T10:31:46.801922Z","shell.execute_reply":"2022-06-19T10:32:00.725361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Understand the quantum of stationarity features\nstat_review.groupby(['adf_stat','kpss_stat'])['variable'].count()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T10:32:27.156668Z","iopub.execute_input":"2022-06-19T10:32:27.157746Z","iopub.status.idle":"2022-06-19T10:32:27.169278Z","shell.execute_reply.started":"2022-06-19T10:32:27.157692Z","shell.execute_reply":"2022-06-19T10:32:27.168262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation review\nAiming to understand how the different transformations impact the feature correlations.\n\nTaking the feature 'P_2_R_L0' as the dependent variable to highlight how the code works. In future should use the dependent variable.","metadata":{}},{"cell_type":"code","source":"class Correlation:\n    '''\n    Perform a correlation analysis between the independent and dependent variables.\n    :df_ind = (DataFrame) final list of independent variable transformations\n    :df_dep = (DataFrame) final dependent variable transformation\n    :dep = (str) string value of the dependent variable name\n\n    Application of the code\n    # Run the correlation analysis\n    df_ind_ = df_ind3[df_indep_list] # \n    df_dep_ = df_dep.loc[:,['ODR_X']] # the final dependent variable transformation\n    dep = 'ODR_X'\n    # Run the Correlation class to create the correlation of independent variables with the dependent variable\n    df_corr = stmd.Correlation(df_ind_, df_dep_, dep).main() # stmd represents the package reference alias\n    df_corr # displays a pivot table of the correlation values\n\n    sns.heatmap(df_corr, annot=True); # produces a heatmap of the pivot table    \n    '''\n    \n    # Constructor\n    def __init__(self, df_ind, df_dep, dep):\n        self.df_ind = df_ind\n        self.df_dep = df_dep\n        self.dep = dep\n\n    # Method - create the merged DataFrame of dependent and independent variables\n    def _merge_data(self):\n        df = pd.merge(self.df_dep\n                      ,self.df_ind\n                      ,how=\"left\"\n                      ,left_index=True\n                      ,right_index=True\n                     )\n        return df\n\n    # Method - create the correlation DataFrame\n    def _correlation_test(self, df):\n        # Create correlations\n        df1 = df.corr()[self.dep][:]\n        df1 = df1.to_frame()\n        # Adjust DataFrame\n        df1.reset_index(inplace=True)\n        df1 = df1.rename(columns = {'index':'var_trans', self.dep:'Dep_corr'})\n        return df1\n\n    # Method - add the required group by columns\n    def _groupby(self, df):\n        # Add the required group by columns\n        corr = df.var_trans.str.split('_', expand = True)\n        # Add variable values back to the dataframe\n        df['variable'] = corr[0] + '_' + corr[1]\n        df['trans'] = corr[2]\n        df['lag'] = corr[3]\n        return df\n\n    # Method - create the pivot table summary\n    def _pivot(self, df):\n        # Create a pivot table to display the range of correlation values for each variable by lag and transformation\n        df_pivot = pd.pivot_table(df,\n                                  values=\"Dep_corr\",\n                                  index=[\"variable\", \"lag\"],\n                                  columns=[\"trans\"]\n                                 )\n        return df_pivot\n\n    # Method - run the methods from above\n    def main(self):\n        df = self._merge_data()\n        df1 = self._correlation_test(df)\n        df1 = self._groupby(df1)\n        df1 = self._pivot(df1)\n        return df1\n\n    # Run the process steps\n#     if __name__ == \"__main__\":\n#         main()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T10:36:33.643256Z","iopub.execute_input":"2022-06-19T10:36:33.643675Z","iopub.status.idle":"2022-06-19T10:36:33.657176Z","shell.execute_reply.started":"2022-06-19T10:36:33.643642Z","shell.execute_reply":"2022-06-19T10:36:33.656021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run the correlation analysis\ndep = 'P_2_R_L0'\ndf_ind_ = agg_data3.loc[:,agg_data3.columns.drop(dep)] \ndf_dep_ = agg_data3.loc[:,[dep]]\n\n# Run the Correlation class to create the correlation of independent variables with the dependent variable\ncorr = Correlation(df_ind_, df_dep_, dep)\ndf_corr = corr.main() # stmd represents the package reference alias\ndf_corr # displays a pivot table of the correlation values","metadata":{"execution":{"iopub.status.busy":"2022-06-19T10:45:35.970474Z","iopub.execute_input":"2022-06-19T10:45:35.971136Z","iopub.status.idle":"2022-06-19T10:45:36.191939Z","shell.execute_reply.started":"2022-06-19T10:45:35.971096Z","shell.execute_reply":"2022-06-19T10:45:36.191231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# produces a heatmap of the pivot table\nplt.figure(figsize=(20,15))\nsns.heatmap(df_corr, annot=True);","metadata":{"execution":{"iopub.status.busy":"2022-06-19T10:47:24.585704Z","iopub.execute_input":"2022-06-19T10:47:24.586268Z","iopub.status.idle":"2022-06-19T10:47:26.398899Z","shell.execute_reply.started":"2022-06-19T10:47:24.586235Z","shell.execute_reply":"2022-06-19T10:47:26.397734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Many of this group of features are showing negative correlations with the variable 'P_2_R_L0'. It is interesting to see that the Raw versions are showing some of the biggest correlations. ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}