{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook I want to test if features are stationary over time.\n\nLooks like vast amount of features have seasonal patterns - it may affect the model in many aspects: such as data drift and bad performance on unseen data. Also even simple feature engineering may be less accurate - if feature has stable positive trend, then max value will be closer to the last payment.\nWe may consider normalize data on daily level to avoid those mistakes. It also may open additional dimension for feature extraction: for example we may use seasonality or trend features as additional features.\n\nI will calculate an average value for each feature for each day resulting a univariate time series for each feature. I will plot them only if the max autocorrelated component is over a threshold (0.6).","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nimport matplotlib.pyplot as plt\n\nfrom statsmodels.tsa.stattools import acf, pacf\nfrom statsmodels.graphics.tsaplots import plot_acf, plot_pacf\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-16T15:36:20.924571Z","iopub.execute_input":"2022-06-16T15:36:20.925423Z","iopub.status.idle":"2022-06-16T15:36:21.548023Z","shell.execute_reply.started":"2022-06-16T15:36:20.925348Z","shell.execute_reply":"2022-06-16T15:36:21.546931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_agg = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet').assign(S_2=lambda dx: pd.to_datetime(dx.S_2)).groupby('S_2').mean()\nend_of_train = pd.to_datetime(train_agg.index).max()","metadata":{"execution":{"iopub.status.busy":"2022-06-16T15:36:21.553011Z","iopub.execute_input":"2022-06-16T15:36:21.553322Z","iopub.status.idle":"2022-06-16T15:37:05.760092Z","shell.execute_reply.started":"2022-06-16T15:36:21.553293Z","shell.execute_reply":"2022-06-16T15:37:05.758232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_agg = []\nfor cols2use in train_agg.columns.values.reshape(-1, 47):\n    test_agg.append(pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet', columns=cols2use.tolist() + ['S_2']).assign(S_2=lambda dx: pd.to_datetime(dx.S_2)).groupby('S_2').mean())\n\n\ntest_agg = pd.concat(test_agg, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-16T15:37:05.762521Z","iopub.execute_input":"2022-06-16T15:37:05.762940Z","iopub.status.idle":"2022-06-16T15:38:23.807611Z","shell.execute_reply.started":"2022-06-16T15:37:05.762904Z","shell.execute_reply":"2022-06-16T15:38:23.806271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_data = pd.concat([train_agg, test_agg])","metadata":{"execution":{"iopub.status.busy":"2022-06-16T15:38:23.810320Z","iopub.execute_input":"2022-06-16T15:38:23.810744Z","iopub.status.idle":"2022-06-16T15:38:23.833818Z","shell.execute_reply.started":"2022-06-16T15:38:23.810707Z","shell.execute_reply":"2022-06-16T15:38:23.832674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for first_letter in list(set([col.split('_')[0] for col in agg_data.columns])):\n    break","metadata":{"execution":{"iopub.status.busy":"2022-06-16T15:45:16.401872Z","iopub.execute_input":"2022-06-16T15:45:16.402406Z","iopub.status.idle":"2022-06-16T15:45:16.410779Z","shell.execute_reply.started":"2022-06-16T15:45:16.402370Z","shell.execute_reply":"2022-06-16T15:45:16.409974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for first_letter in list(set([col.split('_')[0] for col in agg_data.columns])):\n    for feature_name in agg_data.columns:\n        if feature_name[0] != first_letter:\n            continue\n        s = agg_data.loc[:, feature_name]\n        max_acf = np.abs(acf(s, nlags=agg_data.index.size - 1))[1:].max()\n        if max_acf > 0.6:\n            print(feature_name) # for Ctrl + F\n            fig = plt.figure(figsize=(16, 6))\n            sub_pacf = fig.add_subplot(2,2,4)\n            sub_acf = fig.add_subplot(2,2,3) \n            mn = fig.add_subplot(2,2,(1,2)) \n            plot_pacf(s, lags=agg_data.index.size/2-1, ax=sub_acf)\n            plot_acf(s, lags=agg_data.index.size-1, ax=sub_pacf)\n            s.plot(color='green', ax=mn)\n            mn.axvline(end_of_train, color='red', linestyle='--')\n            mn.set_title(feature_name)\n            plt.subplots_adjust(wspace= 0.25, hspace= 0.25)\n            plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-16T15:47:46.667113Z","iopub.execute_input":"2022-06-16T15:47:46.667668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}