{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"#===========================\n# Team 4 - Preprocessing   =\n# KIEU HAI DANG - 19127347 =\n# TRAN DONG BA - 19127334  =\n# LE VAN DONG - 19127363   =\n# LA MINH HIEU - 19127400  =\n#===========================\n\n\n#======================================================================\n# Popular libraries used for data preprocessing & visualization\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns # data visualization\nimport matplotlib.pyplot as plt\nimport gc\n#======================================================================\n\n\n#======================================================================\n# Libraries used for data modeling, training & prediction\nfrom sklearn.decomposition import PCA\n#======================================================================\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-22T15:11:26.224338Z","iopub.execute_input":"2022-12-22T15:11:26.224806Z","iopub.status.idle":"2022-12-22T15:11:27.021525Z","shell.execute_reply.started":"2022-12-22T15:11:26.224691Z","shell.execute_reply":"2022-12-22T15:11:27.020270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read data","metadata":{}},{"cell_type":"markdown","source":"At this step, because of the large of the file's size, we can't read them normally.\n\nFortunately, we found & thanks for the solution called parquet file format.\n\nBig thanks to - https://www.kaggle.com/code/odins0n/load-parquet-files-with-low-memory/","metadata":{}},{"cell_type":"code","source":"%%time\ndf_test_data = pd.read_parquet('/kaggle/input/amex-parquet/test_data.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:11:27.023669Z","iopub.execute_input":"2022-12-22T15:11:27.024359Z","iopub.status.idle":"2022-12-22T15:13:00.714356Z","shell.execute_reply.started":"2022-12-22T15:11:27.024317Z","shell.execute_reply":"2022-12-22T15:13:00.710670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing data","metadata":{}},{"cell_type":"markdown","source":"### Remove cagorical columns","metadata":{}},{"cell_type":"code","source":"cate_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:00.717341Z","iopub.execute_input":"2022-12-22T15:13:00.718242Z","iopub.status.idle":"2022-12-22T15:13:00.730734Z","shell.execute_reply.started":"2022-12-22T15:13:00.718129Z","shell.execute_reply":"2022-12-22T15:13:00.728530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_data = df_test_data.drop(cate_cols, axis=1)\ndf_features = df_test_data.drop('S_2', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:00.736514Z","iopub.execute_input":"2022-12-22T15:13:00.737468Z","iopub.status.idle":"2022-12-22T15:13:25.664953Z","shell.execute_reply.started":"2022-12-22T15:13:00.737431Z","shell.execute_reply":"2022-12-22T15:13:25.663737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_test_data;  gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:25.666432Z","iopub.execute_input":"2022-12-22T15:13:25.666862Z","iopub.status.idle":"2022-12-22T15:13:25.830933Z","shell.execute_reply.started":"2022-12-22T15:13:25.666826Z","shell.execute_reply":"2022-12-22T15:13:25.830095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of column which include >= 50% of missing value","metadata":{}},{"cell_type":"code","source":"missing_data = pd.DataFrame(df_features.isnull().sum()/len(df_features))\nneed_drop = missing_data.loc[missing_data[0] >= 0.5]\nprint('number of column w/ >= 50% missing value = ', len(need_drop))","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:25.832121Z","iopub.execute_input":"2022-12-22T15:13:25.832634Z","iopub.status.idle":"2022-12-22T15:13:29.357307Z","shell.execute_reply.started":"2022-12-22T15:13:25.832602Z","shell.execute_reply":"2022-12-22T15:13:29.356209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look at this, I think we should drop the above columns which have high rate of missing values.","metadata":{}},{"cell_type":"code","source":"df_features = df_features.drop(list(need_drop.T.columns), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:29.358530Z","iopub.execute_input":"2022-12-22T15:13:29.358886Z","iopub.status.idle":"2022-12-22T15:13:32.217964Z","shell.execute_reply.started":"2022-12-22T15:13:29.358847Z","shell.execute_reply":"2022-12-22T15:13:32.216757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fill NaN value","metadata":{}},{"cell_type":"code","source":"df_features_ID = df_features['customer_ID']\ndf_features = df_features.drop(['customer_ID'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:32.219317Z","iopub.execute_input":"2022-12-22T15:13:32.219671Z","iopub.status.idle":"2022-12-22T15:13:34.914261Z","shell.execute_reply.started":"2022-12-22T15:13:32.219638Z","shell.execute_reply":"2022-12-22T15:13:34.913041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in df_features.columns:\n    median = df_features[column].median()\n    df_features[column] = df_features[column].fillna(median)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:34.915861Z","iopub.execute_input":"2022-12-22T15:13:34.916305Z","iopub.status.idle":"2022-12-22T15:14:12.746335Z","shell.execute_reply.started":"2022-12-22T15:13:34.916261Z","shell.execute_reply":"2022-12-22T15:14:12.745379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PCA applying","metadata":{}},{"cell_type":"code","source":"# Grouping columns by categories\n\nDeliquency_variables = [x for x in set(df_features.columns) if x[0]=='D']\nSpend_variables = [x for x in set(df_features.columns) if x[0]=='S']\nPayment_variables = [x for x in set(df_features.columns) if x[0]=='P']\nBalance_variables = [x for x in set(df_features.columns) if x[0]=='B']\nRisk_variables = [x for x in set(df_features.columns) if x[0]=='R']","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:14:12.749544Z","iopub.execute_input":"2022-12-22T15:14:12.750320Z","iopub.status.idle":"2022-12-22T15:14:12.756848Z","shell.execute_reply.started":"2022-12-22T15:14:12.750284Z","shell.execute_reply":"2022-12-22T15:14:12.755955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Delinquency PCA","metadata":{}},{"cell_type":"code","source":"df_features[Deliquency_variables].shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:14:12.758391Z","iopub.execute_input":"2022-12-22T15:14:12.758981Z","iopub.status.idle":"2022-12-22T15:14:14.017669Z","shell.execute_reply.started":"2022-12-22T15:14:12.758938Z","shell.execute_reply":"2022-12-22T15:14:14.016529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distributing the dataset into two components X and Y\nx = df_features[Deliquency_variables].values\nplt.figure(figsize = (18,10))        \n# sns.heatmap(pd.DataFrame(x).corr(),annot = False,cmap=\"Greens\")\n\n\n# Standardizing the Variables\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\n\n# Prinicpal Component Analysis\npcs = PCA(n_components=65)\npcs.fit(x)\n\n# Cumulative Variance chart\nFigure = plt.figure(figsize = (20,6))\nplt.plot(np.cumsum(pcs.explained_variance_ratio_))\nplt.plot(range(65), [0.9] * 65, label = \"threshold\")\nplt.xlabel('Components')\nplt.ylabel('Cumulative Variance')\n\npcsSummary = pd.DataFrame({'Standard deviation': np.sqrt(pcs.explained_variance_),\n'Proportion of variance': pcs.explained_variance_ratio_,\n'Cumulative proportion': np.cumsum(pcs.explained_variance_ratio_)})\npcsSummary = pcsSummary.transpose()\npcsSummary.columns = ['PC{}'.format(i) for i in range(65)]\npcsSummary.round(4)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:14:14.019164Z","iopub.execute_input":"2022-12-22T15:14:14.019608Z","iopub.status.idle":"2022-12-22T15:14:51.465697Z","shell.execute_reply.started":"2022-12-22T15:14:14.019574Z","shell.execute_reply":"2022-12-22T15:14:51.464442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_features = pd.DataFrame(pcs.fit_transform(x))\npca_features.columns = ['Deliquency_{}'.format(i) for i in range(1, len(pcsSummary.columns) + 1)]\npca_features.set_index(df_features.index,inplace=True)\ndeliquency_variables = pca_features[['Deliquency_{}'.format(i) for i in range(1, 39 )]]","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:14:51.467186Z","iopub.execute_input":"2022-12-22T15:14:51.467575Z","iopub.status.idle":"2022-12-22T15:15:26.345344Z","shell.execute_reply.started":"2022-12-22T15:14:51.467542Z","shell.execute_reply":"2022-12-22T15:15:26.344350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Spend PCA","metadata":{}},{"cell_type":"code","source":"df_features[Spend_variables].shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:15:26.346935Z","iopub.execute_input":"2022-12-22T15:15:26.347687Z","iopub.status.idle":"2022-12-22T15:15:26.776447Z","shell.execute_reply.started":"2022-12-22T15:15:26.347652Z","shell.execute_reply":"2022-12-22T15:15:26.774998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## PCA on Spend\n# distributing the dataset into two components X and Y\nx = df_features[Spend_variables].values\nplt.figure(figsize = (18,10))        \n# sns.heatmap(pd.DataFrame(x).corr(),annot = False,cmap=\"Greens\")\n\n\n# Standardizing the Variables\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\n\n# Prinicpal Component Analysis\npcs = PCA(n_components=21)\npcs.fit(x)\n\npcsSummary = pd.DataFrame({'Standard deviation': np.sqrt(pcs.explained_variance_),\n'Proportion of variance': pcs.explained_variance_ratio_,\n'Cumulative proportion': np.cumsum(pcs.explained_variance_ratio_)})\n\n## Cumulative Variance chart\nFigure = plt.figure(figsize = (20,6))\nplt.plot(np.cumsum(pcs.explained_variance_ratio_))\nplt.plot(list(range(21)), [0.9] * 21, label = \"threshold\")\nplt.xlabel('Components')\nplt.ylabel('Cumulative Variance')\n\npcsSummary = pcsSummary.transpose()\npcsSummary.columns = ['PC{}'.format(i) for i in range(21)]\npcsSummary.round(4)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:15:26.777972Z","iopub.execute_input":"2022-12-22T15:15:26.778297Z","iopub.status.idle":"2022-12-22T15:15:34.306968Z","shell.execute_reply.started":"2022-12-22T15:15:26.778267Z","shell.execute_reply":"2022-12-22T15:15:34.305801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_features = pd.DataFrame(pcs.fit_transform(x))\npca_features.columns = ['Spend_{}'.format(i) for i in range(1, len(pcsSummary.columns) + 1)]\npca_features.set_index(df_features.index,inplace=True)\nspend_variables = pca_features[['Spend_{}'.format(i) for i in range(1, 8)]]","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:15:34.308731Z","iopub.execute_input":"2022-12-22T15:15:34.309452Z","iopub.status.idle":"2022-12-22T15:15:41.386937Z","shell.execute_reply.started":"2022-12-22T15:15:34.309405Z","shell.execute_reply":"2022-12-22T15:15:41.385725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Payment PCA","metadata":{}},{"cell_type":"code","source":"df_features[Payment_variables].shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:15:41.388353Z","iopub.execute_input":"2022-12-22T15:15:41.388688Z","iopub.status.idle":"2022-12-22T15:15:41.462889Z","shell.execute_reply.started":"2022-12-22T15:15:41.388657Z","shell.execute_reply":"2022-12-22T15:15:41.461639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distributing the dataset into two components X and Y\nx = df_features[Payment_variables].values\nplt.figure(figsize = (18,10))        \n# sns.heatmap(pd.DataFrame(x).corr(),annot = False,cmap=\"Greens\")\n\n\n# Standardizing the Variables\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\n\n# Prinicpal Component Analysis\npcs = PCA(n_components=3)\npcs.fit(x)\n\npcsSummary = pd.DataFrame({'Standard deviation': np.sqrt(pcs.explained_variance_),\n'Proportion of variance': pcs.explained_variance_ratio_,\n'Cumulative proportion': np.cumsum(pcs.explained_variance_ratio_)})\n\n## Cumulative Variance chart\nFigure = plt.figure(figsize = (20,6))\nplt.plot(np.cumsum(pcs.explained_variance_ratio_))\nplt.plot(list(range(0,3 )), [0.9] * 3, label = \"threshold\")\nplt.xlabel('Components')\nplt.ylabel('Cumulative Variance')\n\npcsSummary = pcsSummary.transpose()\npcsSummary.columns = ['PC{}'.format(i) for i in range(3)]\npcsSummary.round(4)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:15:41.464303Z","iopub.execute_input":"2022-12-22T15:15:41.464710Z","iopub.status.idle":"2022-12-22T15:15:42.495166Z","shell.execute_reply.started":"2022-12-22T15:15:41.464677Z","shell.execute_reply":"2022-12-22T15:15:42.494076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_features = pd.DataFrame(pcs.fit_transform(x))\npca_features.columns = ['Payment_{}'.format(i) for i in range(1, len(pcsSummary.columns) + 1)]\npca_features.set_index(df_features.index,inplace=True)\npayment_variables = pca_features[['Payment_{}'.format(i) for i in range(1, 4 )]]","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:15:42.496444Z","iopub.execute_input":"2022-12-22T15:15:42.496867Z","iopub.status.idle":"2022-12-22T15:15:43.265479Z","shell.execute_reply.started":"2022-12-22T15:15:42.496836Z","shell.execute_reply":"2022-12-22T15:15:43.264522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Balance PCA","metadata":{}},{"cell_type":"code","source":"df_features[Balance_variables].shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:15:43.266851Z","iopub.execute_input":"2022-12-22T15:15:43.267159Z","iopub.status.idle":"2022-12-22T15:15:44.012171Z","shell.execute_reply.started":"2022-12-22T15:15:43.267131Z","shell.execute_reply":"2022-12-22T15:15:44.011259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distributing the dataset into two components X and Y\nx = df_features[Balance_variables].values\nplt.figure(figsize = (18,10))        \n# sns.heatmap(pd.DataFrame(x).corr(),annot = False,cmap=\"Greens\")\n\n# Standardizing the Variables\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\n\n# Prinicpal Component Analysis\npcs = PCA(n_components=34)\npcs.fit(x)\n\npcsSummary = pd.DataFrame({'Standard deviation': np.sqrt(pcs.explained_variance_),\n'Proportion of variance': pcs.explained_variance_ratio_,\n'Cumulative proportion': np.cumsum(pcs.explained_variance_ratio_)})\n\n## Cumulative Variance chart\nFigure = plt.figure(figsize = (20,6))\nplt.plot(np.cumsum(pcs.explained_variance_ratio_))\nplt.plot(list(range(34)), [0.9] * 34, label = \"threshold\")\nplt.xlabel('Components')\nplt.ylabel('Cumulative Variance')\n\npcsSummary = pcsSummary.transpose()\npcsSummary.columns = ['PC{}'.format(i) for i in range(len(pcsSummary.columns))]\npcsSummary.round(4)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:15:44.013395Z","iopub.execute_input":"2022-12-22T15:15:44.013731Z","iopub.status.idle":"2022-12-22T15:16:12.082585Z","shell.execute_reply.started":"2022-12-22T15:15:44.013691Z","shell.execute_reply":"2022-12-22T15:16:12.081468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_features = pd.DataFrame(pcs.fit_transform(x))\npca_features.columns = ['Balance_{}'.format(i) for i in range(1, len(pcsSummary.columns) + 1)]\npca_features.set_index(df_features.index,inplace=True)\nbalance_variables = pca_features[['Balance_{}'.format(i) for i in range(1, 11)]]","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:16:12.083971Z","iopub.execute_input":"2022-12-22T15:16:12.084262Z","iopub.status.idle":"2022-12-22T15:16:36.486894Z","shell.execute_reply.started":"2022-12-22T15:16:12.084235Z","shell.execute_reply":"2022-12-22T15:16:36.485684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Risk PCA","metadata":{}},{"cell_type":"code","source":"df_features[Risk_variables].shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:16:36.488687Z","iopub.execute_input":"2022-12-22T15:16:36.489177Z","iopub.status.idle":"2022-12-22T15:16:37.006645Z","shell.execute_reply.started":"2022-12-22T15:16:36.489134Z","shell.execute_reply":"2022-12-22T15:16:37.005923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distributing the dataset into two components X and Y\nx = df_features[Risk_variables].values\nplt.figure(figsize = (18,10))        \n# sns.heatmap(pd.DataFrame(x).corr(),annot = False,cmap=\"Greens\")\n\n\n# Standardizing the Variables\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\n\n# Prinicpal Component Analysis\npcs = PCA(n_components=26)\npcs.fit(x)\n\npcsSummary = pd.DataFrame({'Standard deviation': np.sqrt(pcs.explained_variance_),\n'Proportion of variance': pcs.explained_variance_ratio_,\n'Cumulative proportion': np.cumsum(pcs.explained_variance_ratio_)})\n\n# Cumulative Variance chart\nFigure = plt.figure(figsize = (20,6))\nplt.plot(np.cumsum(pcs.explained_variance_ratio_))\nplt.plot(list(range(26)), [0.9] * 26, label = \"threshold\")\nplt.xlabel('Components')\nplt.ylabel('Cumulative Variance')\n\npcsSummary = pcsSummary.transpose()\npcsSummary.columns = ['PC{}'.format(i) for i in range(26)]\npcsSummary.round(4)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:16:37.008132Z","iopub.execute_input":"2022-12-22T15:16:37.008734Z","iopub.status.idle":"2022-12-22T15:16:46.496172Z","shell.execute_reply.started":"2022-12-22T15:16:37.008692Z","shell.execute_reply":"2022-12-22T15:16:46.495482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_features = pd.DataFrame(pcs.fit_transform(x))\npca_features.columns = ['Risk_{}'.format(i) for i in range(1, len(pcsSummary.columns) + 1)]\npca_features.set_index(df_features.index,inplace=True)\nrisk_variables = pca_features[['Risk_{}'.format(i) for i in range(1, 15)]]","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:16:46.497439Z","iopub.execute_input":"2022-12-22T15:16:46.498010Z","iopub.status.idle":"2022-12-22T15:16:55.633976Z","shell.execute_reply.started":"2022-12-22T15:16:46.497960Z","shell.execute_reply":"2022-12-22T15:16:55.632778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final Data after dimension reduced","metadata":{}},{"cell_type":"code","source":"df_features_processed = pd.concat([df_features_ID, deliquency_variables, spend_variables, payment_variables, balance_variables, risk_variables], axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:16:55.635486Z","iopub.execute_input":"2022-12-22T15:16:55.636646Z","iopub.status.idle":"2022-12-22T15:16:57.313471Z","shell.execute_reply.started":"2022-12-22T15:16:55.636599Z","shell.execute_reply":"2022-12-22T15:16:57.312333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del deliquency_variables, spend_variables, payment_variables, balance_variables, risk_variables\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:16:57.314912Z","iopub.execute_input":"2022-12-22T15:16:57.315236Z","iopub.status.idle":"2022-12-22T15:16:57.463614Z","shell.execute_reply.started":"2022-12-22T15:16:57.315205Z","shell.execute_reply":"2022-12-22T15:16:57.462508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_features_processed = df_features_processed.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:16:57.469419Z","iopub.execute_input":"2022-12-22T15:16:57.469776Z","iopub.status.idle":"2022-12-22T15:17:02.542217Z","shell.execute_reply.started":"2022-12-22T15:16:57.469749Z","shell.execute_reply":"2022-12-22T15:17:02.541241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save data to ouput serves for training process","metadata":{}},{"cell_type":"code","source":"df_features_processed.to_parquet('test_features.parquet.gzip',\n              compression='gzip', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:17:02.543502Z","iopub.execute_input":"2022-12-22T15:17:02.543831Z","iopub.status.idle":"2022-12-22T15:21:33.339538Z","shell.execute_reply.started":"2022-12-22T15:17:02.543803Z","shell.execute_reply":"2022-12-22T15:21:33.338203Z"},"trusted":true},"execution_count":null,"outputs":[]}]}