{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"#===========================\n# Team 4 - Preprocessing   =\n# KIEU HAI DANG - 19127347 =\n# TRAN DONG BA - 19127334  =\n# LE VAN DONG - 19127363   =\n# LA MINH HIEU - 19127400  =\n#===========================\n\n\n#======================================================================\n# Popular libraries used for data preprocessing & visualization\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns # data visualization\nimport matplotlib.pyplot as plt\nimport gc\n#======================================================================\n\n\n#======================================================================\n# Libraries used for data modeling, training & prediction\nfrom sklearn.decomposition import PCA\n#======================================================================\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-22T15:10:10.296706Z","iopub.execute_input":"2022-12-22T15:10:10.297235Z","iopub.status.idle":"2022-12-22T15:10:10.305522Z","shell.execute_reply.started":"2022-12-22T15:10:10.297193Z","shell.execute_reply":"2022-12-22T15:10:10.303925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read data","metadata":{}},{"cell_type":"markdown","source":"At this step, because of the large of the file's size, we can't read them normally.\n\nFortunately, we found & thanks for the solution called parquet file format.\n\nBig thanks to - https://www.kaggle.com/code/odins0n/load-parquet-files-with-low-memory/","metadata":{}},{"cell_type":"code","source":"%%time\ndf_train_data = pd.read_parquet('/kaggle/input/amex-parquet/train_data.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:10:10.308492Z","iopub.execute_input":"2022-12-22T15:10:10.309097Z","iopub.status.idle":"2022-12-22T15:10:28.956961Z","shell.execute_reply.started":"2022-12-22T15:10:10.309047Z","shell.execute_reply":"2022-12-22T15:10:28.955811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing data","metadata":{}},{"cell_type":"markdown","source":"### Remove cagorical columns","metadata":{}},{"cell_type":"code","source":"cate_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:10:28.958790Z","iopub.execute_input":"2022-12-22T15:10:28.959102Z","iopub.status.idle":"2022-12-22T15:10:28.964505Z","shell.execute_reply.started":"2022-12-22T15:10:28.959073Z","shell.execute_reply":"2022-12-22T15:10:28.963161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_data = df_train_data.drop(cate_cols, axis=1)\ndf_features = df_train_data.drop(['S_2'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:10:28.966143Z","iopub.execute_input":"2022-12-22T15:10:28.967144Z","iopub.status.idle":"2022-12-22T15:10:32.969887Z","shell.execute_reply.started":"2022-12-22T15:10:28.967092Z","shell.execute_reply":"2022-12-22T15:10:32.968592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of column which include >= 50% of missing value","metadata":{}},{"cell_type":"code","source":"missing_data = pd.DataFrame(df_train_data.isnull().sum()/len(df_train_data))\nneed_drop = missing_data.loc[missing_data[0] >= 0.5]\nprint('number of column w/ >= 50% missing value = ', len(need_drop))","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:10:32.973971Z","iopub.execute_input":"2022-12-22T15:10:32.974750Z","iopub.status.idle":"2022-12-22T15:10:35.239136Z","shell.execute_reply.started":"2022-12-22T15:10:32.974702Z","shell.execute_reply":"2022-12-22T15:10:35.237921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look at this, I think we should drop the above columns which have high rate of missing values.","metadata":{}},{"cell_type":"code","source":"cols_need_drop = list(need_drop.T.columns)\ndf_features = df_features.drop(cols_need_drop, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:10:35.240277Z","iopub.execute_input":"2022-12-22T15:10:35.240579Z","iopub.status.idle":"2022-12-22T15:10:37.244806Z","shell.execute_reply.started":"2022-12-22T15:10:35.240550Z","shell.execute_reply":"2022-12-22T15:10:37.243143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fill NaN value","metadata":{}},{"cell_type":"code","source":"df_features_ID = df_features['customer_ID']\ndf_features_target = df_features['target']\ndf_features = df_features.drop(['customer_ID', 'target'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:10:37.246643Z","iopub.execute_input":"2022-12-22T15:10:37.247088Z","iopub.status.idle":"2022-12-22T15:10:39.427173Z","shell.execute_reply.started":"2022-12-22T15:10:37.247049Z","shell.execute_reply":"2022-12-22T15:10:39.425460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in df_features.columns:\n    median = df_features[column].median()\n    df_features[column] = df_features[column].fillna(median)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:10:39.429229Z","iopub.execute_input":"2022-12-22T15:10:39.430594Z","iopub.status.idle":"2022-12-22T15:10:58.950768Z","shell.execute_reply.started":"2022-12-22T15:10:39.430530Z","shell.execute_reply":"2022-12-22T15:10:58.949347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PCA applying","metadata":{}},{"cell_type":"code","source":"# Grouping columns by categories\n\nDeliquency_variables = [x for x in set(df_features.columns) if x[0]=='D']\nSpend_variables = [x for x in set(df_features.columns) if x[0]=='S']\nPayment_variables = [x for x in set(df_features.columns) if x[0]=='P']\nBalance_variables = [x for x in set(df_features.columns) if x[0]=='B']\nRisk_variables = [x for x in set(df_features.columns) if x[0]=='R']","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:10:58.952174Z","iopub.execute_input":"2022-12-22T15:10:58.952560Z","iopub.status.idle":"2022-12-22T15:10:58.962840Z","shell.execute_reply.started":"2022-12-22T15:10:58.952526Z","shell.execute_reply":"2022-12-22T15:10:58.961296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Delinquency PCA","metadata":{}},{"cell_type":"code","source":"df_features[Deliquency_variables].shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:10:58.964599Z","iopub.execute_input":"2022-12-22T15:10:58.965406Z","iopub.status.idle":"2022-12-22T15:10:59.897304Z","shell.execute_reply.started":"2022-12-22T15:10:58.965363Z","shell.execute_reply":"2022-12-22T15:10:59.895639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distributing the dataset into two components X and Y\nx = df_features[Deliquency_variables].values\nplt.figure(figsize = (18,10))        \nsns.heatmap(pd.DataFrame(x).corr(),annot = False,cmap=\"Greens\")\n\n\n# Standardizing the Variables\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\n\n# Prinicpal Component Analysis\npcs = PCA(n_components=65)\npcs.fit(x)\n\n# Cumulative Variance chart\nFigure = plt.figure(figsize = (20,6))\nplt.plot(np.cumsum(pcs.explained_variance_ratio_))\nplt.plot(range(65), [0.9] * 65, label = \"threshold\")\nplt.xlabel('Components')\nplt.ylabel('Cumulative Variance')\n\npcsSummary = pd.DataFrame({'Standard deviation': np.sqrt(pcs.explained_variance_),\n'Proportion of variance': pcs.explained_variance_ratio_,\n'Cumulative proportion': np.cumsum(pcs.explained_variance_ratio_)})\npcsSummary = pcsSummary.transpose()\npcsSummary.columns = ['PC{}'.format(i) for i in range(65)]\npcsSummary.round(4)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:10:59.899422Z","iopub.execute_input":"2022-12-22T15:10:59.900149Z","iopub.status.idle":"2022-12-22T15:12:27.106252Z","shell.execute_reply.started":"2022-12-22T15:10:59.900089Z","shell.execute_reply":"2022-12-22T15:12:27.104804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_features = pd.DataFrame(pcs.fit_transform(x))\npca_features.columns = ['Deliquency_{}'.format(i) for i in range(1, len(pcsSummary.columns) + 1)]\npca_features.set_index(df_features.index,inplace=True)\ndeliquency_variables = pca_features[['Deliquency_{}'.format(i) for i in range(1, 39 )]]","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:12:27.108131Z","iopub.execute_input":"2022-12-22T15:12:27.108890Z","iopub.status.idle":"2022-12-22T15:12:45.861310Z","shell.execute_reply.started":"2022-12-22T15:12:27.108846Z","shell.execute_reply":"2022-12-22T15:12:45.859885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Spend PCA","metadata":{}},{"cell_type":"code","source":"df_features[Spend_variables].shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:12:45.862973Z","iopub.execute_input":"2022-12-22T15:12:45.863318Z","iopub.status.idle":"2022-12-22T15:12:46.099775Z","shell.execute_reply.started":"2022-12-22T15:12:45.863287Z","shell.execute_reply":"2022-12-22T15:12:46.098501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## PCA on Spend\n# distributing the dataset into two components X and Y\nx = df_features[Spend_variables].values\nplt.figure(figsize = (18,10))        \nsns.heatmap(pd.DataFrame(x).corr(),annot = False,cmap=\"Greens\")\n\n\n# Standardizing the Variables\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\n\n# Prinicpal Component Analysis\npcs = PCA(n_components=20)\npcs.fit(x)\n\npcsSummary = pd.DataFrame({'Standard deviation': np.sqrt(pcs.explained_variance_),\n'Proportion of variance': pcs.explained_variance_ratio_,\n'Cumulative proportion': np.cumsum(pcs.explained_variance_ratio_)})\n\n## Cumulative Variance chart\nFigure = plt.figure(figsize = (20,6))\nplt.plot(np.cumsum(pcs.explained_variance_ratio_))\nplt.plot(list(range(20)), [0.9] * 20, label = \"threshold\")\nplt.xlabel('Components')\nplt.ylabel('Cumulative Variance')\n\npcsSummary = pcsSummary.transpose()\npcsSummary.columns = ['PC{}'.format(i) for i in range(20)]\npcsSummary.round(4)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:12:46.104168Z","iopub.execute_input":"2022-12-22T15:12:46.104548Z","iopub.status.idle":"2022-12-22T15:12:57.590063Z","shell.execute_reply.started":"2022-12-22T15:12:46.104513Z","shell.execute_reply":"2022-12-22T15:12:57.588501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_features = pd.DataFrame(pcs.fit_transform(x))\npca_features.columns = ['Spend_{}'.format(i) for i in range(1, len(pcsSummary.columns) + 1)]\npca_features.set_index(df_features.index,inplace=True)\nspend_variables = pca_features[['Spend_{}'.format(i) for i in range(1, 8)]]","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:12:57.592330Z","iopub.execute_input":"2022-12-22T15:12:57.592975Z","iopub.status.idle":"2022-12-22T15:13:01.358296Z","shell.execute_reply.started":"2022-12-22T15:12:57.592908Z","shell.execute_reply":"2022-12-22T15:13:01.357018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Payment PCA","metadata":{}},{"cell_type":"code","source":"df_features[Payment_variables].shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:01.359950Z","iopub.execute_input":"2022-12-22T15:13:01.360292Z","iopub.status.idle":"2022-12-22T15:13:01.401155Z","shell.execute_reply.started":"2022-12-22T15:13:01.360260Z","shell.execute_reply":"2022-12-22T15:13:01.399920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distributing the dataset into two components X and Y\nx = df_features[Payment_variables].values\nplt.figure(figsize = (18,10))        \nsns.heatmap(pd.DataFrame(x).corr(),annot = False,cmap=\"Greens\")\n\n\n# Standardizing the Variables\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\n\n# Prinicpal Component Analysis\npcs = PCA(n_components=3)\npcs.fit(x)\n\npcsSummary = pd.DataFrame({'Standard deviation': np.sqrt(pcs.explained_variance_),\n'Proportion of variance': pcs.explained_variance_ratio_,\n'Cumulative proportion': np.cumsum(pcs.explained_variance_ratio_)})\n\n## Cumulative Variance chart\nFigure = plt.figure(figsize = (20,6))\nplt.plot(np.cumsum(pcs.explained_variance_ratio_))\nplt.plot(list(range(0,3 )), [0.9] * 3, label = \"threshold\")\nplt.xlabel('Components')\nplt.ylabel('Cumulative Variance')\n\npcsSummary = pcsSummary.transpose()\npcsSummary.columns = ['PC{}'.format(i) for i in range(3)]\npcsSummary.round(4)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:01.402433Z","iopub.execute_input":"2022-12-22T15:13:01.403282Z","iopub.status.idle":"2022-12-22T15:13:02.682087Z","shell.execute_reply.started":"2022-12-22T15:13:01.403247Z","shell.execute_reply":"2022-12-22T15:13:02.680744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_features = pd.DataFrame(pcs.fit_transform(x))\npca_features.columns = ['Payment_{}'.format(i) for i in range(1, len(pcsSummary.columns) + 1)]\npca_features.set_index(df_features.index,inplace=True)\npayment_variables = pca_features[['Payment_{}'.format(i) for i in range(1, 4 )]]","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:02.683694Z","iopub.execute_input":"2022-12-22T15:13:02.684511Z","iopub.status.idle":"2022-12-22T15:13:03.094896Z","shell.execute_reply.started":"2022-12-22T15:13:02.684451Z","shell.execute_reply":"2022-12-22T15:13:03.093823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Balance PCA","metadata":{}},{"cell_type":"code","source":"df_features[Balance_variables].shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:03.096187Z","iopub.execute_input":"2022-12-22T15:13:03.096603Z","iopub.status.idle":"2022-12-22T15:13:03.482474Z","shell.execute_reply.started":"2022-12-22T15:13:03.096562Z","shell.execute_reply":"2022-12-22T15:13:03.481223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distributing the dataset into two components X and Y\nx = df_features[Balance_variables].values\nplt.figure(figsize = (18,10))        \nsns.heatmap(pd.DataFrame(x).corr(),annot = False,cmap=\"Greens\")\n\n# Standardizing the Variables\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\n\n# Prinicpal Component Analysis\npcs = PCA(n_components=34)\npcs.fit(x)\n\npcsSummary = pd.DataFrame({'Standard deviation': np.sqrt(pcs.explained_variance_),\n'Proportion of variance': pcs.explained_variance_ratio_,\n'Cumulative proportion': np.cumsum(pcs.explained_variance_ratio_)})\n\n## Cumulative Variance chart\nFigure = plt.figure(figsize = (20,6))\nplt.plot(np.cumsum(pcs.explained_variance_ratio_))\nplt.plot(list(range(34)), [0.9] * 34, label = \"threshold\")\nplt.xlabel('Components')\nplt.ylabel('Cumulative Variance')\n\npcsSummary = pcsSummary.transpose()\npcsSummary.columns = ['PC{}'.format(i) for i in range(len(pcsSummary.columns))]\npcsSummary.round(4)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:03.484405Z","iopub.execute_input":"2022-12-22T15:13:03.484880Z","iopub.status.idle":"2022-12-22T15:13:36.151480Z","shell.execute_reply.started":"2022-12-22T15:13:03.484833Z","shell.execute_reply":"2022-12-22T15:13:36.150459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_features = pd.DataFrame(pcs.fit_transform(x))\npca_features.columns = ['Balance_{}'.format(i) for i in range(1, len(pcsSummary.columns) + 1)]\npca_features.set_index(df_features.index,inplace=True)\nbalance_variables = pca_features[['Balance_{}'.format(i) for i in range(1, 11)]]","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:36.152877Z","iopub.execute_input":"2022-12-22T15:13:36.153390Z","iopub.status.idle":"2022-12-22T15:13:48.123914Z","shell.execute_reply.started":"2022-12-22T15:13:36.153358Z","shell.execute_reply":"2022-12-22T15:13:48.122962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Risk PCA","metadata":{}},{"cell_type":"code","source":"df_features[Risk_variables].shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:48.125516Z","iopub.execute_input":"2022-12-22T15:13:48.126292Z","iopub.status.idle":"2022-12-22T15:13:48.395917Z","shell.execute_reply.started":"2022-12-22T15:13:48.126249Z","shell.execute_reply":"2022-12-22T15:13:48.394778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distributing the dataset into two components X and Y\nx = df_features[Risk_variables].values\nplt.figure(figsize = (18,10))        \nsns.heatmap(pd.DataFrame(x).corr(),annot = False,cmap=\"Greens\")\n\n\n# Standardizing the Variables\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\n\n# Prinicpal Component Analysis\npcs = PCA(n_components=26)\npcs.fit(x)\n\npcsSummary = pd.DataFrame({'Standard deviation': np.sqrt(pcs.explained_variance_),\n'Proportion of variance': pcs.explained_variance_ratio_,\n'Cumulative proportion': np.cumsum(pcs.explained_variance_ratio_)})\n\n# Cumulative Variance chart\nFigure = plt.figure(figsize = (20,6))\nplt.plot(np.cumsum(pcs.explained_variance_ratio_))\nplt.plot(list(range(26)), [0.9] * 26, label = \"threshold\")\nplt.xlabel('Components')\nplt.ylabel('Cumulative Variance')\n\npcsSummary = pcsSummary.transpose()\npcsSummary.columns = ['PC{}'.format(i) for i in range(26)]\npcsSummary.round(4)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:13:48.398817Z","iopub.execute_input":"2022-12-22T15:13:48.399607Z","iopub.status.idle":"2022-12-22T15:14:05.356422Z","shell.execute_reply.started":"2022-12-22T15:13:48.399547Z","shell.execute_reply":"2022-12-22T15:14:05.355271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_features = pd.DataFrame(pcs.fit_transform(x))\npca_features.columns = ['Risk_{}'.format(i) for i in range(1, len(pcsSummary.columns) + 1)]\npca_features.set_index(df_features.index,inplace=True)\nrisk_variables = pca_features[['Risk_{}'.format(i) for i in range(1, 15)]]","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:14:05.358040Z","iopub.execute_input":"2022-12-22T15:14:05.358853Z","iopub.status.idle":"2022-12-22T15:14:09.997127Z","shell.execute_reply.started":"2022-12-22T15:14:05.358813Z","shell.execute_reply":"2022-12-22T15:14:09.996149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final Data after dimension reduced","metadata":{}},{"cell_type":"code","source":"df_features_processed = pd.concat([deliquency_variables, spend_variables, payment_variables, balance_variables, risk_variables], axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:14:09.998373Z","iopub.execute_input":"2022-12-22T15:14:09.998715Z","iopub.status.idle":"2022-12-22T15:14:10.802499Z","shell.execute_reply.started":"2022-12-22T15:14:09.998685Z","shell.execute_reply":"2022-12-22T15:14:10.801509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_features_processed = pd.concat([df_features_processed, df_features_target], axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:14:10.803632Z","iopub.execute_input":"2022-12-22T15:14:10.803964Z","iopub.status.idle":"2022-12-22T15:14:11.585859Z","shell.execute_reply.started":"2022-12-22T15:14:10.803935Z","shell.execute_reply":"2022-12-22T15:14:11.584875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ratio of 0 / 1 value in target","metadata":{}},{"cell_type":"code","source":"draw_chart = pd.DataFrame(df_features_processed['target'].value_counts()).T\nprint('percentage of target value 0 / 1 = ', draw_chart[1]/draw_chart[0], '\\n\\n\\n');\ndraw_chart.plot.barh(align='edge', width=0.5);","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:14:11.587159Z","iopub.execute_input":"2022-12-22T15:14:11.587465Z","iopub.status.idle":"2022-12-22T15:14:11.862124Z","shell.execute_reply.started":"2022-12-22T15:14:11.587437Z","shell.execute_reply":"2022-12-22T15:14:11.860961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look! it seem we have mild imbalance here. Do not need to process.","metadata":{}},{"cell_type":"code","source":"df_features_processed","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:14:11.863606Z","iopub.execute_input":"2022-12-22T15:14:11.864375Z","iopub.status.idle":"2022-12-22T15:14:12.701501Z","shell.execute_reply.started":"2022-12-22T15:14:11.864340Z","shell.execute_reply":"2022-12-22T15:14:12.700234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save data to ouput serves for training process","metadata":{}},{"cell_type":"code","source":"df_features_processed.to_parquet('features.parquet.gzip',\n              compression='gzip', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:14:12.703370Z","iopub.execute_input":"2022-12-22T15:14:12.703797Z","iopub.status.idle":"2022-12-22T15:16:31.966359Z","shell.execute_reply.started":"2022-12-22T15:14:12.703760Z","shell.execute_reply":"2022-12-22T15:16:31.965094Z"},"trusted":true},"execution_count":null,"outputs":[]}]}