{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7600559,"sourceType":"datasetVersion","datasetId":4424545}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# <div style=\"padding: 25px;color:white;margin:10;font-size:60%;text-align:left;display:fill;border-radius:10px;background-color:#FFFFFF;overflow:hidden;background-color:#A51C30\"><b><span style='color:#FFFFFF'></span></b> <b>Feature Group</b></div>","metadata":{}},{"cell_type":"code","source":"import gc\nimport ctypes \nimport pickle\nimport datetime\n\nimport os\nimport pandas as pd\nimport polars as pl\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:16:39.555573Z","iopub.execute_input":"2024-03-15T07:16:39.556805Z","iopub.status.idle":"2024-03-15T07:16:41.187147Z","shell.execute_reply.started":"2024-03-15T07:16:39.556753Z","shell.execute_reply":"2024-03-15T07:16:41.185669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#A51C30'> |</span> Introduction</b>","metadata":{}},{"cell_type":"markdown","source":"<b>This is my First time to explore bank Dataset, And as you know, there'are so many tables in this competition. So If you are Beginner like me, it would be helpful. Let's Started!","metadata":{}},{"cell_type":"markdown","source":"### <b><span style='color:#A51C30'> |</span> Feature Columns</b>","metadata":{}},{"cell_type":"markdown","source":"📌 <b><span style='color:#A51C30'>There's are 465 Variable & 436 Description</span></b>\n\n* P - `Transform DPD` (Days past due)\n\n   Ex) <span style='color:#A51C30'>DPD previous contract, DPD of client with tolerance </span>\n* M - `Masking categories`\n\n   Ex) <span style='color:#A51C30'>District of the person's address, Role of the person's address</span>\n* A - `Transform amount`\n\n   Ex) <span style='color:#A51C30'>Deposit Amount & Tax Deduction Amount</span>\n* D - `Transform date`\n\n   Ex)  <span style='color:#A51C30'>Date of birth of the person, End date of deposit contract</span>\n* T - `Unspecified Transform`\n\n   Ex) <span style='color:#A51C30'>Type of income of the person, Month when the maximum Day Past Due (DPD) occurred for active contracts on credit bureau's records</span>\n* L - `Unspecified Transform`\n\n   Ex) <span style='color:#A51C30'>Type of application process,Interest rate for the active contracts</span>","metadata":{}},{"cell_type":"markdown","source":"**If you want to know data description, check out below**\n\nhttps://www.kaggle.com/competitions/home-credit-credit-risk-model-stability/discussion/473950\n\n**If you want to know feature column description, check out below** \n\nhttps://www.kaggle.com/datasets/kononenko/home-credit-enhanced-feature-definitions","metadata":{}},{"cell_type":"code","source":"feature_group = pd.read_parquet(\"/kaggle/input/home-credit-enhanced-feature-definitions/feature_definitions_dtypes_tables.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:16:44.285221Z","iopub.execute_input":"2024-03-15T07:16:44.286063Z","iopub.status.idle":"2024-03-15T07:16:44.499040Z","shell.execute_reply.started":"2024-03-15T07:16:44.286026Z","shell.execute_reply":"2024-03-15T07:16:44.497921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_group","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:16:45.465721Z","iopub.execute_input":"2024-03-15T07:16:45.466426Z","iopub.status.idle":"2024-03-15T07:16:45.498711Z","shell.execute_reply.started":"2024-03-15T07:16:45.466389Z","shell.execute_reply":"2024-03-15T07:16:45.496879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\n\nsns.countplot(x=feature_group.dtype)\nplt.title(\"Distribution of Dtypes in Feature Group\")\nplt.xlabel(\"Dtypes\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T08:17:01.574087Z","iopub.execute_input":"2024-03-15T08:17:01.575550Z","iopub.status.idle":"2024-03-15T08:17:01.885553Z","shell.execute_reply.started":"2024-03-15T08:17:01.575499Z","shell.execute_reply":"2024-03-15T08:17:01.884468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_group['feature'] = feature_group['Variable'].str[-1]\n\nplt.figure(figsize=(12,6))\n\nsns.countplot(x=feature_group.feature)\nplt.title(\"Distribution of  Feature Group\")\nplt.xlabel(\"Dtypes\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T08:20:04.509749Z","iopub.execute_input":"2024-03-15T08:20:04.510253Z","iopub.status.idle":"2024-03-15T08:20:04.854205Z","shell.execute_reply.started":"2024-03-15T08:20:04.510221Z","shell.execute_reply":"2024-03-15T08:20:04.852677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#A51C30'> |</span> Feature 'P'</b>","metadata":{}},{"cell_type":"code","source":"'''Feature Columns: P'''\nfeature_p = feature_group.loc[feature_group.Variable.str.endswith('P')]\nprint('feature_p shape',feature_p.shape )\nfeature_p.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:17:17.113808Z","iopub.execute_input":"2024-03-15T07:17:17.114270Z","iopub.status.idle":"2024-03-15T07:17:17.138092Z","shell.execute_reply.started":"2024-03-15T07:17:17.114234Z","shell.execute_reply":"2024-03-15T07:17:17.136353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\n\nsns.countplot(x=feature_p.dtype)\nplt.title(\"Distribution of Dtypes in Feature P\")\nplt.xlabel(\"Dtypes\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T06:29:10.757396Z","iopub.execute_input":"2024-03-15T06:29:10.757780Z","iopub.status.idle":"2024-03-15T06:29:11.083788Z","shell.execute_reply.started":"2024-03-15T06:29:10.757751Z","shell.execute_reply":"2024-03-15T06:29:11.082558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Just Only Float 64","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#A51C30'> |</span> Feature 'M'</b>","metadata":{}},{"cell_type":"code","source":"'''Feature Columns: M'''\nfeature_m = feature_group.loc[feature_group.Variable.str.endswith('M')]\nprint('feature_m shape',feature_m.shape )\nfeature_m.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:18:56.492939Z","iopub.execute_input":"2024-03-15T07:18:56.493405Z","iopub.status.idle":"2024-03-15T07:18:56.513674Z","shell.execute_reply.started":"2024-03-15T07:18:56.493370Z","shell.execute_reply":"2024-03-15T07:18:56.512141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\n\nsns.countplot(x=feature_m.dtype)\nplt.title(\"Distribution of Dtypes in Feature M\")\nplt.xlabel(\"Dtypes\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:20:03.612744Z","iopub.execute_input":"2024-03-15T07:20:03.613294Z","iopub.status.idle":"2024-03-15T07:20:03.932670Z","shell.execute_reply.started":"2024-03-15T07:20:03.613256Z","shell.execute_reply":"2024-03-15T07:20:03.930573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Just Only String ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#A51C30'> |</span> Feature 'A'</b>","metadata":{}},{"cell_type":"code","source":"'''Feature Columns: A'''\nfeature_a = feature_group.loc[feature_group.Variable.str.endswith('A')]\nprint('feature_a shape',feature_a.shape )\nfeature_a.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:22:07.965612Z","iopub.execute_input":"2024-03-15T07:22:07.966080Z","iopub.status.idle":"2024-03-15T07:22:07.988130Z","shell.execute_reply.started":"2024-03-15T07:22:07.966047Z","shell.execute_reply":"2024-03-15T07:22:07.986195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\n\nsns.countplot(x=feature_a.dtype)\nplt.title(\"Distribution of Dtypes in Feature A\")\nplt.xlabel(\"Dtypes\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:22:30.194915Z","iopub.execute_input":"2024-03-15T07:22:30.195492Z","iopub.status.idle":"2024-03-15T07:22:30.468991Z","shell.execute_reply.started":"2024-03-15T07:22:30.195407Z","shell.execute_reply":"2024-03-15T07:22:30.467609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# just only Float64","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#A51C30'> |</span> Feature 'D'</b>","metadata":{}},{"cell_type":"code","source":"'''Feature Columns: D'''\nfeature_d = feature_group.loc[feature_group.Variable.str.endswith('D')]\nprint('feature_d shape',feature_d.shape )\nfeature_d.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:23:33.765528Z","iopub.execute_input":"2024-03-15T07:23:33.766089Z","iopub.status.idle":"2024-03-15T07:23:33.787954Z","shell.execute_reply.started":"2024-03-15T07:23:33.766045Z","shell.execute_reply":"2024-03-15T07:23:33.786457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\n\nsns.countplot(x=feature_d.dtype)\nplt.title(\"Distribution of Dtypes in Feature D\")\nplt.xlabel(\"Dtypes\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:24:07.824207Z","iopub.execute_input":"2024-03-15T07:24:07.824780Z","iopub.status.idle":"2024-03-15T07:24:08.104716Z","shell.execute_reply.started":"2024-03-15T07:24:07.824743Z","shell.execute_reply":"2024-03-15T07:24:08.103324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Just only Date","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#A51C30'> |</span> Feature 'T'</b>","metadata":{}},{"cell_type":"code","source":"'''Feature Columns: T'''\nfeature_t = feature_group.loc[feature_group.Variable.str.endswith('T')]\nprint('feature_t shape',feature_t.shape )\nfeature_t.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:24:54.477852Z","iopub.execute_input":"2024-03-15T07:24:54.478277Z","iopub.status.idle":"2024-03-15T07:24:54.499646Z","shell.execute_reply.started":"2024-03-15T07:24:54.478247Z","shell.execute_reply":"2024-03-15T07:24:54.497909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\n\nsns.countplot(x=feature_t.dtype)\nplt.title(\"Distribution of Dtypes in Feature T\")\nplt.xlabel(\"Dtypes\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:25:12.532234Z","iopub.execute_input":"2024-03-15T07:25:12.532956Z","iopub.status.idle":"2024-03-15T07:25:12.857898Z","shell.execute_reply.started":"2024-03-15T07:25:12.532912Z","shell.execute_reply":"2024-03-15T07:25:12.855348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_t.loc[feature_t.dtype == 'String']","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:26:27.429515Z","iopub.execute_input":"2024-03-15T07:26:27.430164Z","iopub.status.idle":"2024-03-15T07:26:27.455443Z","shell.execute_reply.started":"2024-03-15T07:26:27.430116Z","shell.execute_reply":"2024-03-15T07:26:27.453519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# String: 23%, Float64: 77%","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#A51C30'> |</span> Feature 'L'</b>","metadata":{}},{"cell_type":"code","source":"'''Feature Columns: L'''\nfeature_l = feature_group.loc[feature_group.Variable.str.endswith('L')]\nprint('feature_l shape',feature_l.shape )\nfeature_l.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:28:15.530879Z","iopub.execute_input":"2024-03-15T07:28:15.531406Z","iopub.status.idle":"2024-03-15T07:28:15.553180Z","shell.execute_reply.started":"2024-03-15T07:28:15.531374Z","shell.execute_reply":"2024-03-15T07:28:15.551627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\n\nsns.countplot(x=feature_l.dtype)\nplt.title(\"Distribution of Dtypes in Feature L\")\nplt.xlabel(\"Dtypes\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:28:32.876477Z","iopub.execute_input":"2024-03-15T07:28:32.876952Z","iopub.status.idle":"2024-03-15T07:28:33.167228Z","shell.execute_reply.started":"2024-03-15T07:28:32.876908Z","shell.execute_reply":"2024-03-15T07:28:33.165524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Float64 in Feature_t: {:.1%}'.format(feature_l.loc[feature_l.dtype == 'Float64'].shape[0]/feature_l.shape[0]))\nprint('String in Feature_t: {:.1%}'.format(feature_l.loc[feature_l.dtype == 'String'].shape[0]/feature_l.shape[0]))\nprint('Boolean in Feature_t: {:.1%}'.format(feature_l.loc[feature_l.dtype == 'Boolean'].shape[0]/feature_l.shape[0]))","metadata":{"execution":{"iopub.status.busy":"2024-03-15T07:34:51.223048Z","iopub.execute_input":"2024-03-15T07:34:51.223545Z","iopub.status.idle":"2024-03-15T07:34:51.235059Z","shell.execute_reply.started":"2024-03-15T07:34:51.223512Z","shell.execute_reply":"2024-03-15T07:34:51.233294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}