{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8438170,"sourceType":"datasetVersion","datasetId":5026285},{"sourceId":8506258,"sourceType":"datasetVersion","datasetId":5077251},{"sourceId":8506421,"sourceType":"datasetVersion","datasetId":5077368}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"!pip install polars\n!pip install tqdm\n!pip install pyarrow","metadata":{}},{"cell_type":"code","source":"# General\nimport sys, warnings, time, os, copy, gc, re, random, json\nwarnings.filterwarnings('ignore')\nimport pickle as pkl\nfrom IPython.display import display\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport seaborn as sns\nsns.set()\nfrom pprint import pprint\nfrom pathlib import Path\nfrom tqdm import tqdm\ntqdm.pandas()\nfrom datetime import datetime, timedelta\nimport datetime\ncurrent_time = datetime.datetime.now()\n# Model\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.datasets import make_classification\n\n# dataPath = \"data/credit/\" # change this when put in to commmit or into someone computer\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\" # This is path for summit compertition\n","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:04:25.289609Z","iopub.execute_input":"2024-05-27T15:04:25.290042Z","iopub.status.idle":"2024-05-27T15:04:29.188621Z","shell.execute_reply.started":"2024-05-27T15:04:25.290004Z","shell.execute_reply":"2024-05-27T15:04:29.187271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n # implement here all desired dtypes for tables\n # the following is just an example\n for col in df.columns:\n \n if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n df = df.with_columns(pl.col(col).cast(pl.Int32))\n elif col in [\"date_decision\"]:\n df = df.with_columns(pl.col(col).cast(pl.Date))\n elif col[-1] in (\"P\", \"A\"):\n df = df.with_columns(pl.col(col).cast(pl.Float64))\n elif col[-1] in (\"M\",):\n df = df.with_columns(pl.col(col).cast(pl.String))\n elif col[-1] in (\"D\",):\n df = df.with_columns(pl.col(col).cast(pl.Date)) \n \n return df\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n for col in df.columns: \n if df[col].dtype.name in ['object', 'string']:\n df[col] = df[col].fillna('missing')\n df[col] = df[col].astype(\"string\").astype('category')\n current_categories = df[col].cat.categories\n new_categories = current_categories.to_list() + [\"Unknown\"]\n new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n df[col] = df[col].astype(new_dtype)\n return df ","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:04:29.191062Z","iopub.execute_input":"2024-05-27T15:04:29.191742Z","iopub.status.idle":"2024-05-27T15:04:29.205512Z","shell.execute_reply.started":"2024-05-27T15:04:29.191697Z","shell.execute_reply":"2024-05-27T15:04:29.204132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hàm\ndef transformDate(table):\n for column in table.columns:\n if table[column].dtype == pl.Date:\n tempSeries = []\n for iter in table[column]:\n if iter != None:\n timeDuration = current_time.year - iter.year\n tempSeries.append(timeDuration)\n else:\n tempSeries.append(0)\n table.replace(column,pl.Series(tempSeries))\n\n\ndef meanDatamodeString(table):\n table = table.drop(\"num_group1\")\n if table.get_column_index(\"num_group2\"):\n table = table.drop(\"num_group2\")\n table_data = table.groupby(\"case_id\" , maintain_order=True).mean()\n table_string = table.groupby(\"case_id\", maintain_order=True).agg(pl.col(pl.String).mean())\n table = table_data.join(table_string, on=\"case_id\", how=\"left\")\n del table_data, table_string \n return table","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:04:29.209273Z","iopub.execute_input":"2024-05-27T15:04:29.209688Z","iopub.status.idle":"2024-05-27T15:04:29.220276Z","shell.execute_reply.started":"2024-05-27T15:04:29.209635Z","shell.execute_reply":"2024-05-27T15:04:29.218888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_a_2_1 = pl.concat(\n [\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_0.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_1.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_2.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_3.csv\").pipe(set_table_dtypes),\n ],\n how=\"vertical_relaxed\",\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:04:29.223076Z","iopub.execute_input":"2024-05-27T15:04:29.224235Z","iopub.status.idle":"2024-05-27T15:05:10.758885Z","shell.execute_reply.started":"2024-05-27T15:04:29.224191Z","shell.execute_reply":"2024-05-27T15:05:10.757617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(train_credit_bureau_a_2_1)\ntrain_credit_bureau_a_2_1 = meanDatamodeString(train_credit_bureau_a_2_1)\ntrain_credit_bureau_a_2_1.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:05:10.760471Z","iopub.execute_input":"2024-05-27T15:05:10.760828Z","iopub.status.idle":"2024-05-27T15:05:13.442135Z","shell.execute_reply.started":"2024-05-27T15:05:10.760799Z","shell.execute_reply":"2024-05-27T15:05:13.441012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_a_2_2 = pl.concat(\n [\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_4.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_5.csv\").pipe(set_table_dtypes),\n ],\n how=\"vertical_relaxed\",\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:05:13.443348Z","iopub.execute_input":"2024-05-27T15:05:13.443672Z","iopub.status.idle":"2024-05-27T15:05:51.600084Z","shell.execute_reply.started":"2024-05-27T15:05:13.443632Z","shell.execute_reply":"2024-05-27T15:05:51.599132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(train_credit_bureau_a_2_2)\ntrain_credit_bureau_a_2_2 = meanDatamodeString(train_credit_bureau_a_2_2)\ntrain_credit_bureau_a_2_2.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:05:51.601923Z","iopub.execute_input":"2024-05-27T15:05:51.602727Z","iopub.status.idle":"2024-05-27T15:05:54.345814Z","shell.execute_reply.started":"2024-05-27T15:05:51.602689Z","shell.execute_reply":"2024-05-27T15:05:54.344439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_a_2_3 = pl.concat(\n [\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_6.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_7.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_8.csv\").pipe(set_table_dtypes),\n ],\n how=\"vertical_relaxed\",\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:05:54.347225Z","iopub.execute_input":"2024-05-27T15:05:54.347700Z","iopub.status.idle":"2024-05-27T15:06:25.582359Z","shell.execute_reply.started":"2024-05-27T15:05:54.347644Z","shell.execute_reply":"2024-05-27T15:06:25.580953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(train_credit_bureau_a_2_3)\ntrain_credit_bureau_a_2_3 = meanDatamodeString(train_credit_bureau_a_2_3)\ntrain_credit_bureau_a_2_3.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:06:25.583941Z","iopub.execute_input":"2024-05-27T15:06:25.584466Z","iopub.status.idle":"2024-05-27T15:06:27.582705Z","shell.execute_reply.started":"2024-05-27T15:06:25.584346Z","shell.execute_reply":"2024-05-27T15:06:27.581228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_a_2_4 = pl.concat(\n [\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_9.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_10.csv\").pipe(set_table_dtypes),\n ],\n how=\"vertical_relaxed\",\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:06:27.588049Z","iopub.execute_input":"2024-05-27T15:06:27.588407Z","iopub.status.idle":"2024-05-27T15:06:43.489589Z","shell.execute_reply.started":"2024-05-27T15:06:27.588379Z","shell.execute_reply":"2024-05-27T15:06:43.488364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(train_credit_bureau_a_2_4)\ntrain_credit_bureau_a_2_4 = meanDatamodeString(train_credit_bureau_a_2_4)\ntrain_credit_bureau_a_2_4.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:06:43.491017Z","iopub.execute_input":"2024-05-27T15:06:43.491365Z","iopub.status.idle":"2024-05-27T15:06:44.415228Z","shell.execute_reply.started":"2024-05-27T15:06:43.491326Z","shell.execute_reply":"2024-05-27T15:06:44.414358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = pl.concat([train_credit_bureau_a_2_1,train_credit_bureau_a_2_2, train_credit_bureau_a_2_3, train_credit_bureau_a_2_4], how=\"diagonal_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:06:44.416451Z","iopub.execute_input":"2024-05-27T15:06:44.417184Z","iopub.status.idle":"2024-05-27T15:06:44.866710Z","shell.execute_reply.started":"2024-05-27T15:06:44.417152Z","shell.execute_reply":"2024-05-27T15:06:44.865852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_credit_bureau_a_2_1, train_credit_bureau_a_2_2, train_credit_bureau_a_2_3, train_credit_bureau_a_2_4","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:06:44.870172Z","iopub.execute_input":"2024-05-27T15:06:44.870645Z","iopub.status.idle":"2024-05-27T15:06:44.883202Z","shell.execute_reply.started":"2024-05-27T15:06:44.870540Z","shell.execute_reply":"2024-05-27T15:06:44.882039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Xử lý base và lọc các cột dữ liệu thừa\ntrain_basetable = pl.read_csv(dataPath + \"csv_files/train/train_base.csv\").pipe(set_table_dtypes)\n# Chúng ta không sài các chỉ số MONTH ,WEEK_NUM và date_decision nên bỏ luôn những cột đó\ndata = train_basetable.drop(['MONTH', 'WEEK_NUM', 'date_decision'])\ndel train_basetable\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:06:44.885134Z","iopub.execute_input":"2024-05-27T15:06:44.885947Z","iopub.status.idle":"2024-05-27T15:06:45.213112Z","shell.execute_reply.started":"2024-05-27T15:06:44.885903Z","shell.execute_reply":"2024-05-27T15:06:45.212155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\ntemp, how=\"left\", on=\"case_id\"\n)\ndata.shape\ndel temp","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:06:45.215865Z","iopub.execute_input":"2024-05-27T15:06:45.216782Z","iopub.status.idle":"2024-05-27T15:06:45.768611Z","shell.execute_reply.started":"2024-05-27T15:06:45.216750Z","shell.execute_reply":"2024-05-27T15:06:45.766140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_a_1 = pl.concat(\n [\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n ],\n how=\"vertical_relaxed\",\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:06:45.770172Z","iopub.execute_input":"2024-05-27T15:06:45.770621Z","iopub.status.idle":"2024-05-27T15:07:51.708747Z","shell.execute_reply.started":"2024-05-27T15:06:45.770581Z","shell.execute_reply":"2024-05-27T15:07:51.706325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(train_credit_bureau_a_1)\ntrain_credit_bureau_a_1 = meanDatamodeString(train_credit_bureau_a_1)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:07:51.712069Z","iopub.execute_input":"2024-05-27T15:07:51.712640Z","iopub.status.idle":"2024-05-27T15:09:18.124805Z","shell.execute_reply.started":"2024-05-27T15:07:51.712583Z","shell.execute_reply":"2024-05-27T15:09:18.123416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n train_credit_bureau_a_1, how=\"left\", on=\"case_id\"\n)\n\ndel train_credit_bureau_a_1\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:09:18.127006Z","iopub.execute_input":"2024-05-27T15:09:18.127475Z","iopub.status.idle":"2024-05-27T15:09:19.542401Z","shell.execute_reply.started":"2024-05-27T15:09:18.127423Z","shell.execute_reply":"2024-05-27T15:09:19.541139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_applprev = pl.concat(\n [\n pl.read_csv(dataPath + \"csv_files/train/train_applprev_1_0.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_applprev_1_1.csv\").pipe(set_table_dtypes), \n ],\n how=\"vertical_relaxed\",\n)\ntrain_applprev.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:09:19.543991Z","iopub.execute_input":"2024-05-27T15:09:19.544394Z","iopub.status.idle":"2024-05-27T15:09:30.497901Z","shell.execute_reply.started":"2024-05-27T15:09:19.544362Z","shell.execute_reply":"2024-05-27T15:09:30.496610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(train_applprev)\ntrain_applprev = meanDatamodeString(train_applprev)\ntrain_applprev.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:09:30.499207Z","iopub.execute_input":"2024-05-27T15:09:30.499513Z","iopub.status.idle":"2024-05-27T15:10:06.519118Z","shell.execute_reply.started":"2024-05-27T15:09:30.499488Z","shell.execute_reply":"2024-05-27T15:10:06.517761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_applprev.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:10:06.520579Z","iopub.execute_input":"2024-05-27T15:10:06.520932Z","iopub.status.idle":"2024-05-27T15:10:06.527378Z","shell.execute_reply.started":"2024-05-27T15:10:06.520897Z","shell.execute_reply":"2024-05-27T15:10:06.526479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_applprev_2 = pl.read_csv(dataPath + \"csv_files/train/train_applprev_2.csv\").pipe(set_table_dtypes)\n# train_person_1 = pl.read_csv(dataPath + \"csv/train/train_person_1.csv\").pipe(set_table_dtypes) \n# train_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv/train/train_credit_bureau_b_2.csv\").pipe(set_table_dtypes) \ntrain_applprev_2 = train_applprev_2.drop(['cacccardblochreas_147M', 'credacc_cards_status_52L', 'num_group1','num_group2'])\ntrain_applprev_2.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:10:06.528593Z","iopub.execute_input":"2024-05-27T15:10:06.529118Z","iopub.status.idle":"2024-05-27T15:10:09.225712Z","shell.execute_reply.started":"2024-05-27T15:10:06.529088Z","shell.execute_reply":"2024-05-27T15:10:09.224500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_applprev_2 = pl.DataFrame(train_applprev_2)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:10:09.227266Z","iopub.execute_input":"2024-05-27T15:10:09.227724Z","iopub.status.idle":"2024-05-27T15:10:09.234926Z","shell.execute_reply.started":"2024-05-27T15:10:09.227683Z","shell.execute_reply":"2024-05-27T15:10:09.233697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# temp = train_applprev_2.groupby('case_id').agg({'conts_type_509L': 'mean'})\n# case_ID = train_applprev_2['case_id']\n# train_Type = train_applprev_2['conts_type_509L']\ntrain_applprev_2 = train_applprev_2.groupby(\"case_id\").agg(pl.col(\"conts_type_509L\").drop_nulls().mode())\ntrain_applprev_2.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:10:09.236356Z","iopub.execute_input":"2024-05-27T15:10:09.236825Z","iopub.status.idle":"2024-05-27T15:10:19.772740Z","shell.execute_reply.started":"2024-05-27T15:10:09.236787Z","shell.execute_reply":"2024-05-27T15:10:19.771483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n train_applprev, how=\"left\", on=\"case_id\"\n).join(\n train_applprev_2, how=\"left\", on=\"case_id\"\n)\ndel train_applprev\ndel train_applprev_2\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:10:19.774093Z","iopub.execute_input":"2024-05-27T15:10:19.774458Z","iopub.status.idle":"2024-05-27T15:10:21.437583Z","shell.execute_reply.started":"2024-05-27T15:10:19.774430Z","shell.execute_reply":"2024-05-27T15:10:21.436510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I know that the table has soooo many empty cell so we have to define to what extend the column is empty \nSo i choose 90% empty and we drop that column \nYou might wondering why 90% ? someone got it to 99,5% but my computer cannot have that much of power to run \nSo i have to lower the bar\nYes the 4000$ USD machine not THAT strong","metadata":{}},{"cell_type":"markdown","source":"train_credit_bureau_a_1_0 = meanDatamodeString(train_credit_bureau_a_1_0)\ntrain_credit_bureau_a_1_1 = meanDatamodeString(train_credit_bureau_a_1_1)\ntrain_credit_bureau_a_1_2 = meanDatamodeString(train_credit_bureau_a_1_2)\ntrain_credit_bureau_a_1_3 = meanDatamodeString(train_credit_bureau_a_1_3)","metadata":{"execution":{"iopub.status.busy":"2024-05-12T04:35:37.285192Z","iopub.execute_input":"2024-05-12T04:35:37.286022Z","iopub.status.idle":"2024-05-12T04:35:49.947145Z","shell.execute_reply.started":"2024-05-12T04:35:37.285991Z","shell.execute_reply":"2024-05-12T04:35:49.946292Z"}}},{"cell_type":"code","source":"# train_credit_bureau_b_1 = pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_b_1.csv\").pipe(set_table_dtypes)\n# train_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_b_1.csv\").pipe(set_table_dtypes)\ntrain_credit_bureau_b = pl.concat(\n [\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_b_1.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_b_2.csv\").pipe(set_table_dtypes),\n ],\n how=\"diagonal\",\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:10:21.439813Z","iopub.execute_input":"2024-05-27T15:10:21.440166Z","iopub.status.idle":"2024-05-27T15:10:22.817175Z","shell.execute_reply.started":"2024-05-27T15:10:21.440138Z","shell.execute_reply":"2024-05-27T15:10:22.815761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(train_credit_bureau_b)\n# transformDate(train_credit_bureau_b_2)\ntrain_credit_bureau_b = meanDatamodeString(train_credit_bureau_b)\n# meanDatamodeString(train_credit_bureau_b_2)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:10:22.818862Z","iopub.execute_input":"2024-05-27T15:10:22.819395Z","iopub.status.idle":"2024-05-27T15:10:24.612125Z","shell.execute_reply.started":"2024-05-27T15:10:22.819349Z","shell.execute_reply":"2024-05-27T15:10:24.610820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n train_credit_bureau_b, how=\"left\", on=\"case_id\"\n)\ndel train_credit_bureau_b","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:10:24.621042Z","iopub.execute_input":"2024-05-27T15:10:24.621447Z","iopub.status.idle":"2024-05-27T15:10:25.038218Z","shell.execute_reply.started":"2024-05-27T15:10:24.621415Z","shell.execute_reply":"2024-05-27T15:10:25.037233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:10:25.040042Z","iopub.execute_input":"2024-05-27T15:10:25.040458Z","iopub.status.idle":"2024-05-27T15:10:25.047793Z","shell.execute_reply.started":"2024-05-27T15:10:25.040421Z","shell.execute_reply":"2024-05-27T15:10:25.046592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_credit_bureau_b = pl.concat(\n# [\n# pl.read_csv(dataPath + \"csv/train/train_credit_bureau_b_1.csv\").pipe(set_table_dtypes),\n# pl.read_csv(dataPath + \"csv/train/train_credit_bureau_b_2.csv\").pipe(set_table_dtypes),\n# ],\n# how=\"vertical_relaxed\",\n# )\n\ntrain_debitcard_1 = pl.read_csv(dataPath + \"csv_files/train/train_debitcard_1.csv\").pipe(set_table_dtypes)\n\ntrain_deposit_1 = pl.read_csv(dataPath + \"csv_files/train/train_deposit_1.csv\").pipe(set_table_dtypes)\n\ntrain_other_1 = pl.read_csv(dataPath + \"csv_files/train/train_other_1.csv\").pipe(set_table_dtypes)\n\n# train_person = pl.concat(\n# [\n# pl.read_csv(dataPath + \"csv/train/train_person_1.csv\").pipe(set_table_dtypes),\n# pl.read_csv(dataPath + \"csv/train/train_person_2.csv\").pipe(set_table_dtypes),\n# ],\n# how=\"vertical_relaxed\",\n# )\n\ntrain_person_1 = pl.read_csv(dataPath + \"csv_files/train/train_person_1.csv\").pipe(set_table_dtypes)\n\ntrain_person_2 = pl.read_csv(dataPath + \"csv_files/train/train_person_2.csv\").pipe(set_table_dtypes)\n\ntrain_static = pl.concat(\n [\n pl.read_csv(dataPath + \"csv_files/train/train_static_0_0.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"csv_files/train/train_static_0_1.csv\").pipe(set_table_dtypes),\n ],\n how=\"vertical_relaxed\",\n)\n\n# train_static = pl.read_csv(dataPath + \"csv/train/train_static_0_0.csv\").pipe(set_table_dtypes)\n\ntrain_static_cb = pl.read_csv(dataPath + \"csv_files/train/train_static_cb_0.csv\").pipe(set_table_dtypes)\n\ntrain_tax_registry_a_1 = pl.read_csv(dataPath + \"csv_files/train/train_tax_registry_a_1.csv\").pipe(set_table_dtypes)\n\ntrain_tax_registry_b_1 = pl.read_csv(dataPath + \"csv_files/train/train_tax_registry_b_1.csv\").pipe(set_table_dtypes)\n\ntrain_tax_registry_c_1 = pl.read_csv(dataPath + \"csv_files/train/train_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:10:25.049461Z","iopub.execute_input":"2024-05-27T15:10:25.050851Z","iopub.status.idle":"2024-05-27T15:10:43.407339Z","shell.execute_reply.started":"2024-05-27T15:10:25.050808Z","shell.execute_reply":"2024-05-27T15:10:43.406266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(train_debitcard_1)\ntransformDate(train_deposit_1)\ntrain_debitcard_1 = meanDatamodeString(train_debitcard_1)\ntrain_deposit_1 = meanDatamodeString(train_deposit_1)\n\ntransformDate(train_other_1)\ntransformDate(train_static)\ntrain_other_1 = meanDatamodeString(train_other_1)\ntrain_static = meanDatamodeString(train_static)\n\ntransformDate(train_person_1)\ntransformDate(train_person_2)\ntrain_person_1 = meanDatamodeString(train_person_1)\ntrain_person_2 = meanDatamodeString(train_person_2)\n# train_debitcard_1.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:10:43.408606Z","iopub.execute_input":"2024-05-27T15:10:43.409785Z","iopub.status.idle":"2024-05-27T15:11:08.387554Z","shell.execute_reply.started":"2024-05-27T15:10:43.409749Z","shell.execute_reply":"2024-05-27T15:11:08.386590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n train_debitcard_1, how=\"left\", on=\"case_id\"\n).join(\n train_deposit_1, how=\"left\", on=\"case_id\"\n).join(\n train_other_1, how=\"left\", on=\"case_id\"\n).join(\n train_static, how=\"left\", on=\"case_id\"\n).join(\n train_person_1, how=\"left\", on=\"case_id\"\n).join(\n train_person_2, how=\"left\", on=\"case_id\"\n)\ndel train_person_1\ndel train_person_2\ndel train_debitcard_1\ndel train_deposit_1\ndel train_other_1\ndel train_static\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:11:08.388730Z","iopub.execute_input":"2024-05-27T15:11:08.389245Z","iopub.status.idle":"2024-05-27T15:11:11.295463Z","shell.execute_reply.started":"2024-05-27T15:11:08.389216Z","shell.execute_reply":"2024-05-27T15:11:11.294370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(train_static_cb)\ntransformDate(train_tax_registry_a_1)\ntrain_static_cb = meanDatamodeString(train_static_cb)\ntrain_tax_registry_a_1 = meanDatamodeString(train_tax_registry_a_1)\n\ntransformDate(train_tax_registry_b_1)\ntransformDate(train_tax_registry_c_1)\ntrain_tax_registry_b_1 = meanDatamodeString(train_tax_registry_b_1)\ntrain_tax_registry_c_1 = meanDatamodeString(train_tax_registry_c_1)\n\n# train_static_cb.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:11:11.297069Z","iopub.execute_input":"2024-05-27T15:11:11.297379Z","iopub.status.idle":"2024-05-27T15:11:23.360026Z","shell.execute_reply.started":"2024-05-27T15:11:11.297354Z","shell.execute_reply":"2024-05-27T15:11:23.358868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n train_static_cb, how=\"left\", on=\"case_id\"\n).join(\n train_tax_registry_a_1, how=\"left\", on=\"case_id\"\n).join(\n train_tax_registry_b_1, how=\"left\", on=\"case_id\"\n).join(\n train_tax_registry_c_1, how=\"left\", on=\"case_id\"\n)\n\ndel train_static_cb\ndel train_tax_registry_a_1\ndel train_tax_registry_b_1\ndel train_tax_registry_c_1\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:11:23.361565Z","iopub.execute_input":"2024-05-27T15:11:23.361899Z","iopub.status.idle":"2024-05-27T15:11:24.415715Z","shell.execute_reply.started":"2024-05-27T15:11:23.361873Z","shell.execute_reply":"2024-05-27T15:11:24.414566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cắt các dòng chiếm đa số là null\ndvalues = data.select([\"case_id\",\"target\"])\nfeature = data.drop([\"case_id\",\"target\"])","metadata":{"execution":{"iopub.status.busy":"2024-05-25T19:25:25.584725Z","iopub.execute_input":"2024-05-25T19:25:25.585667Z","iopub.status.idle":"2024-05-25T19:25:25.592254Z","shell.execute_reply.started":"2024-05-25T19:25:25.585626Z","shell.execute_reply":"2024-05-25T19:25:25.590920Z"}}},{"cell_type":"markdown","source":"feature[[s.name for s in feature if not (s.null_count() == feature.height)]]","metadata":{"execution":{"iopub.status.busy":"2024-05-25T19:25:25.593640Z","iopub.execute_input":"2024-05-25T19:25:25.594194Z","iopub.status.idle":"2024-05-25T19:25:25.621885Z","shell.execute_reply.started":"2024-05-25T19:25:25.594157Z","shell.execute_reply":"2024-05-25T19:25:25.620566Z"}}},{"cell_type":"markdown","source":"del data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-25T19:25:25.623503Z","iopub.execute_input":"2024-05-25T19:25:25.623844Z","iopub.status.idle":"2024-05-25T19:25:25.737188Z","shell.execute_reply.started":"2024-05-25T19:25:25.623815Z","shell.execute_reply":"2024-05-25T19:25:25.735977Z"}}},{"cell_type":"markdown","source":"temp = pl.concat([feature, dvalues], how=\"diagonal_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-05-25T19:25:25.738872Z","iopub.execute_input":"2024-05-25T19:25:25.739803Z"}}},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:11:24.416931Z","iopub.execute_input":"2024-05-27T15:11:24.417233Z","iopub.status.idle":"2024-05-27T15:11:24.594686Z","shell.execute_reply.started":"2024-05-27T15:11:24.417208Z","shell.execute_reply":"2024-05-27T15:11:24.593494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lưu dữ liệu\n\n# Lưu bằng pandas\ndata_pandas = data.to_pandas()\ndata_pandas.to_csv('train_data_full.csv')\n\n# # Lưu bằng polars\n# import pathlib\n# path = \"train_data.csv\"\n# data.write_csv(path, separator=\",\")\n# The data will be save in !ls /kaggle/working\ndel data_pandas","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:11:24.596135Z","iopub.execute_input":"2024-05-27T15:11:24.596535Z","iopub.status.idle":"2024-05-27T15:19:09.461677Z","shell.execute_reply.started":"2024-05-27T15:11:24.596509Z","shell.execute_reply":"2024-05-27T15:19:09.459569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Vì dữ liệu quá lớn mà RAM thì quá thiếu nên không thể xử lý theo cách thông thường được nên \npath = \"/kaggle/input/train-data/train_data_full.csv\"\ndata = pl.read_csv(path)\n# data = pd.read_csv('train_data_full.csv') ","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:19:09.463715Z","iopub.execute_input":"2024-05-27T15:19:09.464164Z","iopub.status.idle":"2024-05-27T15:19:57.452046Z","shell.execute_reply.started":"2024-05-27T15:19:09.464123Z","shell.execute_reply":"2024-05-27T15:19:57.450677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import collections\n# Bỏ những dữ liệu có 98% null \n# Dictionary of counts of the different classes in the training data\ndt_N_y_train = dict(collections.Counter(data[\"target\"]))\ndt_N_y_train[\"ratio\"] = dt_N_y_train[0] / dt_N_y_train[1]\n# Initialise figure and axes\nplt.figure(figsize=(6.4, 4.8))\nax = plt.axes()\nplt.title(\"Number of counts per class\", pad=20)\n\n# Define count plot for the occurrence of the different digits in the training data \nsns.countplot(\n ax=ax,\n x=data[\"target\"].to_numpy(),\n color=\"blue\",\n alpha=0.5,\n edgecolor=\"black\",\n linewidth=1.0,\n width=0.075,\n hatch=\"////\",\n zorder=2\n)\n# Define axes labels \nax.set_xlabel(r\"Class, $y$\", fontdict={\"fontsize\": 10})\nax.set_ylabel(r\"Counts\", fontdict={\"fontsize\": 10})\n\n# Enable axes' minor ticks\nax.minorticks_on()\n\n# Define grid\nax.grid(\n visible=True,\n which=\"major\",\n color=\"lightgray\",\n linestyle=\"solid\",\n linewidth=0.5\n)\nax.grid(\n visible=True,\n which=\"minor\",\n color=\"lightgray\",\n linestyle=\"dotted\",\n linewidth=0.5\n)\n\n# Show plot\nplt.show() \n\n# Display ratio between numbers of settled (y=0) and defaulted (y=1) credit cases\nprint()\n\n# Display ratio between numbers of settled (y=0) and defaulted (y=1) credit cases\nprint()\ndisplay(pd.DataFrame(data={\"$n^-/n^+$\": dt_N_y_train[\"ratio\"]},\n index=[0])\\\n .style\\\n .format({\"$n^-/n^+$\": \"{:.2f}\"})\\\n .set_caption(\"Ratio between numbers of settled ($y=0$) and defaulted ($y=1$) credit contract cases\")\\\n .set_table_styles([\n # Column width\n {\"selector\": \"th.col_heading,td\",\n \"props\": [(\"width\", \"300px\")]\n },\n # Caption style\n {\"selector\": \"caption\",\n \"props\": [(\"font-size\", \"16px\"),\n (\"font-weight\", \"bold\"),\n (\"font-style\", \"italic\")]\n }\n ]))","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:19:57.453700Z","iopub.execute_input":"2024-05-27T15:19:57.454124Z","iopub.status.idle":"2024-05-27T15:19:58.596926Z","shell.execute_reply.started":"2024-05-27T15:19:57.454085Z","shell.execute_reply":"2024-05-27T15:19:58.595637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ---> Get columns' emptiness value in the training set\n\n# Dictionary describing columns emptiness\ndt_emp = {\n # Columns' emptiness value\n \"emptiness\": {col: data[col].null_count() / len(data[col]) for\n col in data.columns},\n # Columns' almost emptiness indicator: if true, more than 99.5 % of the entries are\n # empty\n \"almost_empty\": {col: data[col].null_count() / len(data[col]) > 0.98 for\n col in data.columns}\n}\n","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:19:58.598566Z","iopub.execute_input":"2024-05-27T15:19:58.599224Z","iopub.status.idle":"2024-05-27T15:19:58.618225Z","shell.execute_reply.started":"2024-05-27T15:19:58.599183Z","shell.execute_reply":"2024-05-27T15:19:58.617048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ---> Plot emptiness distribution of the training set\n\n# Initialise figure and axes\nplt.figure(figsize=(6.4, 4.8))\nax = plt.axes()\nplt.title(\"Number of counts per emptiness range in the training dataset\", pad=20)\n\n# Define histogram plot for counts (of columns) versus emptiness range\nsns.histplot(\n ax=ax,\n data=dt_emp[\"emptiness\"].values(),\n stat=\"count\",\n bins=20,\n binrange=(0, 1),\n palette=[\"blue\"],\n alpha=0.5,\n edgecolor=\"black\",\n linewidth=1.0,\n hatch=\"////\",\n zorder=2,\n legend=False\n)\n\n# Plot verical line emptiness = 0.98\nplt.axvline(\n x=0.98,\n color=\"black\",\n linewidth=2,\n alpha=1,\n linestyle=\"dashed\",\n label=\"$\\mathrm{emptiness}=0.98$\"\n)\n\n# Define axes labels \nax.set_xlabel(r\"Emptiness\", fontdict={\"fontsize\": 10})\nax.set_ylabel(r\"Counts\", fontdict={\"fontsize\": 10})\n\n# Define axes limits\nax.set_xlim(\n left=0,\n right=1\n)\n \n# Enable axes' minor ticks\nax.minorticks_on()\n\n# Define grid\nax.grid(\n visible=True,\n which=\"major\",\n color=\"lightgray\",\n linestyle=\"solid\",\n linewidth=0.5\n)\nax.grid(\n visible=True,\n which=\"minor\",\n color=\"lightgray\",\n linestyle=\"dotted\",\n linewidth=0.5\n)\n\n# Define legend\nax.legend(fontsize=8)\n\n# Show plot\nplt.show() \n\n# ---> Display number of \"almost empty\" columns\nprint()\ndisplay(pd.DataFrame(data={\"Counts ($\\mathrm{emptiness}>0.98$)\":\n sum(dt_emp[\"almost_empty\"].values())},\n index=[0])\\\n .style\\\n .format({\"Counts ($\\mathrm{emptiness}>0.98$)\": \"{:d}\"})\\\n .set_caption(\"Number of almost empty columns in the training dataset\")\\\n .set_table_styles([\n # Column width\n {\"selector\": \"th.col_heading,td\",\n \"props\": [(\"width\", \"300px\")]\n },\n # Caption style\n {\"selector\": \"caption\",\n \"props\": [(\"font-size\", \"16px\"),\n (\"font-weight\", \"bold\"),\n (\"font-style\", \"italic\")]\n }\n ]))\n","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:19:58.619984Z","iopub.execute_input":"2024-05-27T15:19:58.620313Z","iopub.status.idle":"2024-05-27T15:19:59.320197Z","shell.execute_reply.started":"2024-05-27T15:19:58.620286Z","shell.execute_reply":"2024-05-27T15:19:59.318931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ---> Drop almost empty columns\ncols_drop = [key for key in dt_emp[\"almost_empty\"].keys() if dt_emp[\"almost_empty\"][key] == True]\ndata = data.drop(cols_drop)\n# data[\"test\"] = data[\"test\"].drop(cols_drop)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:19:59.321858Z","iopub.execute_input":"2024-05-27T15:19:59.322284Z","iopub.status.idle":"2024-05-27T15:20:00.153034Z","shell.execute_reply.started":"2024-05-27T15:19:59.322247Z","shell.execute_reply":"2024-05-27T15:20:00.151707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:00.154627Z","iopub.execute_input":"2024-05-27T15:20:00.155126Z","iopub.status.idle":"2024-05-27T15:20:00.167951Z","shell.execute_reply.started":"2024-05-27T15:20:00.155086Z","shell.execute_reply":"2024-05-27T15:20:00.166712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cutting = round(data.height/5)\ndata1 = data[0:cutting]\ndata2 = data[cutting:2*cutting]\ndata3 = data[2*cutting:3*cutting]\ndata4 = data[3*cutting:4*cutting]\ndata5 = data[4*cutting:5*cutting]\ndata1.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:00.169967Z","iopub.execute_input":"2024-05-27T15:20:00.170709Z","iopub.status.idle":"2024-05-27T15:20:00.191250Z","shell.execute_reply.started":"2024-05-27T15:20:00.170619Z","shell.execute_reply":"2024-05-27T15:20:00.188882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:00.193058Z","iopub.execute_input":"2024-05-27T15:20:00.193434Z","iopub.status.idle":"2024-05-27T15:20:00.313840Z","shell.execute_reply.started":"2024-05-27T15:20:00.193404Z","shell.execute_reply":"2024-05-27T15:20:00.312393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train test split the data\n# According to this StackOverFlow: https://stackoverflow.com/questions/76443099/how-do-i-do-a-train-and-test-split-in-a-polars-dataframe\n# we can split data using samples \n# There no in-use function for use to use \ndef train_test_split_lazy(\n df: pl.DataFrame, train_fraction: float = 0.75\n) -> tuple[pl.DataFrame, pl.DataFrame]:\n \"\"\"Split polars dataframe into two sets.\n Args:\n df (pl.DataFrame): Dataframe to split\n train_fraction (float, optional): Fraction that goes to train. Defaults to 0.75.\n Returns:\n Tuple[pl.DataFrame, pl.DataFrame]: Tuple of train and test dataframes\n \"\"\"\n df = df.with_columns(pl.all().shuffle(seed=1)).with_row_count()\n df_train = df.filter(pl.col(\"row_nr\") < pl.col(\"row_nr\").max() * train_fraction)\n df_test = df.filter(pl.col(\"row_nr\") >= pl.col(\"row_nr\").max() * train_fraction)\n return df_train, df_test\n","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:00.315602Z","iopub.execute_input":"2024-05-27T15:20:00.316413Z","iopub.status.idle":"2024-05-27T15:20:00.327253Z","shell.execute_reply.started":"2024-05-27T15:20:00.316379Z","shell.execute_reply":"2024-05-27T15:20:00.325696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1 , test1 = train_test_split_lazy(data1)\ntrain2 , test2 = train_test_split_lazy(data2)\ntrain3 , test3 = train_test_split_lazy(data3)\ntrain4 , test4 = train_test_split_lazy(data4)\ntrain5 , test5 = train_test_split_lazy(data5)\n# train , test = train_test_split_lazy(data)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:00.329041Z","iopub.execute_input":"2024-05-27T15:20:00.329407Z","iopub.status.idle":"2024-05-27T15:20:08.419365Z","shell.execute_reply.started":"2024-05-27T15:20:00.329375Z","shell.execute_reply":"2024-05-27T15:20:08.418108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:08.420910Z","iopub.execute_input":"2024-05-27T15:20:08.421617Z","iopub.status.idle":"2024-05-27T15:20:08.441200Z","shell.execute_reply.started":"2024-05-27T15:20:08.421573Z","shell.execute_reply":"2024-05-27T15:20:08.439736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data1\ndel data2\ndel data3\ndel data4\ndel data5\n# del data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:08.443470Z","iopub.execute_input":"2024-05-27T15:20:08.444292Z","iopub.status.idle":"2024-05-27T15:20:09.399103Z","shell.execute_reply.started":"2024-05-27T15:20:08.444253Z","shell.execute_reply":"2024-05-27T15:20:09.398015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting the data into feature and target \nX_train = train.drop(\"target\")\nY_train = train.select(\"target\")\nX_test = test.drop(\"target\")\nY_test = test.select(\"target\")\nX_train.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T09:37:25.766919Z","iopub.execute_input":"2024-05-23T09:37:25.767287Z","iopub.status.idle":"2024-05-23T09:37:25.797054Z","shell.execute_reply.started":"2024-05-23T09:37:25.767244Z","shell.execute_reply":"2024-05-23T09:37:25.796029Z"}}},{"cell_type":"code","source":"def split(dfs):\n X = dfs.drop(\"target\")\n Y = dfs.select(\"target\")\n return X , Y","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:09.400540Z","iopub.execute_input":"2024-05-27T15:20:09.401284Z","iopub.status.idle":"2024-05-27T15:20:09.406574Z","shell.execute_reply.started":"2024-05-27T15:20:09.401244Z","shell.execute_reply":"2024-05-27T15:20:09.405548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(X_train1, Y_train1) = split(train1)\n(X_test1, Y_test1) = split(test1)\n\n(X_train2, Y_train2) = split(train2)\n(X_test2, Y_test2) = split(test2)\n\n(X_train3, Y_train3) = split(train3)\n(X_test3, Y_test3) = split(test3)\n\n(X_train4, Y_train4) = split(train4)\n(X_test4, Y_test4) = split(test4)\n\n(X_train5, Y_train5) = split(train5)\n(X_test5, Y_test5) = split(test5)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:09.408129Z","iopub.execute_input":"2024-05-27T15:20:09.408470Z","iopub.status.idle":"2024-05-27T15:20:09.429629Z","shell.execute_reply.started":"2024-05-27T15:20:09.408441Z","shell.execute_reply":"2024-05-27T15:20:09.428631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_test1.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:09.431093Z","iopub.execute_input":"2024-05-27T15:20:09.432362Z","iopub.status.idle":"2024-05-27T15:20:09.440198Z","shell.execute_reply.started":"2024-05-27T15:20:09.432329Z","shell.execute_reply":"2024-05-27T15:20:09.438938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ---> Define auxiliary functions\n\n# Function for converting object columns of pandas dataframes to category columns\n# [NOTE: if there is a column of the tuple of dataframes that is type \"object\", the\n# respective columns in the that and the other dataframes are converted to the type\n# category.]\n# [NOTE: a column of type \"object\" is such that it has entries of solely type \"str\" or\n# entries of multiple different types (e.g. \"NoneType\" and \"float\").]\n# [NOTE: category columns are columns whose entries pertain to a finite list of text\n# values.]\n# [NOTE: the category \"Unknown\" is added to the list of categories to support the case\n# in which validation and test dataframes have exclusive categories not pertaining to\n# the training dataframe - these exclusive categories which are not supported by the\n# trained model should be replaced by the \"Unknown\" category.]\ndef convert_cols_obj_to_cols_cat(*dfs):\n # List of columns of the tuple of dataframes that are of type \"object\" in at least\n # one of the dataframes\n cols_object = list(set().union(*(df.select_dtypes(include=[\"object\"]).columns for df in dfs)))\n # For each column of dtype \"object\"\n for col in cols_object:\n # For each dataframe of the tuple\n for df in dfs:\n # Convert current column to dtype \"category\"\n df[col] = df[col].astype(\"category\")\n # New categorical dtype whose categories correspond to the ones of the\n # current column and the category \"Unknown\", being ordered\n new_dtype = pd.CategoricalDtype(categories=df[col].cat.categories.to_list() +\n [\"Unknown\"],\n ordered=True)\n # Assign new dtype to current column\n df[col] = df[col].astype(new_dtype)\n return dfs\n\n# Function for converting entries of date columns of a pandas dataframe to ordinals,\n# that is, the number of days, being day 1 of January of year 1 corresponding to the\n# number 1\n# [NOTE: this would make dates to be handled like numbers.]\ndef convert_cols_date_to_cols_ord(df):\n # List of columns of date type\n cols_date = df.select_dtypes(include=[\"datetime64\"]).columns\n # For each column of date type\n for col in cols_date:\n # Convert each date in the current column to the respective ordianal\n # [NOTE: if the entry is missing, let one associate a negative value (-1000) as\n # ordinal.]\n df[col] = df[col].apply(lambda x: x.toordinal() if not\n pd.isnull else -1000)\n return df\n\n\n# Function for converting boolean columns of pandas dataframes to integer columns\ndef convert_cols_bool_to_cols_int(df):\n # List of columns of the pandas dataframe that are of boolean type\n cols_bool = df.select_dtypes(include=[\"bool\"]).columns\n # For each column of type bool\n for col in cols_bool:\n # Convert column to int\n df[col] = df[col].astype(\"int64\")\n return df\n\n# Function for making categories of some pandas dataframe that do not pertain to a\n# reference one be replaced by the category \"Unknown\"\ndef make_cat_excl_unknown(df, df_ref):\n # For each categorical column of the reference pandas dataframe\n for col in df_ref.select_dtypes(include=[\"category\"]).columns:\n # List of categories in the reference pandas dataframe\n cat_ref = df_ref[col].cat.categories.to_list()\n # List of categories in the pandas dataframe of interest\n cat = df[col].cat.categories.to_list()\n # List of common categories\n cat_common = list(set(cat).intersection(cat_ref))\n # List of exclusive categories\n cat_exc = list(set(cat).difference(cat_common))\n # New categorical dtype whose categories correspond to the common ones\n new_dtype = pd.CategoricalDtype(categories=cat_common,\n ordered=True)\n # Replace current column's entries associated with exclusive categories as\n # \"Unknown\"\n df[col] = df[col].replace(to_replace=cat_exc, value=\"Unknown\")\n # Assign the new dtype to the current column\n df[col] = df[col].astype(new_dtype)\n return df\n\n# Convert categorical columns to code ones while saving the respective mapping\ndef covert_cols_cat_to_cols_code(df):\n # Dictionary dictionaries of category codes (keys) and respective category names\n # (values) (a dictionary per feature categorical column)\n # [NOTE: these may be used later to convert the codes to the category names through\n # pandas Series' map method.]\n dt_map = {}\n \n # For each categorical column of the pandas dataframe\n for col in df.select_dtypes(include=[\"category\"]).columns:\n \n # Update dictionary of maps\n dt_map.update({col:\n dict(enumerate(df[col].cat.categories))\n })\n \n # Convert categorical column to a code one\n df[col] = df[col].cat.codes\n\n return (df, dt_map)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:09.442399Z","iopub.execute_input":"2024-05-27T15:20:09.442849Z","iopub.status.idle":"2024-05-27T15:20:09.458932Z","shell.execute_reply.started":"2024-05-27T15:20:09.442809Z","shell.execute_reply":"2024-05-27T15:20:09.457734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train1 = X_train1.to_pandas()\nY_train1 = Y_train1.to_pandas()\nX_test1 = X_test1.to_pandas()\nY_test1 = Y_test1.to_pandas()\n(X_train1, X_test1) = convert_cols_obj_to_cols_cat(\n X_train1, X_test1\n)\n\n# For compatibility reasons, make categories of the feature test dataframes which do not\n# pertain to the the training dataframe be replaced by the category \"Unknown\"\nX_test1 = make_cat_excl_unknown(\n df=X_test1, df_ref=X_train1\n)\n\n# Convert feature categorical columns to integer ones using categories' codes\n# [NOTE: category codes are the indices of the categories in column's categories array.]\n# [NOTE: such conversion is required to compute Shapley values when using SHAP. SHAP\n# cannot handle categorical columns.]\ntrain_cat_map = pd.Series()\ntest_cat_map = pd.Series()\n(X_train1, train_cat_map) = covert_cols_cat_to_cols_code(X_train1)\n(X_test1, test_cat_map) = covert_cols_cat_to_cols_code(X_test1)\n\n# Convert feature boolean columns to integer ones\n# [NOTE: such conversion is required to compute Shapley values when using SHAP. SHAP\n# cannot handle boolean columns.]\nX_train1 = convert_cols_bool_to_cols_int(X_train1)\nX_test1 = convert_cols_bool_to_cols_int(X_test1)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:20:09.460321Z","iopub.execute_input":"2024-05-27T15:20:09.461199Z","iopub.status.idle":"2024-05-27T15:45:16.744134Z","shell.execute_reply.started":"2024-05-27T15:20:09.461166Z","shell.execute_reply":"2024-05-27T15:45:16.742040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train2 = X_train2.to_pandas()\nY_train2 = Y_train2.to_pandas()\nX_test2 = X_test2.to_pandas()\nY_test2 = Y_test2.to_pandas()\n(X_train2, X_test2) = convert_cols_obj_to_cols_cat(\n X_train2, X_test2\n)\n\nX_test2 = make_cat_excl_unknown(\n df=X_test2, df_ref=X_train2\n)\n\n(X_train2, train_cat_map2) = covert_cols_cat_to_cols_code(X_train2)\n(X_test2, test_cat_map2) = covert_cols_cat_to_cols_code(X_test2)\n\n\nX_train2 = convert_cols_bool_to_cols_int(X_train2)\nX_test2 = convert_cols_bool_to_cols_int(X_test2)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:45:16.746039Z","iopub.execute_input":"2024-05-27T15:45:16.746427Z","iopub.status.idle":"2024-05-27T15:58:17.944305Z","shell.execute_reply.started":"2024-05-27T15:45:16.746394Z","shell.execute_reply":"2024-05-27T15:58:17.942908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train3 = X_train3.to_pandas()\nY_train3 = Y_train3.to_pandas()\nX_test3 = X_test3.to_pandas()\nY_test3 = Y_test3.to_pandas()\n(X_train3, X_test3) = convert_cols_obj_to_cols_cat(\n X_train3, X_test3\n)\n\nX_test3 = make_cat_excl_unknown(\n df=X_test3, df_ref=X_train3\n)\n\n(X_train3, train_cat_map) = covert_cols_cat_to_cols_code(X_train3)\n(X_test3, test_cat_map) = covert_cols_cat_to_cols_code(X_test3)\n\n\nX_train3 = convert_cols_bool_to_cols_int(X_train3)\nX_test3 = convert_cols_bool_to_cols_int(X_test3)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T15:58:17.945935Z","iopub.execute_input":"2024-05-27T15:58:17.946320Z","iopub.status.idle":"2024-05-27T16:24:01.303633Z","shell.execute_reply.started":"2024-05-27T15:58:17.946281Z","shell.execute_reply":"2024-05-27T16:24:01.302580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train4 = X_train4.to_pandas()\nY_train4 = Y_train4.to_pandas()\nX_test4 = X_test4.to_pandas()\nY_test4 = Y_test4.to_pandas()\n(X_train4, X_test4) = convert_cols_obj_to_cols_cat(\n X_train4, X_test4\n)\n\nX_test4 = make_cat_excl_unknown(\n df=X_test4, df_ref=X_train4\n)\n\n(X_train4, train_cat_map4) = covert_cols_cat_to_cols_code(X_train4)\n(X_test4, test_cat_map4) = covert_cols_cat_to_cols_code(X_test4)\n\n\nX_train4 = convert_cols_bool_to_cols_int(X_train4)\nX_test4 = convert_cols_bool_to_cols_int(X_test4)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T16:24:01.305158Z","iopub.execute_input":"2024-05-27T16:24:01.306212Z","iopub.status.idle":"2024-05-27T17:17:04.141175Z","shell.execute_reply.started":"2024-05-27T16:24:01.306177Z","shell.execute_reply":"2024-05-27T17:17:04.139557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train5 = X_train5.to_pandas()\nY_train5 = Y_train5.to_pandas()\nX_test5 = X_test5.to_pandas()\nY_test5 = Y_test5.to_pandas()\n(X_train5, X_test5) = convert_cols_obj_to_cols_cat(\n X_train5, X_test5\n)\n\nX_test5 = make_cat_excl_unknown(\n df=X_test5, df_ref=X_train5\n)\n\n(X_train5, train_cat_map5) = covert_cols_cat_to_cols_code(X_train5)\n(X_test5, test_cat_map5) = covert_cols_cat_to_cols_code(X_test5)\n\n\nX_train5 = convert_cols_bool_to_cols_int(X_train5)\nX_test5 = convert_cols_bool_to_cols_int(X_test5)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T17:17:04.144381Z","iopub.execute_input":"2024-05-27T17:17:04.144899Z","iopub.status.idle":"2024-05-27T18:01:27.622348Z","shell.execute_reply.started":"2024-05-27T17:17:04.144855Z","shell.execute_reply":"2024-05-27T18:01:27.621153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = pd.concat([X_train1, X_train2,X_train3,X_train4,X_train5])\nY_train = pd.concat([Y_train1, Y_train2,Y_train3,Y_train4,Y_train5])","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:01:27.624066Z","iopub.execute_input":"2024-05-27T18:01:27.624423Z","iopub.status.idle":"2024-05-27T18:01:32.943942Z","shell.execute_reply.started":"2024-05-27T18:01:27.624390Z","shell.execute_reply":"2024-05-27T18:01:32.942718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train.to_csv(\"feature.csv\", index=False)\nY_train.to_csv(\"target.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:01:32.945566Z","iopub.execute_input":"2024-05-27T18:01:32.946020Z","iopub.status.idle":"2024-05-27T18:01:33.453522Z","shell.execute_reply.started":"2024-05-27T18:01:32.945978Z","shell.execute_reply":"2024-05-27T18:01:33.452098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = pd.concat([X_test1, X_test2,X_test3,X_test4,X_test5])\nY_test = pd.concat([Y_test1, Y_test2,Y_test3,Y_test4,Y_test5])","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:01:33.455202Z","iopub.execute_input":"2024-05-27T18:01:33.455676Z","iopub.status.idle":"2024-05-27T18:01:35.330965Z","shell.execute_reply.started":"2024-05-27T18:01:33.455613Z","shell.execute_reply":"2024-05-27T18:01:35.329612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import HistGradientBoostingClassifier\nfrom sklearn.datasets import load_iris\nclf = HistGradientBoostingClassifier(class_weight='balanced').fit(X_train, Y_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:01:35.332764Z","iopub.execute_input":"2024-05-27T18:01:35.333215Z","iopub.status.idle":"2024-05-27T18:04:25.630486Z","shell.execute_reply.started":"2024-05-27T18:01:35.333174Z","shell.execute_reply":"2024-05-27T18:04:25.629030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predicted = clf.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:04:25.632131Z","iopub.execute_input":"2024-05-27T18:04:25.633170Z","iopub.status.idle":"2024-05-27T18:04:29.121824Z","shell.execute_reply.started":"2024-05-27T18:04:25.633131Z","shell.execute_reply":"2024-05-27T18:04:29.118707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, accuracy_score, roc_auc_score\ncm_hgb = confusion_matrix(Y_test, y_predicted)\nprint(cm_hgb)\nfrom mlxtend.plotting import plot_confusion_matrix\nfig, ax = plot_confusion_matrix(conf_mat=cm_hgb, figsize=(6, 6), cmap=plt.cm.Greens)\nplt.xlabel('Predictions', fontsize=18)\nplt.ylabel('Actuals', fontsize=18)\nplt.title('Confusion Matrix', fontsize=18)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:04:29.126900Z","iopub.execute_input":"2024-05-27T18:04:29.128392Z","iopub.status.idle":"2024-05-27T18:04:29.642099Z","shell.execute_reply.started":"2024-05-27T18:04:29.128286Z","shell.execute_reply":"2024-05-27T18:04:29.640909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import cross_val_score\naccuracy_score(Y_test, y_predicted)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:04:29.643682Z","iopub.execute_input":"2024-05-27T18:04:29.644803Z","iopub.status.idle":"2024-05-27T18:04:29.710167Z","shell.execute_reply.started":"2024-05-27T18:04:29.644761Z","shell.execute_reply":"2024-05-27T18:04:29.708390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc_score(Y_test, y_predicted)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:04:29.715140Z","iopub.execute_input":"2024-05-27T18:04:29.717714Z","iopub.status.idle":"2024-05-27T18:04:29.880443Z","shell.execute_reply.started":"2024-05-27T18:04:29.717672Z","shell.execute_reply":"2024-05-27T18:04:29.878696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_train1,X_train2,X_train3,X_train4,X_train5\ndel Y_train1,Y_train2,Y_train3,Y_train4,Y_train5\ndel X_test1,X_test2,X_test3,X_test4,X_test5\ndel Y_test1,Y_test2,Y_test3,Y_test4,Y_test5\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:04:29.882145Z","iopub.execute_input":"2024-05-27T18:04:29.882900Z","iopub.status.idle":"2024-05-27T18:04:30.319269Z","shell.execute_reply.started":"2024-05-27T18:04:29.882864Z","shell.execute_reply":"2024-05-27T18:04:30.316964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# fit xgboost on an imbalanced classification dataset\nfrom numpy import mean\nfrom sklearn.datasets import make_classification\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import RepeatedStratifiedKFold\nfrom xgboost import XGBClassifier\n# define model\nmodel = XGBClassifier()\n# define evaluation procedure\ncv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n# evaluate model\nscores = cross_val_score(model, X_train, Y_train, scoring='roc_auc', cv=cv, n_jobs=-1)\n# summarize performance\nprint('Mean ROC AUC: %.5f' % mean(scores))","metadata":{"execution":{"iopub.status.busy":"2024-05-24T11:57:06.033327Z","iopub.execute_input":"2024-05-24T11:57:06.034741Z","iopub.status.idle":"2024-05-24T11:57:57.442786Z","shell.execute_reply.started":"2024-05-24T11:57:06.034690Z","shell.execute_reply":"2024-05-24T11:57:57.440992Z"}}},{"cell_type":"code","source":"# from collections import Counter\n# counter = Counter(Y_train)\n# # estimate scale_pos_weight value\n# estimate = counter[0] / counter[1]\n# print('Estimate: %.3f' % estimate)\n# # define model\n# from xgboost import XGBClassifier\n# model = XGBClassifier(scale_pos_weight=estimate)\n# # model = XGBClassifier() \n# eval_set=[(X_train, Y_train), (X_test, Y_test)]\n# model.fit(X_train, Y_train, eval_metric=\"auc\", eval_set=eval_set, verbose=True)\nfrom xgboost import XGBClassifier\nmodel = XGBClassifier(scale_pos_weight=31)#Loook at above diagram if you confuse \neval_set=[(X_train, Y_train), (X_test, Y_test)]\nmodel.fit(X_train, Y_train, eval_metric=\"auc\", eval_set=eval_set, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:04:30.323210Z","iopub.execute_input":"2024-05-27T18:04:30.324559Z","iopub.status.idle":"2024-05-27T18:50:03.485971Z","shell.execute_reply.started":"2024-05-27T18:04:30.324422Z","shell.execute_reply":"2024-05-27T18:50:03.484590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:03.488255Z","iopub.execute_input":"2024-05-27T18:50:03.488779Z","iopub.status.idle":"2024-05-27T18:50:05.566426Z","shell.execute_reply.started":"2024-05-27T18:50:03.488734Z","shell.execute_reply":"2024-05-27T18:50:05.565337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, accuracy_score, roc_auc_score\ncm_hgb = confusion_matrix(Y_test, y_pred)\nprint(cm_hgb)\nfrom mlxtend.plotting import plot_confusion_matrix\nfig, ax = plot_confusion_matrix(conf_mat=cm_hgb, figsize=(6, 6), cmap=plt.cm.Greens)\nplt.xlabel('Predictions', fontsize=18)\nplt.ylabel('Actuals', fontsize=18)\nplt.title('Confusion Matrix', fontsize=18)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:05.567993Z","iopub.execute_input":"2024-05-27T18:50:05.568668Z","iopub.status.idle":"2024-05-27T18:50:05.838810Z","shell.execute_reply.started":"2024-05-27T18:50:05.568621Z","shell.execute_reply":"2024-05-27T18:50:05.837076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import cross_val_score\naccuracy_score(Y_test, y_pred)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:05.840258Z","iopub.execute_input":"2024-05-27T18:50:05.841089Z","iopub.status.idle":"2024-05-27T18:50:05.890979Z","shell.execute_reply.started":"2024-05-27T18:50:05.841040Z","shell.execute_reply":"2024-05-27T18:50:05.889742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc_score(Y_test, y_pred)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:05.892542Z","iopub.execute_input":"2024-05-27T18:50:05.893033Z","iopub.status.idle":"2024-05-27T18:50:05.994517Z","shell.execute_reply.started":"2024-05-27T18:50:05.892990Z","shell.execute_reply":"2024-05-27T18:50:05.993153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/\"\n# Xử lý base và lọc các cột dữ liệu thừa\ntest_basetable = pl.read_csv(dataPath + \"test/test_base.csv\").pipe(set_table_dtypes)\n# Chúng ta không sài các chỉ số MONTH ,WEEK_NUM và date_decision nên bỏ luôn những cột đó\ndata = test_basetable.drop(['MONTH', 'WEEK_NUM', 'date_decision'])\ndel test_basetable\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:05.996039Z","iopub.execute_input":"2024-05-27T18:50:05.996439Z","iopub.status.idle":"2024-05-27T18:50:06.021602Z","shell.execute_reply.started":"2024-05-27T18:50:05.996407Z","shell.execute_reply":"2024-05-27T18:50:06.020589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev = pl.concat(\n [\n pl.read_csv(dataPath + \"test/test_applprev_1_0.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_applprev_1_1.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_applprev_1_2.csv\").pipe(set_table_dtypes),\n ],\n how=\"vertical_relaxed\",\n)\ntest_applprev.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.023830Z","iopub.execute_input":"2024-05-27T18:50:06.025173Z","iopub.status.idle":"2024-05-27T18:50:06.089006Z","shell.execute_reply.started":"2024-05-27T18:50:06.025122Z","shell.execute_reply":"2024-05-27T18:50:06.087999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(test_applprev)\ntest_applprev = meanDatamodeString(test_applprev)\n# test_applprev.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.090283Z","iopub.execute_input":"2024-05-27T18:50:06.090803Z","iopub.status.idle":"2024-05-27T18:50:06.101515Z","shell.execute_reply.started":"2024-05-27T18:50:06.090773Z","shell.execute_reply":"2024-05-27T18:50:06.099830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_2 = pl.read_csv(dataPath + \"test/test_applprev_2.csv\").pipe(set_table_dtypes)\n# train_person_1 = pl.read_csv(dataPath + \"csv/train/train_person_1.csv\").pipe(set_table_dtypes) \n# train_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv/train/train_credit_bureau_b_2.csv\").pipe(set_table_dtypes) \ntest_applprev_2 = test_applprev_2.drop(['cacccardblochreas_147M', 'credacc_cards_status_52L', 'num_group1','num_group2'])\ntest_applprev_2.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.102968Z","iopub.execute_input":"2024-05-27T18:50:06.103352Z","iopub.status.idle":"2024-05-27T18:50:06.126219Z","shell.execute_reply.started":"2024-05-27T18:50:06.103320Z","shell.execute_reply":"2024-05-27T18:50:06.125014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_2 = pl.DataFrame(test_applprev_2)\ntest_applprev_2 = test_applprev_2.groupby(\"case_id\").agg(pl.col(\"conts_type_509L\").drop_nulls().mode())\ntest_applprev_2.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.128249Z","iopub.execute_input":"2024-05-27T18:50:06.128862Z","iopub.status.idle":"2024-05-27T18:50:06.142251Z","shell.execute_reply.started":"2024-05-27T18:50:06.128623Z","shell.execute_reply":"2024-05-27T18:50:06.141098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n test_applprev, how=\"left\", on=\"case_id\"\n).join(\n test_applprev_2, how=\"left\", on=\"case_id\"\n)\ndel test_applprev\ndel test_applprev_2\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.143940Z","iopub.execute_input":"2024-05-27T18:50:06.145019Z","iopub.status.idle":"2024-05-27T18:50:06.251336Z","shell.execute_reply.started":"2024-05-27T18:50:06.144975Z","shell.execute_reply":"2024-05-27T18:50:06.249992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_a_1 = pl.concat(\n [\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_1_4.csv\").pipe(set_table_dtypes),\n ],\n how=\"vertical_relaxed\",\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.252718Z","iopub.execute_input":"2024-05-27T18:50:06.253094Z","iopub.status.idle":"2024-05-27T18:50:06.312239Z","shell.execute_reply.started":"2024-05-27T18:50:06.253065Z","shell.execute_reply":"2024-05-27T18:50:06.310970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(test_credit_bureau_a_1)\ntest_credit_bureau_a_1 = meanDatamodeString(test_credit_bureau_a_1)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.313741Z","iopub.execute_input":"2024-05-27T18:50:06.314100Z","iopub.status.idle":"2024-05-27T18:50:06.325523Z","shell.execute_reply.started":"2024-05-27T18:50:06.314071Z","shell.execute_reply":"2024-05-27T18:50:06.324515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n test_credit_bureau_a_1, how=\"left\", on=\"case_id\"\n)\n\ndel test_credit_bureau_a_1\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.327195Z","iopub.execute_input":"2024-05-27T18:50:06.328585Z","iopub.status.idle":"2024-05-27T18:50:06.352998Z","shell.execute_reply.started":"2024-05-27T18:50:06.328544Z","shell.execute_reply":"2024-05-27T18:50:06.351757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_a_2 = pl.concat(\n [\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_0.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_1.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_2.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_3.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_4.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_5.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_6.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_7.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_8.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_9.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_10.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_a_2_11.csv\").pipe(set_table_dtypes),\n ],\n how=\"vertical_relaxed\",\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.354449Z","iopub.execute_input":"2024-05-27T18:50:06.355583Z","iopub.status.idle":"2024-05-27T18:50:06.463998Z","shell.execute_reply.started":"2024-05-27T18:50:06.355534Z","shell.execute_reply":"2024-05-27T18:50:06.462574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(test_credit_bureau_a_2)\ntest_credit_bureau_a_2_1 = meanDatamodeString(test_credit_bureau_a_2)\ntest_credit_bureau_a_2_1.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.476132Z","iopub.execute_input":"2024-05-27T18:50:06.477164Z","iopub.status.idle":"2024-05-27T18:50:06.490276Z","shell.execute_reply.started":"2024-05-27T18:50:06.477118Z","shell.execute_reply":"2024-05-27T18:50:06.489078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_b = pl.concat(\n [\n pl.read_csv(dataPath + \"test/test_credit_bureau_b_1.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes),\n ],\n how=\"diagonal\",\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.491767Z","iopub.execute_input":"2024-05-27T18:50:06.492118Z","iopub.status.idle":"2024-05-27T18:50:06.509117Z","shell.execute_reply.started":"2024-05-27T18:50:06.492080Z","shell.execute_reply":"2024-05-27T18:50:06.508056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(test_credit_bureau_b)\ntest_credit_bureau_b = meanDatamodeString(test_credit_bureau_b)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.511844Z","iopub.execute_input":"2024-05-27T18:50:06.512647Z","iopub.status.idle":"2024-05-27T18:50:06.522061Z","shell.execute_reply.started":"2024-05-27T18:50:06.512600Z","shell.execute_reply":"2024-05-27T18:50:06.520881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n test_credit_bureau_b, how=\"left\", on=\"case_id\"\n)\ndel test_credit_bureau_b","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.523519Z","iopub.execute_input":"2024-05-27T18:50:06.523933Z","iopub.status.idle":"2024-05-27T18:50:06.533709Z","shell.execute_reply.started":"2024-05-27T18:50:06.523901Z","shell.execute_reply":"2024-05-27T18:50:06.532634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_debitcard_1 = pl.read_csv(dataPath + \"test/test_debitcard_1.csv\").pipe(set_table_dtypes)\n\ntest_deposit_1 = pl.read_csv(dataPath + \"test/test_deposit_1.csv\").pipe(set_table_dtypes)\n\ntest_other_1 = pl.read_csv(dataPath + \"test/test_other_1.csv\").pipe(set_table_dtypes)\n\ntest_person_1 = pl.read_csv(dataPath + \"test/test_person_1.csv\").pipe(set_table_dtypes)\n\ntest_person_2 = pl.read_csv(dataPath + \"test/test_person_2.csv\").pipe(set_table_dtypes)\n\ntest_static = pl.concat(\n [\n pl.read_csv(dataPath + \"test/test_static_0_0.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_static_0_1.csv\").pipe(set_table_dtypes),\n pl.read_csv(dataPath + \"test/test_static_0_2.csv\").pipe(set_table_dtypes),\n ],\n how=\"vertical_relaxed\",\n)\n\ntest_static_cb = pl.read_csv(dataPath + \"test/test_static_cb_0.csv\").pipe(set_table_dtypes)\n\ntest_tax_registry_a_1 = pl.read_csv(dataPath + \"test/test_tax_registry_a_1.csv\").pipe(set_table_dtypes)\n\ntest_tax_registry_b_1 = pl.read_csv(dataPath + \"test/test_tax_registry_b_1.csv\").pipe(set_table_dtypes)\n\ntest_tax_registry_c_1 = pl.read_csv(dataPath + \"test/test_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.535356Z","iopub.execute_input":"2024-05-27T18:50:06.535863Z","iopub.status.idle":"2024-05-27T18:50:06.645989Z","shell.execute_reply.started":"2024-05-27T18:50:06.535829Z","shell.execute_reply":"2024-05-27T18:50:06.644722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformDate(test_debitcard_1)\ntransformDate(test_deposit_1)\ntest_debitcard_1 = meanDatamodeString(test_debitcard_1)\ntest_deposit_1 = meanDatamodeString(test_deposit_1)\n\ntransformDate(test_other_1)\ntransformDate(test_static)\ntest_other_1 = meanDatamodeString(test_other_1)\ntest_static = meanDatamodeString(test_static)\n\ntransformDate(test_person_1)\ntransformDate(test_person_2)\ntest_person_1 = meanDatamodeString(test_person_1)\ntest_person_2 = meanDatamodeString(test_person_2)\n\ntransformDate(test_static_cb)\ntransformDate(test_tax_registry_a_1)\ntrain_static_cb = meanDatamodeString(test_static_cb)\ntrain_tax_registry_a_1 = meanDatamodeString(test_tax_registry_a_1)\n\ntransformDate(test_tax_registry_b_1)\ntransformDate(test_tax_registry_c_1)\ntrain_tax_registry_b_1 = meanDatamodeString(test_tax_registry_b_1)\ntrain_tax_registry_c_1 = meanDatamodeString(test_tax_registry_c_1)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.647791Z","iopub.execute_input":"2024-05-27T18:50:06.648259Z","iopub.status.idle":"2024-05-27T18:50:06.736164Z","shell.execute_reply.started":"2024-05-27T18:50:06.648217Z","shell.execute_reply":"2024-05-27T18:50:06.734739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n test_debitcard_1, how=\"left\", on=\"case_id\"\n).join(\n test_deposit_1, how=\"left\", on=\"case_id\"\n).join(\n test_other_1, how=\"left\", on=\"case_id\"\n).join(\n test_static, how=\"left\", on=\"case_id\"\n).join(\n test_person_1, how=\"left\", on=\"case_id\"\n)\n# .join(\n# test_person_2, how=\"left\", on=\"case_id\"\n# )\ndata = pl.concat([data, test_person_2], how=\"diagonal_relaxed\")\ndel test_person_1\ndel test_person_2\ndel test_debitcard_1\ndel test_deposit_1\ndel test_other_1\ndel test_static\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.737292Z","iopub.execute_input":"2024-05-27T18:50:06.737698Z","iopub.status.idle":"2024-05-27T18:50:06.865309Z","shell.execute_reply.started":"2024-05-27T18:50:06.737627Z","shell.execute_reply":"2024-05-27T18:50:06.863866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n test_static_cb, how=\"left\", on=\"case_id\"\n).join(\n test_tax_registry_a_1, how=\"left\", on=\"case_id\"\n).join(\n test_tax_registry_b_1, how=\"left\", on=\"case_id\"\n)\n# .join(\n# test_tax_registry_c_1, how=\"left\", on=\"case_id\"\n# )\ndata = pl.concat([data, test_tax_registry_c_1], how=\"diagonal_relaxed\")\ndel test_static_cb\ndel test_tax_registry_a_1\ndel test_tax_registry_b_1\ndel test_tax_registry_c_1\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.866725Z","iopub.execute_input":"2024-05-27T18:50:06.867086Z","iopub.status.idle":"2024-05-27T18:50:06.919140Z","shell.execute_reply.started":"2024-05-27T18:50:06.867057Z","shell.execute_reply":"2024-05-27T18:50:06.917861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.to_pandas()\ndata.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.920372Z","iopub.execute_input":"2024-05-27T18:50:06.920751Z","iopub.status.idle":"2024-05-27T18:50:06.954787Z","shell.execute_reply.started":"2024-05-27T18:50:06.920719Z","shell.execute_reply":"2024-05-27T18:50:06.953893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.956415Z","iopub.execute_input":"2024-05-27T18:50:06.956888Z","iopub.status.idle":"2024-05-27T18:50:06.993716Z","shell.execute_reply.started":"2024-05-27T18:50:06.956857Z","shell.execute_reply":"2024-05-27T18:50:06.992766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data = convert_cols_obj_to_cols_cat(data)\n\n# # # data = make_cat_excl_unknown(\n# # # df=data, df_ref=data\n# # # )\n\n# # Dictionary dictionaries of category codes (keys) and respective category names\n# # (values) (a dictionary per feature categorical column)\n# # [NOTE: these may be used later to convert the codes to the category names through\n# # pandas Series' map method.]\n# dt_map = {}\n\n# # For each categorical column of the pandas dataframe\n# for col in data.select_dtypes(include=[\"category\"]).columns:\n\n# # Update dictionary of maps\n# dt_map.update({col:\n# dict(enumerate(data[col].cat.categories))\n# })\n\n# # Convert categorical column to a code one\n# data[col] = data[col].cat.codes\n\n\n# data = convert_cols_bool_to_cols_int(data)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:06.995124Z","iopub.execute_input":"2024-05-27T18:50:06.995731Z","iopub.status.idle":"2024-05-27T18:50:09.537021Z","shell.execute_reply.started":"2024-05-27T18:50:06.995689Z","shell.execute_reply":"2024-05-27T18:50:09.534729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# result = clf.predict(data)","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:09.538376Z","iopub.status.idle":"2024-05-27T18:50:09.538871Z","shell.execute_reply.started":"2024-05-27T18:50:09.538638Z","shell.execute_reply":"2024-05-27T18:50:09.538672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = pd.DataFrame({\n# \"case_id\": data[\"case_id\"].to_numpy(),\n# \"score\": result\n# }).set_index('case_id')\n# submission.to_csv(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:09.541802Z","iopub.status.idle":"2024-05-27T18:50:09.542921Z","shell.execute_reply.started":"2024-05-27T18:50:09.542558Z","shell.execute_reply":"2024-05-27T18:50:09.542589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # X_train.head()\n# from collections import Counter\n# counter = Counter(Y_train)\n# # estimate scale_pos_weight value\n# estimate = counter[0] / counter[1]\n# print('Estimate: %.3f' % estimate)\n# # define model\n# model = XGBClassifier(scale_pos_weight=estimate)\n# # define evaluation procedure\n# cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n# # evaluate model\n# scores = cross_val_score(model, X_train, Y_train, scoring='roc_auc', cv=cv, n_jobs=-1)\n# # summarize performance\n# print('Mean ROC AUC: %.5f' % mean(scores))\n","metadata":{"execution":{"iopub.status.busy":"2024-05-27T18:50:09.544804Z","iopub.status.idle":"2024-05-27T18:50:09.545790Z","shell.execute_reply.started":"2024-05-27T18:50:09.545454Z","shell.execute_reply":"2024-05-27T18:50:09.545481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}