{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# suppress enjoying warnings from seaborn\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\n# for fiding file names\nfrom pathlib import Path\nfrom glob import glob\nimport gc\n \n# data processing libraries\nimport polars as pl\nimport numpy as np\nimport pandas as pd\n\n# for visualizing data\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# sklearn functions\nfrom sklearn.model_selection import train_test_split, StratifiedGroupKFold\nfrom sklearn.preprocessing import LabelEncoder\n# for validating models\nfrom sklearn.metrics import roc_auc_score, roc_curve\n\n# LightGBM modeling\nimport lightgbm as lgb\n# Có thể thực hiện huấn luyện mô hình với các cách chọn feature khác nhau\n\n# define default colors for plots in notebook\nfrom matplotlib import cycler\nfrom matplotlib.colors import LinearSegmentedColormap\nCOLORS = [\"#068D9D\", \"#53599A\", \"#607BB0\", \"#6D9DC5\", \"#77BECF\", \"#80DED9\", \"#AEECEF\"]\nplt.rc('axes', facecolor='#E6E6E6', edgecolor='none', axisbelow=True, grid=True, prop_cycle=cycler('color', COLORS))\n\n# project CONSTANTS\nROOT = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR = ROOT / \"parquet_files\" / \"test\"\nSEED = 42","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-16T12:56:16.073514Z","iopub.execute_input":"2024-04-16T12:56:16.074035Z","iopub.status.idle":"2024-04-16T12:56:16.086541Z","shell.execute_reply.started":"2024-04-16T12:56:16.073997Z","shell.execute_reply":"2024-04-16T12:56:16.085248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Thông tin\n- Chọn đọc file parquet vì được cho là cung cấp tốc độ đọc file nhanh hơn\n- Ở đây chưa xử lý các feature dạng T và L\n- Đối với dữ liệu category thì nếu có những nhãn có cùng tần xuất xuất hiện cao nhất thì chọn cái nằm ở numgroup đầu tiên","metadata":{}},{"cell_type":"markdown","source":"\n# 1. Helper functions\n\nMost of the functions borrowed from the notebook https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm .\n\n
\n
\nBack to top\n
","metadata":{}},{"cell_type":"code","source":"class Pipeline:\n \"\"\"\n Helper class taken from notebook:\n https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n \"\"\"\n def set_table_dtypes(df):\n # Thiếu xử lý feature dạng T và L\n # Mấy cái đó nó tự áp kiểu dữ liệu vào luôn ?\n for col in df.columns:\n if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n df = df.with_columns(pl.col(col).cast(pl.Int64))\n elif col in [\"date_decision\"]:\n df = df.with_columns(pl.col(col).cast(pl.Date))\n elif col[-1] in (\"P\", \"A\"):\n df = df.with_columns(pl.col(col).cast(pl.Float64))\n elif col[-1] in (\"M\",):\n df = df.with_columns(pl.col(col).cast(pl.String))\n elif col[-1] in (\"D\",):\n df = df.with_columns(pl.col(col).cast(pl.Date))\n return df\n def sort_df(df, depth):\n # Sắp xếp để hàm first có ý nghĩa\n if depth == 0:\n return df.sort(\"case_id\")\n elif depth == 1:\n return df.sort([\"case_id\", \"num_group1\"])\n else:\n return df.sort([\"case_id\", \"num_group1\", \"num_group2\"])\n \n def handle_dates(df):\n # Lấy số ngày từ date_decision\n # Hàm này được gọi khi mà đã thực hiện các phép tổng hợp xong\n for col in df.columns:\n if col[-1] in (\"D\",):\n df = df.with_columns(pl.col(col) - pl.col(\"date_decision\")) #!!?\n # Hàm này thay đổi giá trị của cột col luôn chứ không thêm cột mới !\n df = df.with_columns(pl.col(col).dt.total_days()) # t - t-1\n df = df.drop(\"date_decision\", \"MONTH\")\n return df\n\n def filter_cols(df):\n # Hàm này được áp dụng khi mà đã thực hiện các hàm tổng hợp !\n for col in df.columns:\n if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n # Bỏ các cột có tỉ lệ null là trên 30%\n isnull = df[col].is_null().mean()\n if isnull > 0.7:\n df = df.drop(col)\n \n for col in df.columns:\n if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n # Bỏ các cột category có số giá trị unique trên 200\n freq = df[col].n_unique()\n if (freq == 1) | (freq > 200):\n df = df.drop(col)\n \n return df","metadata":{"execution":{"iopub.status.busy":"2024-04-16T12:56:16.089171Z","iopub.execute_input":"2024-04-16T12:56:16.089925Z","iopub.status.idle":"2024-04-16T12:56:16.112301Z","shell.execute_reply.started":"2024-04-16T12:56:16.089863Z","shell.execute_reply":"2024-04-16T12:56:16.110513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Aggregator:\n \"\"\"\n Helper class taken from notebook:\n https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n \"\"\" \n def num_expr(df):\n cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n expr_mean = [pl.median(col).alias(f\"mean_{col}\") for col in cols]\n expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n expr_std = [pl.std(col).alias(f\"std_{col}\") for col in cols]\n return expr_mean + expr_median + expr_max + expr_min + expr_std\n \n def date_expr(df):\n cols = [col for col in df.columns if col[-1] in (\"D\")]\n expr_mean = [pl.median(col).alias(f\"mean_{col}\") for col in cols]\n expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n expr_std = [pl.std(col).alias(f\"std_{col}\") for col in cols]\n return expr_mean + expr_median + expr_max + expr_min + expr_std\n \n def str_expr(df):\n cols = [col for col in df.columns if col[-1] in (\"M\",)]\n \n # Lấy chuỗi có độ dài dài nhất\n expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n return expr_max\n \n def other_expr(df):\n # Chổ này cần xem xét kiểu dữ liệu để áp dụng phù hợp\n cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n expr = []\n for col in cols:\n if df[col].dtype.is_numeric():\n expr += [pl.median(col).alias(f\"mean_{col}\"), \n pl.median(col).alias(f\"median_{col}\"), \n pl.max(col).alias(f\"max_{col}\"), \n pl.min(col).alias(f\"min_{col}\"), \n pl.std(col).alias(f\"std_{col}\")]\n else:\n expr += [pl.std(col).alias(f\"std_{col}\"),\n pl.max(col).alias(f\"max_{col}\"),\n pl.min(col).alias(f\"min_{col}\")]\n return expr\n \n def count_expr(df):\n # Chổ này làm qq gì đây ?\n cols = [col for col in df.columns if \"num_group\" in col]\n# expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols] # max & replace col name\n expr_count = [pl.count(col).alias(f\"count_{col}\") for col in cols]\n expr_n_unique = [pl.n_unique(col).alias(f\"n_unique_{col}\") for col in cols] # max & replace col name\n return expr_count + expr_n_unique\n \n def get_exprs(df):\n exprs = Aggregator.num_expr(df) + \\\n Aggregator.date_expr(df) + \\\n Aggregator.str_expr(df) + \\\n Aggregator.other_expr(df) + \\\n Aggregator.count_expr(df)\n\n return exprs","metadata":{"execution":{"iopub.status.busy":"2024-04-16T12:56:16.115012Z","iopub.execute_input":"2024-04-16T12:56:16.115509Z","iopub.status.idle":"2024-04-16T12:56:16.141615Z","shell.execute_reply.started":"2024-04-16T12:56:16.115462Z","shell.execute_reply":"2024-04-16T12:56:16.140259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path, depth):\n \"\"\"\n Helper function taken from notebook:\n https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n \"\"\"\n df = pl.read_parquet(path)\n df = df.pipe(Pipeline.set_table_dtypes)\n df = Pipeline.sort_df(df, depth)\n if depth in [1,2]:\n df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df)) \n return df","metadata":{"execution":{"iopub.status.busy":"2024-04-16T12:56:16.144354Z","iopub.execute_input":"2024-04-16T12:56:16.145054Z","iopub.status.idle":"2024-04-16T12:56:16.160517Z","shell.execute_reply.started":"2024-04-16T12:56:16.145014Z","shell.execute_reply":"2024-04-16T12:56:16.159064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_files(regex_path, depth):\n \"\"\"\n Helper function taken from notebook:\n https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n \"\"\"\n chunks = []\n \n for path in glob(str(regex_path)):\n df = pl.read_parquet(path)\n df = df.pipe(Pipeline.set_table_dtypes)\n df = Pipeline.sort_df(df, depth)\n if depth in [1, 2]:\n # Nếu group ở đây thì lỡ nó không liên kết nhau thì sao ?\n # 1 giá trị case_id xuất hiện trong cả file_1 và file_2 ?\n df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n # Hợp lý\n chunks.append(df)\n \n df = pl.concat(chunks, how=\"vertical_relaxed\")\n # relaxed: chuyển đổi để cùng kiểu dữ liệu giữa các hàng\n \n # Đây là lệnh dùng để bỏ các giá trị case_id trùng ở trên \n # Nhưng thực hiện như vầy dẫn đến mất dữ liệu\n print(len(df))\n df = df.unique(subset=[\"case_id\"])\n print(len(df))\n return df","metadata":{"execution":{"iopub.status.busy":"2024-04-16T12:56:16.165239Z","iopub.execute_input":"2024-04-16T12:56:16.166059Z","iopub.status.idle":"2024-04-16T12:56:16.175177Z","shell.execute_reply.started":"2024-04-16T12:56:16.166021Z","shell.execute_reply":"2024-04-16T12:56:16.174178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n \"\"\"\n Helper function taken from notebook:\n https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n \"\"\"\n df_base = (\n df_base\n .with_columns(\n month_decision = pl.col(\"date_decision\").dt.month(),\n weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n )\n # thêm 2 cột mới\n )\n # Lấy giá trị ngày và giá trị tuần\n # Join mấy cái file lại với nhau\n for i, df in enumerate(depth_0 + depth_1 + depth_2):\n # Left join ở đây nên sẽ có các column mới\n df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n # Có thể ảnh hưởng từ left join\n df_base = df_base.pipe(Pipeline.handle_dates)\n return df_base","metadata":{"execution":{"iopub.status.busy":"2024-04-16T12:56:16.176633Z","iopub.execute_input":"2024-04-16T12:56:16.177238Z","iopub.status.idle":"2024-04-16T12:56:16.194713Z","shell.execute_reply.started":"2024-04-16T12:56:16.177206Z","shell.execute_reply":"2024-04-16T12:56:16.193225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n # Danh sách cột Category: cat_cols\n \"\"\"\n Helper function taken from notebook:\n https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n \"\"\"\n df_data = df_data.to_pandas()\n if cat_cols is None:\n cat_cols = list(df_data.select_dtypes(\"object\").columns)\n df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-04-16T12:56:16.196548Z","iopub.execute_input":"2024-04-16T12:56:16.198563Z","iopub.status.idle":"2024-04-16T12:56:16.206502Z","shell.execute_reply.started":"2024-04-16T12:56:16.198495Z","shell.execute_reply":"2024-04-16T12:56:16.205249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, df_name, verbose=True):\n \"\"\"\n This function changes pandas numerical dtypes (reduces bit size if possible)\n to reduce memory usage\n \n :param df: pandas DataFrame\n :param df_name: str, name of DataFrame\n :param verbose: bool, if True prints out message of how much memory usage was reduced\n \n :return: pandas DataFrame \n \"\"\"\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n start_mem = df.memory_usage().sum() / 1024**2\n for col in df.columns:\n col_type = df[col].dtypes\n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int64)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64)\n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float32)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float64)\n else:\n df[col] = df[col].astype(np.float64)\n # calculate memory after reduction\n end_mem = df.memory_usage().sum() / 1024**2\n if verbose:\n # reduced memory usage in percent\n diff_pst = 100 * (start_mem - end_mem) / start_mem\n msg = f'{df_name} mem. usage decreased to {end_mem:5.2f} Mb ({diff_pst:.1f}% reduction)'\n print(msg)\n return df","metadata":{"execution":{"iopub.status.busy":"2024-04-16T12:56:16.208457Z","iopub.execute_input":"2024-04-16T12:56:16.209023Z","iopub.status.idle":"2024-04-16T12:56:16.227199Z","shell.execute_reply.started":"2024-04-16T12:56:16.208979Z","shell.execute_reply":"2024-04-16T12:56:16.225451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# 2. Load data\n\n
\n
\nBack to top\n
","metadata":{}},{"cell_type":"code","source":"%%time\ndata_train = {\n \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\", 0),\n \"depth_0\": [\n read_file(TRAIN_DIR / \"train_static_cb_0.parquet\", 0),\n read_files(TRAIN_DIR / \"train_static_0_*.parquet\", 0),\n ],\n \"depth_1\": [\n read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n ],\n \"depth_2\": [\n read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n read_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-04-16T12:56:16.229442Z","iopub.execute_input":"2024-04-16T12:56:16.230268Z","iopub.status.idle":"2024-04-16T13:02:29.055408Z","shell.execute_reply.started":"2024-04-16T12:56:16.230227Z","shell.execute_reply":"2024-04-16T13:02:29.053681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_test = {\n \"df_base\": read_file(TEST_DIR / \"test_base.parquet\", 0),\n \"depth_0\": [\n read_file(TEST_DIR / \"test_static_cb_0.parquet\", 0),\n read_files(TEST_DIR / \"test_static_0_*.parquet\", 0),\n ],\n \"depth_1\": [\n read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n ],\n \"depth_2\": [\n read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:02:29.063756Z","iopub.execute_input":"2024-04-16T13:02:29.064287Z","iopub.status.idle":"2024-04-16T13:02:29.494735Z","shell.execute_reply.started":"2024-04-16T13:02:29.064248Z","shell.execute_reply":"2024-04-16T13:02:29.493597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lấy danh sách các cột\n# get column names of original/raw features\nFEATS_ORIG = []\n\n# get column names\nfor _key in data_train.keys():\n if isinstance(data_train[_key], list):\n for _df in data_train[_key]:\n FEATS_ORIG += _df.columns\n \n# leave only unique values\nFEATS_ORIG = list(set(FEATS_ORIG))\n# drop case_id column\nFEATS_ORIG.remove(\"case_id\")","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:02:29.496333Z","iopub.execute_input":"2024-04-16T13:02:29.496778Z","iopub.status.idle":"2024-04-16T13:02:29.504569Z","shell.execute_reply.started":"2024-04-16T13:02:29.496738Z","shell.execute_reply":"2024-04-16T13:02:29.503343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(FEATS_ORIG))\nprint(FEATS_ORIG[:5])","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:02:29.506376Z","iopub.execute_input":"2024-04-16T13:02:29.506982Z","iopub.status.idle":"2024-04-16T13:02:29.519215Z","shell.execute_reply.started":"2024-04-16T13:02:29.506939Z","shell.execute_reply":"2024-04-16T13:02:29.517829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# 3. Feature engineering\n\n
\n
\nBack to top\n
","metadata":{}},{"cell_type":"code","source":"%%time\n# Join các table lại với nhau\n# Lấy giá trị tháng và giá trị tuần từ date_decision thành week_decision và month_decision\n# Trừ giá trị ngày trong các cột D với giá trị của cột date_decision\n# Vậy giá trị week_decision và month_decision dùng làm gì ?\ndf_train = feature_eng(**data_train)\nprint(\"train data shape:\\t\", df_train.shape)\n# clean memory\ndel data_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:02:29.521213Z","iopub.execute_input":"2024-04-16T13:02:29.521742Z","iopub.status.idle":"2024-04-16T13:03:06.985845Z","shell.execute_reply.started":"2024-04-16T13:02:29.521698Z","shell.execute_reply":"2024-04-16T13:03:06.984666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_test = feature_eng(**data_test)\nprint(\"test data shape:\\t\", df_test.shape)\n# clean memory\ndel data_test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:03:06.987473Z","iopub.execute_input":"2024-04-16T13:03:06.987800Z","iopub.status.idle":"2024-04-16T13:03:07.341443Z","shell.execute_reply.started":"2024-04-16T13:03:06.987773Z","shell.execute_reply":"2024-04-16T13:03:07.340141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get column names of new created features\nFEATS_NEW = set(df_train.columns)\n# remove speific columns\nFEATS_NEW = FEATS_NEW.difference({'case_id', 'WEEK_NUM', 'target'})\nFEATS_NEW = FEATS_NEW.difference(set(FEATS_ORIG))\nFEATS_NEW = list(FEATS_NEW)\n\nprint(f\"Before feature engineering: {len(FEATS_ORIG)} featues.\")\nprint(f\"{len(FEATS_NEW)} new features created.\")","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:03:07.342801Z","iopub.execute_input":"2024-04-16T13:03:07.343186Z","iopub.status.idle":"2024-04-16T13:03:07.354133Z","shell.execute_reply.started":"2024-04-16T13:03:07.343156Z","shell.execute_reply":"2024-04-16T13:03:07.352624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(FEATS_NEW)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:03:07.355861Z","iopub.execute_input":"2024-04-16T13:03:07.356623Z","iopub.status.idle":"2024-04-16T13:03:07.374413Z","shell.execute_reply.started":"2024-04-16T13:03:07.356592Z","shell.execute_reply":"2024-04-16T13:03:07.372689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop the insignificant features\ndf_train = df_train.pipe(Pipeline.filter_cols)\n# Bỏ cột target ?\ndf_test = df_test.select([col for col in df_train.columns if col != \"target\"])\n\nprint(\"train data shape: \", df_train.shape)\nprint(\"test data shape: \", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:03:07.378383Z","iopub.execute_input":"2024-04-16T13:03:07.379383Z","iopub.status.idle":"2024-04-16T13:03:12.971997Z","shell.execute_reply.started":"2024-04-16T13:03:07.379278Z","shell.execute_reply":"2024-04-16T13:03:12.970794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get column names of the remaining features\nFEATS_REMAIN = set(df_train.columns)\n# remove speific columns\nFEATS_REMAIN = FEATS_REMAIN.difference({'case_id', 'WEEK_NUM', 'target'})\n# remaining in original features\n_1 = [_ for _ in FEATS_REMAIN if _ in FEATS_ORIG]\n# remaining of the new features\n_2 = [_ for _ in FEATS_REMAIN if _ in FEATS_NEW]\n\nprint(\"After removing insignificant features:\")\nprint(f\"{len(_1)} original features left\") # Số feature còn lại trong feature gốc\nprint(f\"{len(_2)} new features left\") # Số feature còn lại trong feature mới","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:03:12.973548Z","iopub.execute_input":"2024-04-16T13:03:12.973905Z","iopub.status.idle":"2024-04-16T13:03:12.989568Z","shell.execute_reply.started":"2024-04-16T13:03:12.973862Z","shell.execute_reply":"2024-04-16T13:03:12.988365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# convert back to pandas\ndf_train, cat_cols = to_pandas(df_train)\ndf_test, cat_cols = to_pandas(df_test, cat_cols)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:03:12.991181Z","iopub.execute_input":"2024-04-16T13:03:12.991611Z","iopub.status.idle":"2024-04-16T13:03:44.557723Z","shell.execute_reply.started":"2024-04-16T13:03:12.991564Z","shell.execute_reply":"2024-04-16T13:03:44.556213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# reduce memory usage if available\ndf_train = reduce_mem_usage(df_train, \"df_train\")\ndf_test = reduce_mem_usage(df_test, \"df_test\")","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:03:44.559360Z","iopub.execute_input":"2024-04-16T13:03:44.559700Z","iopub.status.idle":"2024-04-16T13:04:04.423414Z","shell.execute_reply.started":"2024-04-16T13:03:44.559671Z","shell.execute_reply":"2024-04-16T13:04:04.421284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# 4. Describe features\n\n\n## 4.1. Numerical features\n\n
\n
\nBack to top\n
","metadata":{}},{"cell_type":"code","source":"%%time\n# select numeric columns\n# 3 cái đầu là: 'case_id', 'WEEK_NUM', 'target'\nnum_cols = df_train.iloc[:, 3:].select_dtypes(include='number').columns\n# get column names without NaN values\n_ = df_train[num_cols].isna().sum()\n# get numerical column names without NaN\nnum_cols_no_nan = _[_==0].index.tolist()\nnum_cols_nan = _[_>0].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:04:04.426817Z","iopub.execute_input":"2024-04-16T13:04:04.427447Z","iopub.status.idle":"2024-04-16T13:04:27.562193Z","shell.execute_reply.started":"2024-04-16T13:04:04.427388Z","shell.execute_reply":"2024-04-16T13:04:27.560437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Tổng số cột:', len(num_cols))\nprint('Số cột có nan:', len(num_cols) - len(num_cols_no_nan))\nprint('Số cột không nan:', len(num_cols_no_nan))","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:04:27.564541Z","iopub.execute_input":"2024-04-16T13:04:27.565077Z","iopub.status.idle":"2024-04-16T13:04:27.574399Z","shell.execute_reply.started":"2024-04-16T13:04:27.565031Z","shell.execute_reply":"2024-04-16T13:04:27.572885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get train data set stats\n_df_1 = df_train[num_cols_no_nan].describe().T\n_df_1.drop(columns=['count'], inplace=True)\n_df_1.columns = [f\"train {_}\" for _ in _df_1.columns]\n# get test data set stats\n_df_2 = df_test[num_cols_no_nan].describe().T\n_df_2.drop(columns=['count'], inplace=True)\n_df_2.columns = [f\"test {_}\" for _ in _df_2.columns]\npd.merge(_df_1, _df_2, left_index=True, right_index=True, how=\"outer\").sort_values(by=\"train mean\")","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:04:27.576296Z","iopub.execute_input":"2024-04-16T13:04:27.576793Z","iopub.status.idle":"2024-04-16T13:04:31.561085Z","shell.execute_reply.started":"2024-04-16T13:04:27.576752Z","shell.execute_reply":"2024-04-16T13:04:31.559573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## 4.2. Categorical features\n\nFind features with no missing values in both `train` and `test` data sets.\n\n
\n
\nBack to top\n
","metadata":{}},{"cell_type":"code","source":"%%time\n# select numeric columns\ncat_cols = df_train.iloc[:, 3:].select_dtypes(include='category').columns\n# get column names without NaN values\n_ = df_train[cat_cols].isna().sum()\n# get numerical column names without NaN\ncat_cols_no_nan = _[_==0].index.tolist()\ncat_cols_nan = _[_>0].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:04:31.562631Z","iopub.execute_input":"2024-04-16T13:04:31.563045Z","iopub.status.idle":"2024-04-16T13:04:36.668728Z","shell.execute_reply.started":"2024-04-16T13:04:31.563007Z","shell.execute_reply":"2024-04-16T13:04:36.667242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Tổng số cột:', len(cat_cols))\nprint('Số cột có nan:', len(cat_cols) - len(cat_cols_no_nan))\nprint('Số cột không nan:', len(cat_cols_no_nan))","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:04:36.670819Z","iopub.execute_input":"2024-04-16T13:04:36.671342Z","iopub.status.idle":"2024-04-16T13:04:36.679185Z","shell.execute_reply.started":"2024-04-16T13:04:36.671302Z","shell.execute_reply":"2024-04-16T13:04:36.677580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get train data set stats\n_df_1 = df_train[cat_cols_no_nan].describe().T\n_df_1.drop(columns=['count'], inplace=True)\n_df_1.columns = [f\"train {_}\" for _ in _df_1.columns]\n# get test data set stats\n_df_2 = df_test[cat_cols_no_nan].describe().T\n_df_2.drop(columns=['count'], inplace=True)\n_df_2.columns = [f\"test {_}\" for _ in _df_2.columns]\npd.merge(_df_1, _df_2, left_index=True, right_index=True, how=\"outer\").sort_values(by=\"train unique\")","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:04:36.680781Z","iopub.execute_input":"2024-04-16T13:04:36.681274Z","iopub.status.idle":"2024-04-16T13:04:36.988183Z","shell.execute_reply.started":"2024-04-16T13:04:36.681206Z","shell.execute_reply":"2024-04-16T13:04:36.986575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## 4.2. Nếu thực hiện dropnan","metadata":{}},{"cell_type":"code","source":"print('Trước khi dropnan:', len(df_train))\nprint('Sau khi dropnan:', len(df_train.dropna()))","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:04:36.989813Z","iopub.execute_input":"2024-04-16T13:04:36.990311Z","iopub.status.idle":"2024-04-16T13:04:40.187849Z","shell.execute_reply.started":"2024-04-16T13:04:36.990276Z","shell.execute_reply":"2024-04-16T13:04:40.186304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# 5. Lưu file\n\n
\n
\nBack to top\n
","metadata":{}},{"cell_type":"code","source":"print(len(df_train.columns))\nprint(len(df_test.columns))","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:09:28.772161Z","iopub.execute_input":"2024-04-16T13:09:28.773862Z","iopub.status.idle":"2024-04-16T13:09:28.780727Z","shell.execute_reply.started":"2024-04-16T13:09:28.773823Z","shell.execute_reply":"2024-04-16T13:09:28.779340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.to_csv('train.csv', index=False)\ndf_test.to_csv('test.csv', index=False)\n\ndf_train.to_parquet('train.parquet', index=False)\ndf_test.to_parquet('test.parquet', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T13:04:40.195406Z","iopub.execute_input":"2024-04-16T13:04:40.195805Z","iopub.status.idle":"2024-04-16T13:08:39.912733Z","shell.execute_reply.started":"2024-04-16T13:04:40.195777Z","shell.execute_reply":"2024-04-16T13:08:39.910659Z"},"trusted":true},"execution_count":null,"outputs":[]}]}