{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# suppress enjoying warnings from seaborn\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\n# for fiding file names\nfrom pathlib import Path\nfrom glob import glob\nimport gc\n    \n# data processing libraries\nimport polars as pl\nimport numpy as np\nimport pandas as pd\n\n# for visualizing data\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# sklearn functions\nfrom sklearn.model_selection import train_test_split, StratifiedGroupKFold\nfrom sklearn.preprocessing import LabelEncoder\n# for validating models\nfrom sklearn.metrics import roc_auc_score, roc_curve\n\n# LightGBM modeling\nimport lightgbm as lgb\n# Có thể thực hiện huấn luyện mô hình với các cách chọn feature khác nhau\n\n# define default colors for plots in notebook\nfrom matplotlib import cycler\nfrom matplotlib.colors import LinearSegmentedColormap\nCOLORS = [\"#068D9D\", \"#53599A\", \"#607BB0\", \"#6D9DC5\", \"#77BECF\", \"#80DED9\", \"#AEECEF\"]\nplt.rc('axes', facecolor='#E6E6E6', edgecolor='none', axisbelow=True, grid=True, prop_cycle=cycler('color', COLORS))\n\n# project CONSTANTS\nROOT = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR = ROOT / \"parquet_files\" / \"test\"\nSEED = 42","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-16T09:01:04.902208Z","iopub.execute_input":"2024-04-16T09:01:04.902629Z","iopub.status.idle":"2024-04-16T09:01:08.924763Z","shell.execute_reply.started":"2024-04-16T09:01:04.902594Z","shell.execute_reply":"2024-04-16T09:01:08.923571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Pipeline:\n    \"\"\"\n    Helper class taken from notebook:\n    https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n    \"\"\"\n    def set_table_dtypes(df):\n        # Thiếu xử lý feature dạng T và L\n        # Mấy cái đó nó tự áp kiểu dữ liệu vào luôn ?\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n        return df\n    def sort_df(df, depth):\n        # Sắp xếp để hàm first có ý nghĩa\n        if depth == 0:\n            return df.sort(\"case_id\")\n        elif depth == 1:\n            return df.sort([\"case_id\", \"num_group1\"])\n        else:\n            return df.sort([\"case_id\", \"num_group1\", \"num_group2\"])\n    \n    def handle_dates(df):\n        # Lấy số ngày từ date_decision\n        # Hàm này được gọi khi mà đã thực hiện các phép tổng hợp xong\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))  #!!?\n                # Hàm này thay đổi giá trị của cột col luôn chứ không thêm cột mới !\n                df = df.with_columns(pl.col(col).dt.total_days()) # t - t-1\n        df = df.drop(\"date_decision\", \"MONTH\")\n        return df\n\n    def filter_cols(df):\n        # Hàm này được áp dụng khi mà đã thực hiện các hàm tổng hợp !\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                # Bỏ các cột có tỉ lệ null là trên 30%\n                isnull = df[col].is_null().mean()\n                if isnull > 0.7:\n                    df = df.drop(col)\n        \n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                # Bỏ các cột category có số giá trị unique trên 200\n                freq = df[col].n_unique()\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n        \n        return df","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:01:08.926958Z","iopub.execute_input":"2024-04-16T09:01:08.927547Z","iopub.status.idle":"2024-04-16T09:01:08.941432Z","shell.execute_reply.started":"2024-04-16T09:01:08.927514Z","shell.execute_reply":"2024-04-16T09:01:08.940189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Aggregator:\n    # Toàn là max ???\n    \"\"\"\n    Helper class taken from notebook:\n    https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n    \"\"\"    \n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_mean = [pl.median(col).alias(f\"mean_{col}\") for col in cols]\n        expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_std = [pl.std(col).alias(f\"std_{col}\") for col in cols]\n        return expr_mean + expr_median + expr_max + expr_min + expr_std\n    \n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\")]\n        expr_mean = [pl.median(col).alias(f\"mean_{col}\") for col in cols]\n        expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_std = [pl.std(col).alias(f\"std_{col}\") for col in cols]\n        return expr_mean + expr_median + expr_max + expr_min + expr_std\n    \n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        \n        # Lấy chuỗi có độ dài dài nhất\n        expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        return expr_max\n    \n    def other_expr(df):\n        # Chổ này cần xem xét kiểu dữ liệu để áp dụng phù hợp\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr = []\n        for col in cols:\n            if df[col].dtype.is_numeric():\n                expr += [pl.median(col).alias(f\"mean_{col}\"), \n                         pl.median(col).alias(f\"median_{col}\"), \n                         pl.max(col).alias(f\"max_{col}\"), \n                         pl.min(col).alias(f\"min_{col}\"), \n                         pl.std(col).alias(f\"std_{col}\")]\n            else:\n                expr +=  [pl.std(col).alias(f\"std_{col}\"),\n                          pl.max(col).alias(f\"max_{col}\"),\n                          pl.min(col).alias(f\"min_{col}\")]\n        return expr\n    \n    def count_expr(df):\n        # Chổ này làm qq gì đây ?\n        cols = [col for col in df.columns if \"num_group\" in col]\n#         expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]  # max & replace col name\n        expr_count = [pl.count(col).alias(f\"count_{col}\") for col in cols]\n        expr_n_unique = [pl.n_unique(col).alias(f\"n_unique_{col}\") for col in cols]  # max & replace col name\n        return expr_count + expr_n_unique\n    \n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:01:08.944964Z","iopub.execute_input":"2024-04-16T09:01:08.945329Z","iopub.status.idle":"2024-04-16T09:01:08.962075Z","shell.execute_reply.started":"2024-04-16T09:01:08.945298Z","shell.execute_reply":"2024-04-16T09:01:08.960829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path, depth):\n    \"\"\"\n    Helper function taken from notebook:\n    https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n    \"\"\"\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    df = Pipeline.sort_df(df, depth)\n    if depth in [1,2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df)) \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:01:08.965224Z","iopub.execute_input":"2024-04-16T09:01:08.966002Z","iopub.status.idle":"2024-04-16T09:01:08.976334Z","shell.execute_reply.started":"2024-04-16T09:01:08.965959Z","shell.execute_reply":"2024-04-16T09:01:08.975233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_files(regex_path, depth):\n    \"\"\"\n    Helper function taken from notebook:\n    https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n    \"\"\"\n    chunks = []\n    \n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        df = Pipeline.sort_df(df, depth)\n        if depth in [1, 2]:\n            # Nếu group ở đây thì lỡ nó không liên kết nhau thì sao ?\n            # 1 giá trị case_id xuất hiện trong cả file_1 và file_2 ?\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n            # Hợp lý\n        chunks.append(df)\n    \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    # relaxed: chuyển đổi để cùng kiểu dữ liệu giữa các hàng\n    \n    # Đây là lệnh dùng để bỏ các giá trị case_id trùng ở trên \n    # Nhưng thực hiện như vầy dẫn đến mất dữ liệu\n    print(len(df))\n    df = df.unique(subset=[\"case_id\"])\n    print(len(df))\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:01:08.978112Z","iopub.execute_input":"2024-04-16T09:01:08.979426Z","iopub.status.idle":"2024-04-16T09:01:08.988649Z","shell.execute_reply.started":"2024-04-16T09:01:08.979364Z","shell.execute_reply":"2024-04-16T09:01:08.987516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n    \"\"\"\n    Helper function taken from notebook:\n    https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n    \"\"\"\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n        # thêm 2 cột mới\n    )\n    # Lấy giá trị ngày và giá trị tuần\n    # Join mấy cái file lại với nhau\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        # Left join ở đây nên sẽ có các column mới\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n        # Có thể ảnh hưởng từ left join\n    df_base = df_base.pipe(Pipeline.handle_dates)\n    return df_base","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:01:08.990195Z","iopub.execute_input":"2024-04-16T09:01:08.990606Z","iopub.status.idle":"2024-04-16T09:01:09.001568Z","shell.execute_reply.started":"2024-04-16T09:01:08.990561Z","shell.execute_reply":"2024-04-16T09:01:09.000578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n    # Danh sách cột Category: cat_cols\n    \"\"\"\n    Helper function taken from notebook:\n    https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n    \"\"\"\n    df_data = df_data.to_pandas()\n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:01:09.002947Z","iopub.execute_input":"2024-04-16T09:01:09.003289Z","iopub.status.idle":"2024-04-16T09:01:09.010986Z","shell.execute_reply.started":"2024-04-16T09:01:09.003259Z","shell.execute_reply":"2024-04-16T09:01:09.010116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, df_name, verbose=True):\n    \"\"\"\n    This function changes pandas numerical dtypes (reduces bit size if possible)\n    to reduce memory usage\n    \n    :param df: pandas DataFrame\n    :param df_name: str, name of DataFrame\n    :param verbose: bool, if True prints out message of how much memory usage was reduced\n        \n    :return:  pandas DataFrame       \n    \"\"\"\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int64)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float64)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    # calculate memory after reduction\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose:\n        # reduced memory usage in percent\n        diff_pst = 100 * (start_mem - end_mem) / start_mem\n        msg = f'{df_name} mem. usage decreased to {end_mem:5.2f} Mb ({diff_pst:.1f}% reduction)'\n        print(msg)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:01:09.012293Z","iopub.execute_input":"2024-04-16T09:01:09.012972Z","iopub.status.idle":"2024-04-16T09:01:09.025632Z","shell.execute_reply.started":"2024-04-16T09:01:09.012943Z","shell.execute_reply":"2024-04-16T09:01:09.024302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_train = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\", 0),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\", 0),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\", 0),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:01:09.027072Z","iopub.execute_input":"2024-04-16T09:01:09.027429Z","iopub.status.idle":"2024-04-16T09:06:27.288185Z","shell.execute_reply.started":"2024-04-16T09:01:09.027391Z","shell.execute_reply":"2024-04-16T09:06:27.285214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_test = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\", 0),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\", 0),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\", 0),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n        read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:06:27.297098Z","iopub.execute_input":"2024-04-16T09:06:27.297601Z","iopub.status.idle":"2024-04-16T09:06:27.740668Z","shell.execute_reply.started":"2024-04-16T09:06:27.297566Z","shell.execute_reply":"2024-04-16T09:06:27.739402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lấy danh sách các cột\n# get column names of original/raw features\nFEATS_ORIG = []\n\n# get column names\nfor _key in data_train.keys():\n    if isinstance(data_train[_key], list):\n        for _df in data_train[_key]:\n            FEATS_ORIG += _df.columns\n            \n# leave only unique values\nFEATS_ORIG = list(set(FEATS_ORIG))\n# drop case_id column\nFEATS_ORIG.remove(\"case_id\")","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:06:27.741835Z","iopub.execute_input":"2024-04-16T09:06:27.742232Z","iopub.status.idle":"2024-04-16T09:06:27.748886Z","shell.execute_reply.started":"2024-04-16T09:06:27.742203Z","shell.execute_reply":"2024-04-16T09:06:27.748023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Join các table lại với nhau\n# Lấy giá trị tháng và giá trị tuần từ date_decision thành week_decision và month_decision\n# Trừ giá trị ngày trong các cột D với giá trị của cột date_decision\n# Vậy giá trị week_decision và month_decision dùng làm gì ?\ndf_train = feature_eng(**data_train)\nprint(\"train data shape:\\t\", df_train.shape)\n# clean memory\ndel data_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:06:27.750262Z","iopub.execute_input":"2024-04-16T09:06:27.750603Z","iopub.status.idle":"2024-04-16T09:06:58.198360Z","shell.execute_reply.started":"2024-04-16T09:06:27.750575Z","shell.execute_reply":"2024-04-16T09:06:58.197356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_test = feature_eng(**data_test)\nprint(\"test data shape:\\t\", df_test.shape)\n# clean memory\ndel data_test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:06:58.200303Z","iopub.execute_input":"2024-04-16T09:06:58.200748Z","iopub.status.idle":"2024-04-16T09:06:58.481261Z","shell.execute_reply.started":"2024-04-16T09:06:58.200709Z","shell.execute_reply":"2024-04-16T09:06:58.480298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get column names of new created features\nFEATS_NEW = set(df_train.columns)\n# remove speific columns\nFEATS_NEW = FEATS_NEW.difference({'case_id', 'WEEK_NUM', 'target'})\nFEATS_NEW = FEATS_NEW.difference(set(FEATS_ORIG))\nFEATS_NEW = list(FEATS_NEW)\n\nprint(f\"Before feature engineering: {len(FEATS_ORIG)} featues.\")\nprint(f\"{len(FEATS_NEW)} new features created.\")","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:06:58.482738Z","iopub.execute_input":"2024-04-16T09:06:58.483044Z","iopub.status.idle":"2024-04-16T09:06:58.490640Z","shell.execute_reply.started":"2024-04-16T09:06:58.483018Z","shell.execute_reply":"2024-04-16T09:06:58.489603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop the insignificant features\ndf_train = df_train.pipe(Pipeline.filter_cols)\n# Bỏ cột target ?\ndf_test = df_test.select([col for col in df_train.columns if col != \"target\"])\n\nprint(\"train data shape: \", df_train.shape)\nprint(\"test data shape: \", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:06:58.491887Z","iopub.execute_input":"2024-04-16T09:06:58.492181Z","iopub.status.idle":"2024-04-16T09:07:03.527603Z","shell.execute_reply.started":"2024-04-16T09:06:58.492156Z","shell.execute_reply":"2024-04-16T09:07:03.526495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get column names of the remaining features\nFEATS_REMAIN = set(df_train.columns)\n# remove speific columns\nFEATS_REMAIN = FEATS_REMAIN.difference({'case_id', 'WEEK_NUM', 'target'})\n# remaining in original features\n_1 = [_ for _ in FEATS_REMAIN if _ in FEATS_ORIG]\n# remaining of the new features\n_2 = [_ for _ in FEATS_REMAIN if _ in FEATS_NEW]\n\nprint(\"After removing insignificant features:\")\nprint(f\"{len(_1)} original features left\") # Số feature còn lại trong feature gốc\nprint(f\"{len(_2)} new features left\") # Số feature còn lại trong feature mới","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:07:03.528759Z","iopub.execute_input":"2024-04-16T09:07:03.529901Z","iopub.status.idle":"2024-04-16T09:07:03.544674Z","shell.execute_reply.started":"2024-04-16T09:07:03.529857Z","shell.execute_reply":"2024-04-16T09:07:03.543671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# convert back to pandas\ndf_train, cat_cols = to_pandas(df_train)\ndf_test, cat_cols = to_pandas(df_test, cat_cols)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:07:03.546166Z","iopub.execute_input":"2024-04-16T09:07:03.546709Z","iopub.status.idle":"2024-04-16T09:07:24.938221Z","shell.execute_reply.started":"2024-04-16T09:07:03.546676Z","shell.execute_reply":"2024-04-16T09:07:24.937040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# reduce memory usage if available\ndf_train = reduce_mem_usage(df_train, \"df_train\")\ndf_test = reduce_mem_usage(df_test, \"df_test\")","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:07:24.939736Z","iopub.execute_input":"2024-04-16T09:07:24.940573Z","iopub.status.idle":"2024-04-16T09:07:50.155243Z","shell.execute_reply.started":"2024-04-16T09:07:24.940534Z","shell.execute_reply":"2024-04-16T09:07:50.153601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# select numeric columns\n# 3 cái đầu là: 'case_id', 'WEEK_NUM', 'target'\nnum_cols = df_train.iloc[:, 3:].select_dtypes(include='number').columns\n# get column names without NaN values\n_ = df_train[num_cols].isna().sum()\n# get numerical column names without NaN\nnum_cols_no_nan = _[_==0].index.tolist()\nnum_cols_nan = _[_>0].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:07:50.157116Z","iopub.execute_input":"2024-04-16T09:07:50.157548Z","iopub.status.idle":"2024-04-16T09:08:15.727902Z","shell.execute_reply.started":"2024-04-16T09:07:50.157506Z","shell.execute_reply":"2024-04-16T09:08:15.726771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Tổng số cột:', len(num_cols))\nprint('Số cột có nan:', len(num_cols) - len(num_cols_no_nan))\nprint('Số cột không nan:', len(num_cols_no_nan))","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:08:15.729816Z","iopub.execute_input":"2024-04-16T09:08:15.731193Z","iopub.status.idle":"2024-04-16T09:08:15.738159Z","shell.execute_reply.started":"2024-04-16T09:08:15.731140Z","shell.execute_reply":"2024-04-16T09:08:15.736970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get train data set stats\n_df_1 = df_train[num_cols_no_nan].describe().T\n_df_1.drop(columns=['count'], inplace=True)\n_df_1.columns = [f\"train {_}\" for _ in _df_1.columns]\n# get test data set stats\n_df_2 = df_test[num_cols_no_nan].describe().T\n_df_2.drop(columns=['count'], inplace=True)\n_df_2.columns = [f\"test {_}\" for _ in _df_2.columns]\npd.merge(_df_1, _df_2, left_index=True, right_index=True, how=\"outer\").sort_values(by=\"train mean\")","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:08:15.740152Z","iopub.execute_input":"2024-04-16T09:08:15.740639Z","iopub.status.idle":"2024-04-16T09:08:19.380495Z","shell.execute_reply.started":"2024-04-16T09:08:15.740586Z","shell.execute_reply":"2024-04-16T09:08:19.379443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# select numeric columns\ncat_cols = df_train.iloc[:, 3:].select_dtypes(include='category').columns\n# get column names without NaN values\n_ = df_train[cat_cols].isna().sum()\n# get numerical column names without NaN\ncat_cols_no_nan = _[_==0].index.tolist()\ncat_cols_nan = _[_>0].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:08:19.381827Z","iopub.execute_input":"2024-04-16T09:08:19.382147Z","iopub.status.idle":"2024-04-16T09:08:23.976212Z","shell.execute_reply.started":"2024-04-16T09:08:19.382120Z","shell.execute_reply":"2024-04-16T09:08:23.975117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Tổng số cột:', len(cat_cols))\nprint('Số cột có nan:', len(cat_cols) - len(cat_cols_no_nan))\nprint('Số cột không nan:', len(cat_cols_no_nan))","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:08:23.977436Z","iopub.execute_input":"2024-04-16T09:08:23.977775Z","iopub.status.idle":"2024-04-16T09:08:23.983722Z","shell.execute_reply.started":"2024-04-16T09:08:23.977746Z","shell.execute_reply":"2024-04-16T09:08:23.982464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get train data set stats\n_df_1 = df_train[cat_cols_no_nan].describe().T\n_df_1.drop(columns=['count'], inplace=True)\n_df_1.columns = [f\"train {_}\" for _ in _df_1.columns]\n# get test data set stats\n_df_2 = df_test[cat_cols_no_nan].describe().T\n_df_2.drop(columns=['count'], inplace=True)\n_df_2.columns = [f\"test {_}\" for _ in _df_2.columns]\npd.merge(_df_1, _df_2, left_index=True, right_index=True, how=\"outer\").sort_values(by=\"train unique\")","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:08:23.985231Z","iopub.execute_input":"2024-04-16T09:08:23.985733Z","iopub.status.idle":"2024-04-16T09:08:24.255406Z","shell.execute_reply.started":"2024-04-16T09:08:23.985696Z","shell.execute_reply":"2024-04-16T09:08:24.254188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tất các các kiểu dữ liệu\n# Bỏ các kiểu dữ liệu category\nall_types = set(df_train.dtypes.values)\nall_catetory_types = set(df_train.select_dtypes(include='category').dtypes.values)\nnot_category_type = all_types - all_catetory_types\nprint(not_category_type)\n# Không có dữ liệu datetime","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:08:24.256963Z","iopub.execute_input":"2024-04-16T09:08:24.257418Z","iopub.status.idle":"2024-04-16T09:08:24.314374Z","shell.execute_reply.started":"2024-04-16T09:08:24.257358Z","shell.execute_reply":"2024-04-16T09:08:24.313173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"values_fill = {}\nfor col in num_cols_nan:\n    values_fill[col] = df_train[col].mean()\n    df_train[col] = df_train[col].fillna(values_fill[col])\nprint('Category')\nfor col in cat_cols_nan:\n    values_fill[col] = df_train[col].mode()[0] # Trường hợp có nhiều giá trị mode\n    print(col, values_fill[col])\n    df_train[col] = df_train[col].fillna(values_fill[col])","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:08:24.315884Z","iopub.execute_input":"2024-04-16T09:08:24.316824Z","iopub.status.idle":"2024-04-16T09:08:35.489413Z","shell.execute_reply.started":"2024-04-16T09:08:24.316789Z","shell.execute_reply":"2024-04-16T09:08:35.488256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = df_train.isna().sum()\n# get numerical column names without NaN\ncols_nan = _[_>0].index.tolist()\nlen(cols_nan)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:08:35.490868Z","iopub.execute_input":"2024-04-16T09:08:35.491314Z","iopub.status.idle":"2024-04-16T09:08:37.507756Z","shell.execute_reply.started":"2024-04-16T09:08:35.491276Z","shell.execute_reply":"2024-04-16T09:08:37.506697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drawHistNumeric(df, col, path=None):\n    if df[col].dtypes.name != 'category':\n        sr_0 = df[df['target'] == 0][col]\n        sr_1 = df[df['target'] == 1][col]\n        min_nunique = min(sr_0.nunique(), sr_0.nunique())\n        min_x = int(min(min(sr_0), min(sr_1)))\n        max_x = int(max(max(sr_0), max(sr_1)))\n#         bins = range(min_x, max_x + 1)\n        if min_x == max_x:\n            res = f'{col} chỉ có 1 giá trị {df[col].unique()[0]}'\n            if path:\n                with open(f'{path}/{col}.txt', 'w') as file:\n                    file.write(res)\n            else:\n                print(res)\n            return\n        bins = np.linspace(min_x, max_x, min(min_nunique, 30))\n        # Mà này là đếm số lượng hay tính tỉ lệ ?\n\n        # Này là đúng chuẩn tính theo phần trăm\n        hist0, bins0, _ = plt.hist(sr_0, bins=bins, density=True, alpha=0.5, label='target_0', color ='blue')\n        hist1, bins1, _ = plt.hist(sr_1, bins=bins, density=True, alpha=0.5, label='target_1', color ='red')\n        \n        convertBins = lambda bins: [(val + val_af) / 2 for val, val_af in zip(bins[:-1], bins[1:])]\n        bins0 = convertBins(bins0)\n        bins1 = convertBins(bins1)\n\n        plt.plot(bins0, hist0, color='blue', alpha=0.5, linestyle='-', linewidth=2)\n        plt.plot(bins1, hist1, color='red', alpha=0.5, linestyle='-', linewidth=2)\n        \n        y_max = max(max(hist0), max(hist1))\n        plt.ylim(0, y_max)\n        plt.xlabel('Giá trị')\n        plt.ylabel('Tỉ lệ')\n        plt.title(f'Histogram của {col}')\n        plt.legend()\n        if path:\n            plt.savefig(f'{path}/{col}.png')\n        else:\n            plt.show()\n        plt.close()\n    else:\n        print('Sai kiểu')","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:29:59.041272Z","iopub.execute_input":"2024-04-16T09:29:59.041668Z","iopub.status.idle":"2024-04-16T09:29:59.054543Z","shell.execute_reply.started":"2024-04-16T09:29:59.041640Z","shell.execute_reply":"2024-04-16T09:29:59.053457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_NUM_HISTOGRAM_DIR = 'numeric_histogram'\n! rm -fr {PATH_NUM_HISTOGRAM_DIR}\n! mkdir {PATH_NUM_HISTOGRAM_DIR}\nfor col in num_cols:\n    print(col)\n    drawHistNumeric(df_train, col, PATH_NUM_HISTOGRAM_DIR)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T09:30:01.333019Z","iopub.execute_input":"2024-04-16T09:30:01.333464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drawProportionCategory(df, col, path=None):\n    if df[col].dtypes.name == 'category':\n        # Khác số lượng có sao không nhờ ?\n        sr_0 = df[df['target'] == 0][col]\n        sr_1 = df[df['target'] == 1][col]\n\n        count_percent_0 = sr_0.value_counts() / len(sr_0)\n        count_percent_1 = sr_1.value_counts() / len(sr_1)\n        count_percent_0.name = 'target 0'\n        count_percent_1.name = 'target 1'\n\n        df_percent = pd.concat([count_percent_0, count_percent_1], axis=1, join='outer').fillna(0)\n        df_percent = df_percent.sort_values(by='target 0', ascending=False)\n        \n        width = 0.25  # the width of the bars\n        multiplier = 0\n        fig, ax = plt.subplots(layout='constrained')\n        x = np.arange(len(df_percent))  # the label locations\n        \n        for column in df_percent.columns:\n            offset = width * multiplier\n            rects = ax.bar(x + offset, df_percent[column].values, width, label=column)\n            ax.bar_label(rects, padding=3)\n            multiplier += 1\n        \n        ax.set_ylabel('Tỉ lệ')\n        ax.set_xlabel('Giá trị')\n        ax.set_title(f'Tỉ lệ của {col}')\n        ax.set_xticks(x + width / 2, df_percent.index)\n        ax.legend(loc='upper right', ncols=3)\n        if path:\n            plt.savefig(f'{path}/{col}.png')\n        else:\n            plt.show()\n        plt.close()\n    else:\n        print('Sai kiểu')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_CAT_PROPORTION_DIR = 'category_proportion'\n! rm -fr {PATH_CAT_PROPORTION_DIR}\n! mkdir {PATH_CAT_PROPORTION_DIR}\nfor col in cat_cols:\n    print(col)\n    drawProportionCategory(df_train, col, PATH_CAT_PROPORTION_DIR)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}