{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Ноутбук для аггрегации и фильтрации данных с помощью polars + ML\n\n1) Загрузка, аггрегация, фильтрация данных\n\n\n2) ML\n\nАвтор: Николаев О.В.\n\nСсылка на соревнование: https://www.kaggle.com/competitions/home-credit-credit-risk-model-stability/overview\n\n\n\n\n\nВ основе лежит `class Aggregator`, который аггрегирует в зависимости от типов колонок. Аггрегации подобраны так, чтобы влезать в память.\n\nДописывание и оптимизация аггреаций и фильтров приветствуется. В дополнение, придставлен раздел с ML.\n\n\nИсточники: https://www.kaggle.com/code/jetakow/home-credit-2024-starter-notebook, https://www.kaggle.com/code/greysky/home-credit-baseline","metadata":{}},{"cell_type":"code","source":"import gc\nimport zipfile\nimport os\nfrom glob import glob\nfrom time import time\nfrom datetime import datetime\n\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport polars as pl\nimport pandas as pd\nimport numpy as np\n\nfrom polars.datatypes.classes import Float64, UInt32, Float32, Int32, Int64, Boolean, String, Int8\nfrom collections import Counter\n\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \nfrom sklearn.preprocessing import LabelEncoder, OrdinalEncoder, StandardScaler\n\nbase_dataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:29:57.678528Z","iopub.execute_input":"2024-05-15T12:29:57.678993Z","iopub.status.idle":"2024-05-15T12:30:01.182587Z","shell.execute_reply.started":"2024-05-15T12:29:57.678956Z","shell.execute_reply":"2024-05-15T12:30:01.180696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cбор и аггрегация сырых данных ","metadata":{}},{"cell_type":"code","source":"class data_Pipeline:\n    @staticmethod\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int32))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float32))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))            \n\n        return df\n    \n    @staticmethod\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n                df = df.with_columns(pl.col(col).dt.total_days())\n                df = df.with_columns(pl.col(col).cast(pl.Float32))\n        df = df.drop(\"date_decision\", \"MONTH\")\n\n        return df\n    \n    @staticmethod\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n\n                if isnull > 0.8:\n                    df = df.drop(col)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n\n        return df\n\n\nclass Aggregator:\n    @staticmethod\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        # expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n        # expr_n_unique = [pl.n_unique(col).alias(f\"n_unique_{col}\") for col in cols]\n        # expr_std = [pl.std(col).alias(f\"std_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        # expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        expr_len = [pl.len().alias(f\"len_{col}\") for col in cols]\n        expr_sum = [pl.sum(col).alias(f\"sum_{col}\") for col in cols]\n        \n        return expr_max + expr_min + expr_median + expr_len + expr_sum # + expr_n_unique + expr_first + expr_last + expr_mean + expr_std\n\n    @staticmethod\n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\",)]\n\n        # expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        # expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n        # expr_n_unique = [pl.n_unique(col).alias(f\"n_unique_{col}\") for col in cols]\n        # expr_std = [pl.std(col).alias(f\"std_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        # expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        expr_len = [pl.len().alias(f\"len_{col}\") for col in cols]\n        # expr_sum = [pl.sum(col).alias(f\"sum_{col}\") for col in cols]\n        \n        return expr_median #  + expr_mean + expr_max + expr_min + expr_len + expr_first + expr_last \n\n    @staticmethod\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        \n        # expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        # expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n        # expr_n_unique = [pl.n_unique(col).alias(f\"n_unique_{col}\") for col in cols]\n        # expr_std = [pl.std(col).alias(f\"std_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        # expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        expr_len = [pl.len().alias(f\"len_{col}\") for col in cols]\n        # expr_sum = [pl.sum(col).alias(f\"sum_{col}\") for col in cols]\n        \n        return expr_median + expr_len # + expr_n_unique + expr_std + expr_mean + expr_sum + expr_max + expr_min + expr_first + expr_last \n\n    @staticmethod\n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        # expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        # expr_mean = [pl.mean(col).fill_null(0).alias(f\"mean_{col}\") for col in cols]\n        expr_median = [pl.median(col).fill_null(0).alias(f\"median_{col}\") for col in cols]\n        # expr_n_unique = [pl.n_unique(col).alias(f\"n_unique_{col}\") for col in cols]\n        # expr_std = [pl.std(col).alias(f\"std_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        # expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        expr_len = [pl.len().alias(f\"len_{col}\") for col in cols]\n        # expr_sum = [pl.sum(col).fill_null(0).alias(f\"sum_{col}\") for col in cols]\n        \n        return expr_median + expr_len # + expr_n_unique + expr_mean + expr_sum + expr_max + expr_min + expr_std + expr_last +  expr_first\n    \n    @staticmethod\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n\n        # expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        # expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n        # expr_n_unique = [pl.n_unique(col).alias(f\"n_unique_{col}\") for col in cols]\n        # expr_std = [pl.std(col).alias(f\"std_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        # expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        expr_len = [pl.len().alias(f\"len_{col}\") for col in cols]\n        # expr_sum = [pl.sum(col).alias(f\"sum_{col}\") for col in cols]\n        \n        return expr_median + expr_len # + expr_n_unique + expr_mean + expr_sum + expr_max + expr_min + expr_std + expr_first + expr_last \n\n    @staticmethod\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df)  + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs\n\n\ndef read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(data_Pipeline.set_table_dtypes)\n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(data_Pipeline.set_table_dtypes)\n        \n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        \n        chunks.append(df)\n        \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    \n    return df\n\n\ndef feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n        \n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n        \n    df_base = df_base.pipe(data_Pipeline.handle_dates)\n    \n    return df_base\n\n\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:31:16.008856Z","iopub.execute_input":"2024-05-15T12:31:16.009332Z","iopub.status.idle":"2024-05-15T12:31:16.054202Z","shell.execute_reply.started":"2024-05-15T12:31:16.009290Z","shell.execute_reply":"2024-05-15T12:31:16.052875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time()\n\n# Общее хранилище данных (считывает сырые parquet и аггрегирует по case_id в зависимоти от типов данных и глубины)\ndataPath = os.path.join(base_dataPath, \"parquet_files\", \"train\")\ntrain_data_store = {\n    \"df_base\": read_file(os.path.join(dataPath, \"train_base.parquet\")),\n    \"depth_0\": [\n        read_file(os.path.join(dataPath, \"train_static_cb_0.parquet\")),\n        read_files(os.path.join(dataPath, \"train_static_0_*.parquet\")),\n    ],\n    \"depth_1\": [\n        read_files(os.path.join(dataPath, \"train_applprev_1_*.parquet\"), 1),\n        read_file(os.path.join(dataPath, \"train_tax_registry_a_1.parquet\"), 1),\n        read_file(os.path.join(dataPath, \"train_tax_registry_b_1.parquet\"), 1),\n        read_file(os.path.join(dataPath, \"train_tax_registry_c_1.parquet\"), 1),\n        read_files(os.path.join(dataPath, \"train_credit_bureau_a_1_*.parquet\"), 1),\n        read_file(os.path.join(dataPath,  \"train_credit_bureau_b_1.parquet\"), 1),\n        read_file(os.path.join(dataPath, \"train_other_1.parquet\"), 1),\n        read_file(os.path.join(dataPath,  \"train_person_1.parquet\"), 1),\n        read_file(os.path.join(dataPath,  \"train_deposit_1.parquet\"), 1),\n        read_file(os.path.join(dataPath,  \"train_debitcard_1.parquet\"), 1),\n    ],\n    \"depth_2\": [\n        read_file(os.path.join(dataPath, \"train_credit_bureau_b_2.parquet\"), 2),\n        read_files(os.path.join(dataPath, \"train_credit_bureau_a_2_*.parquet\"), 2),\n        read_files(os.path.join(dataPath, \"train_person_2.parquet\"), 2),\n        read_files(os.path.join(dataPath, \"train_applprev_2.parquet\"), 2),\n    ]\n}\n\nprint(\"train data loaded\", time() - start, \"s\")","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:31:16.498939Z","iopub.execute_input":"2024-05-15T12:31:16.499591Z","iopub.status.idle":"2024-05-15T12:36:08.738647Z","shell.execute_reply.started":"2024-05-15T12:31:16.499541Z","shell.execute_reply":"2024-05-15T12:36:08.736226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time()\n\n# объединяет источники в один dataframe и генерирует новые признаки\ntrain_data = feature_eng(**train_data_store)\ndel train_data_store\n\n\nprint(f\"{train_data.estimated_size(unit='gb'):.2f} Гб занимает сырой train\")\n\n# накладывает фильтры на признаки (например, убирает признаки у которых > 0.8 нулей)\ntrain_data_filtered = train_data.pipe(data_Pipeline.filter_cols)\nselected_cols = [col for col in train_data_filtered.columns if col != 'target']\n\ndel train_data\n\nprint(f\"{train_data_filtered.estimated_size(unit='gb'):.2f} Гб занимает отфильтрованный train\")\n\n\ngc.collect()\nprint(time() - start, \"s\")","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:36:08.743094Z","iopub.execute_input":"2024-05-15T12:36:08.744798Z","iopub.status.idle":"2024-05-15T12:36:38.305268Z","shell.execute_reply.started":"2024-05-15T12:36:08.744732Z","shell.execute_reply":"2024-05-15T12:36:38.304110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_data_filtered)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:37:08.907949Z","iopub.execute_input":"2024-05-15T12:37:08.908415Z","iopub.status.idle":"2024-05-15T12:37:08.950661Z","shell.execute_reply.started":"2024-05-15T12:37:08.908377Z","shell.execute_reply":"2024-05-15T12:37:08.949548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ML","metadata":{}},{"cell_type":"markdown","source":"## Выделение категорий","metadata":{}},{"cell_type":"code","source":"cat_cols = train_data_filtered.select([pl.col(pl.Object), pl.col(pl.String)]).columns","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:51:30.906397Z","iopub.execute_input":"2024-05-15T12:51:30.907840Z","iopub.status.idle":"2024-05-15T12:51:30.923783Z","shell.execute_reply.started":"2024-05-15T12:51:30.907788Z","shell.execute_reply":"2024-05-15T12:51:30.922037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# категориальные признаки необходимо энкодить - lgbm dataset кастит в int\nstart = time()\n\nlabelencoder = OrdinalEncoder(\n    handle_unknown=\"use_encoded_value\", \n    unknown_value=-9999\n) # -1: Met negative value in categorical features, will convert it to NaN\n\nlabelencoder.fit(train_data_filtered[cat_cols])\nprint(\"fit done\")\n\ntrain_data_filtered[cat_cols] = labelencoder.transform(train_data_filtered[cat_cols]) # object_cols\nprint(\"labelencoder train_data done\")\n\n\n\nprint(time() - start, \"s\")","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:51:31.145775Z","iopub.execute_input":"2024-05-15T12:51:31.146279Z","iopub.status.idle":"2024-05-15T12:52:08.677229Z","shell.execute_reply.started":"2024-05-15T12:51:31.146242Z","shell.execute_reply":"2024-05-15T12:52:08.675800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = train_data_filtered.to_pandas()\n\ndel train_data_filtered","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:52:08.679665Z","iopub.execute_input":"2024-05-15T12:52:08.680041Z","iopub.status.idle":"2024-05-15T12:52:14.687520Z","shell.execute_reply.started":"2024-05-15T12:52:08.680009Z","shell.execute_reply":"2024-05-15T12:52:14.686128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = df_train.select_dtypes(include=['object']).columns\n\ndf_train[cols] = df_train[cols].apply(pd.to_numeric)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:52:14.689335Z","iopub.execute_input":"2024-05-15T12:52:14.690768Z","iopub.status.idle":"2024-05-15T12:52:15.102740Z","shell.execute_reply.started":"2024-05-15T12:52:14.690728Z","shell.execute_reply":"2024-05-15T12:52:15.101450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:52:15.106262Z","iopub.execute_input":"2024-05-15T12:52:15.106803Z","iopub.status.idle":"2024-05-15T12:52:15.158873Z","shell.execute_reply.started":"2024-05-15T12:52:15.106762Z","shell.execute_reply":"2024-05-15T12:52:15.157915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train-Val-Split","metadata":{}},{"cell_type":"code","source":"# CONSTANTS\n\nRANDOM_SEED = 2024\nTEST_SIZE = 0.2","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:52:15.160313Z","iopub.execute_input":"2024-05-15T12:52:15.160990Z","iopub.status.idle":"2024-05-15T12:52:15.165939Z","shell.execute_reply.started":"2024-05-15T12:52:15.160955Z","shell.execute_reply":"2024-05-15T12:52:15.164746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time()\n\nX_train, X_val, y_train, y_val = train_test_split(\n    df_train.drop(columns=[\"case_id\", \"WEEK_NUM\", \"target\"]), \n    df_train[\"target\"], \n    test_size=TEST_SIZE, \n    random_state=RANDOM_SEED, \n    stratify=df_train[\"target\"],\n    shuffle=True\n)\n\n\n\nprint(time() - start, \"s\")","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:52:15.167489Z","iopub.execute_input":"2024-05-15T12:52:15.168670Z","iopub.status.idle":"2024-05-15T12:52:35.773140Z","shell.execute_reply.started":"2024-05-15T12:52:15.168628Z","shell.execute_reply":"2024-05-15T12:52:35.771425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_train\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:52:35.775267Z","iopub.execute_input":"2024-05-15T12:52:35.775837Z","iopub.status.idle":"2024-05-15T12:52:35.938629Z","shell.execute_reply.started":"2024-05-15T12:52:35.775790Z","shell.execute_reply":"2024-05-15T12:52:35.936902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_train = lgb.Dataset(X_train, y_train, categorical_feature=cat_cols, free_raw_data=False)\nlgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train, categorical_feature=cat_cols, free_raw_data=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:52:35.940448Z","iopub.execute_input":"2024-05-15T12:52:35.940941Z","iopub.status.idle":"2024-05-15T12:52:35.954107Z","shell.execute_reply.started":"2024-05-15T12:52:35.940899Z","shell.execute_reply":"2024-05-15T12:52:35.952636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_train, y_train\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:52:35.958531Z","iopub.execute_input":"2024-05-15T12:52:35.958927Z","iopub.status.idle":"2024-05-15T12:52:36.092240Z","shell.execute_reply.started":"2024-05-15T12:52:35.958897Z","shell.execute_reply":"2024-05-15T12:52:36.091043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'boosting_type': 'gbdt',\n    'num_leaves': 50,\n    'objective': 'binary',\n    'is_unbalance': True,\n    'metric': 'auc',\n    'nthread': 25,\n    'learning_rate': 0.03280887026004621,\n    'verbose': 1,\n    'seed': RANDOM_SEED}","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:52:36.096243Z","iopub.execute_input":"2024-05-15T12:52:36.098004Z","iopub.status.idle":"2024-05-15T12:52:36.105076Z","shell.execute_reply.started":"2024-05-15T12:52:36.097952Z","shell.execute_reply":"2024-05-15T12:52:36.103479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_booster = lgb.train(\n    params, \n    lgb_train,\n    num_boost_round=750, \n    valid_sets=[lgb_eval, lgb_train], \n    valid_names=['валидация', \"обучение\"],\n    callbacks=[\n        lgb.early_stopping(stopping_rounds=15, min_delta=0.),\n        lgb.log_evaluation(period=10) # чтобы выводились результаты подсчета метрики для early_stopping\n   ]\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:52:36.106428Z","iopub.execute_input":"2024-05-15T12:52:36.106808Z","iopub.status.idle":"2024-05-15T12:54:45.294016Z","shell.execute_reply.started":"2024-05-15T12:52:36.106776Z","shell.execute_reply":"2024-05-15T12:54:45.292072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val_pred_proba = lgb_booster.predict(\n    X_val,\n    raw_score=False\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:54:45.298389Z","iopub.execute_input":"2024-05-15T12:54:45.298888Z","iopub.status.idle":"2024-05-15T12:54:47.855250Z","shell.execute_reply.started":"2024-05-15T12:54:45.298849Z","shell.execute_reply":"2024-05-15T12:54:47.854129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(roc_auc_score(y_val, y_val_pred_proba))","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:55:13.736159Z","iopub.execute_input":"2024-05-15T12:55:13.736593Z","iopub.status.idle":"2024-05-15T12:55:13.867540Z","shell.execute_reply.started":"2024-05-15T12:55:13.736545Z","shell.execute_reply":"2024-05-15T12:55:13.865692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_train = pl.read_csv(base_dataPath + \"csv_files/train/train_base.csv\").to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:05:31.732401Z","iopub.execute_input":"2024-05-15T12:05:31.732856Z","iopub.status.idle":"2024-05-15T12:05:32.151669Z","shell.execute_reply.started":"2024-05-15T12:05:31.732821Z","shell.execute_reply":"2024-05-15T12:05:32.150437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_val = base_train.loc[X_val.index]\nbase_val[\"score\"] = y_val_pred_proba","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:05:32.153880Z","iopub.execute_input":"2024-05-15T12:05:32.154373Z","iopub.status.idle":"2024-05-15T12:05:32.238252Z","shell.execute_reply.started":"2024-05-15T12:05:32.154327Z","shell.execute_reply":"2024-05-15T12:05:32.236789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_val","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:05:32.380133Z","iopub.execute_input":"2024-05-15T12:05:32.380559Z","iopub.status.idle":"2024-05-15T12:05:32.402265Z","shell.execute_reply.started":"2024-05-15T12:05:32.380526Z","shell.execute_reply":"2024-05-15T12:05:32.400976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    \n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std, gini_in_time\n\n\nstability_score_valid, gini_in_time = gini_stability(base_val)\n\n\nprint(f'The stability score on the valid set is: {stability_score_valid}') ","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:05:33.111614Z","iopub.execute_input":"2024-05-15T12:05:33.113030Z","iopub.status.idle":"2024-05-15T12:05:33.468508Z","shell.execute_reply.started":"2024-05-15T12:05:33.112970Z","shell.execute_reply":"2024-05-15T12:05:33.467260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission\n\nScoring the submission dataset is below, we need to take care of new categories. Then we save the score as a last step. ","metadata":{}},{"cell_type":"code","source":"del lgb_train, lgb_eval\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:05:35.398866Z","iopub.execute_input":"2024-05-15T12:05:35.399330Z","iopub.status.idle":"2024-05-15T12:05:35.575014Z","shell.execute_reply.started":"2024-05-15T12:05:35.399292Z","shell.execute_reply":"2024-05-15T12:05:35.573012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_val, y_val, base_train, base_val\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:05:39.499030Z","iopub.execute_input":"2024-05-15T12:05:39.499535Z","iopub.status.idle":"2024-05-15T12:05:39.689249Z","shell.execute_reply.started":"2024-05-15T12:05:39.499496Z","shell.execute_reply":"2024-05-15T12:05:39.687919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time()\n\ndataPath = os.path.join(base_dataPath, \"parquet_files\", \"test\")\ntest_data_store = {\n    \"df_base\": read_file(os.path.join(dataPath, \"test_base.parquet\")),\n    \"depth_0\": [\n        read_file(os.path.join(dataPath, \"test_static_cb_0.parquet\")),\n        read_files(os.path.join(dataPath, \"test_static_0_*.parquet\")),\n    ],\n    \"depth_1\": [\n        read_files(os.path.join(dataPath, \"test_applprev_1_*.parquet\"), 1),\n        read_file(os.path.join(dataPath, \"test_tax_registry_a_1.parquet\"), 1),\n        read_file(os.path.join(dataPath, \"test_tax_registry_b_1.parquet\"), 1),\n        read_file(os.path.join(dataPath, \"test_tax_registry_c_1.parquet\"), 1),\n        read_files(os.path.join(dataPath, \"test_credit_bureau_a_1_*.parquet\"), 1),\n        read_file(os.path.join(dataPath,  \"test_credit_bureau_b_1.parquet\"), 1),\n        read_file(os.path.join(dataPath, \"test_other_1.parquet\"), 1),\n        read_file(os.path.join(dataPath,  \"test_person_1.parquet\"), 1),\n        read_file(os.path.join(dataPath,  \"test_deposit_1.parquet\"), 1),\n        read_file(os.path.join(dataPath,  \"test_debitcard_1.parquet\"), 1),\n    ],\n    \"depth_2\": [\n        read_file(os.path.join(dataPath, \"test_credit_bureau_b_2.parquet\"), 2),\n        read_files(os.path.join(dataPath, \"test_credit_bureau_a_2_*.parquet\"), 2),\n        read_files(os.path.join(dataPath, \"test_person_2.parquet\"), 2),\n        read_files(os.path.join(dataPath, \"test_applprev_2.parquet\"), 2),\n    ]\n}\n\nprint(\"test data loaded\", time() - start, \"s\")","metadata":{"execution":{"iopub.status.busy":"2024-05-15T12:07:59.001379Z","iopub.execute_input":"2024-05-15T12:07:59.001870Z","iopub.status.idle":"2024-05-15T12:07:59.310191Z","shell.execute_reply.started":"2024-05-15T12:07:59.001832Z","shell.execute_reply":"2024-05-15T12:07:59.308744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = feature_eng(**test_data_store)\ndel test_data_store","metadata":{"execution":{"iopub.status.busy":"2024-05-15T10:04:25.799134Z","iopub.execute_input":"2024-05-15T10:04:25.799604Z","iopub.status.idle":"2024-05-15T10:04:26.009307Z","shell.execute_reply.started":"2024-05-15T10:04:25.799567Z","shell.execute_reply":"2024-05-15T10:04:26.008323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"{test_data.estimated_size(unit='mb'):.5f} Мб занимает сырой test\")\ntest_data_filtered = test_data.select(selected_cols)\ndel test_data\n\nprint(f\"{test_data_filtered.estimated_size(unit='mb'):.5f} Мб занимает отфильтрованный test\")\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T10:05:19.840731Z","iopub.execute_input":"2024-05-15T10:05:19.841204Z","iopub.status.idle":"2024-05-15T10:05:19.998344Z","shell.execute_reply.started":"2024-05-15T10:05:19.841168Z","shell.execute_reply":"2024-05-15T10:05:19.997199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_filtered[cat_cols] = labelencoder.transform(test_data_filtered[cat_cols]) # object_cols\nprint(\"labelencoder data_test done\")","metadata":{"execution":{"iopub.status.busy":"2024-05-15T10:05:24.119467Z","iopub.execute_input":"2024-05-15T10:05:24.122556Z","iopub.status.idle":"2024-05-15T10:05:24.164862Z","shell.execute_reply.started":"2024-05-15T10:05:24.122516Z","shell.execute_reply":"2024-05-15T10:05:24.164040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = test_data_filtered.to_pandas()\ndel test_data_filtered","metadata":{"execution":{"iopub.status.busy":"2024-05-15T10:05:25.037348Z","iopub.execute_input":"2024-05-15T10:05:25.037776Z","iopub.status.idle":"2024-05-15T10:05:25.078143Z","shell.execute_reply.started":"2024-05-15T10:05:25.037742Z","shell.execute_reply":"2024-05-15T10:05:25.076911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test[cols] = df_test[cols].apply(pd.to_numeric)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T10:05:27.413433Z","iopub.execute_input":"2024-05-15T10:05:27.413830Z","iopub.status.idle":"2024-05-15T10:05:27.426442Z","shell.execute_reply.started":"2024-05-15T10:05:27.413801Z","shell.execute_reply":"2024-05-15T10:05:27.425234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = (\n    df_test.drop(columns=[\"case_id\", \"WEEK_NUM\"])\n)\n\ndel df_test\nlgb_test = lgb.Dataset(X_test, categorical_feature=cat_cols, free_raw_data=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T10:05:29.663526Z","iopub.execute_input":"2024-05-15T10:05:29.663925Z","iopub.status.idle":"2024-05-15T10:05:29.672993Z","shell.execute_reply.started":"2024-05-15T10:05:29.663896Z","shell.execute_reply":"2024-05-15T10:05:29.671510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_pred_proba = lgb_booster.predict(\n    X_test,\n    raw_score=False\n)\nbase_test = pl.read_csv(base_dataPath + \"csv_files/test/test_base.csv\").to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T10:05:30.264567Z","iopub.execute_input":"2024-05-15T10:05:30.265693Z","iopub.status.idle":"2024-05-15T10:05:30.297568Z","shell.execute_reply.started":"2024-05-15T10:05:30.265644Z","shell.execute_reply":"2024-05-15T10:05:30.296357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": base_test[\"case_id\"].to_numpy(),\n    \"score\": y_test_pred_proba\n}).set_index('case_id')","metadata":{"execution":{"iopub.status.busy":"2024-05-15T10:05:31.637694Z","iopub.execute_input":"2024-05-15T10:05:31.638143Z","iopub.status.idle":"2024-05-15T10:05:31.646173Z","shell.execute_reply.started":"2024-05-15T10:05:31.638108Z","shell.execute_reply":"2024-05-15T10:05:31.644759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2024-05-15T10:05:32.211349Z","iopub.execute_input":"2024-05-15T10:05:32.211754Z","iopub.status.idle":"2024-05-15T10:05:32.223710Z","shell.execute_reply.started":"2024-05-15T10:05:32.211724Z","shell.execute_reply":"2024-05-15T10:05:32.222285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:01:16.768328Z","iopub.execute_input":"2024-05-14T15:01:16.768871Z","iopub.status.idle":"2024-05-14T15:01:16.776693Z","shell.execute_reply.started":"2024-05-14T15:01:16.768830Z","shell.execute_reply":"2024-05-14T15:01:16.775142Z"},"trusted":true},"execution_count":null,"outputs":[]}]}