{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8395049,"sourceType":"datasetVersion","datasetId":4994201}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nfrom glob import glob\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\n#import lightgbm as lgb\nfrom catboost import CatBoostClassifier, Pool\n\n#from sklearn.base import BaseEstimator, ClassifierMixin\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:06:51.841015Z","iopub.execute_input":"2024-05-12T21:06:51.841440Z","iopub.status.idle":"2024-05-12T21:06:54.101205Z","shell.execute_reply.started":"2024-05-12T21:06:51.841404Z","shell.execute_reply":"2024-05-12T21:06:54.099639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_memory_usage(df, name):\n    print(f\"Memory usage of dataframe \\\"{name}\\\" is {round(df.estimated_size('mb'), 4)} MB.\")\n\n    int_types = [\n        pl.Int8,\n        pl.Int16,\n        pl.Int32,\n        pl.Int64,\n        pl.UInt8,\n        pl.UInt16,\n        pl.UInt32,\n        pl.UInt64,\n    ]\n    float_types = [pl.Float32, pl.Float64]\n\n    for col in df.columns:\n        col_type = df[col].dtype\n        if col_type in int_types + float_types:\n            c_min = df[col].min()\n            c_max = df[col].max()\n\n            if c_min is not None and c_max is not None:\n                if col_type in int_types:\n                    if c_min >= 0:\n                        if (\n                            c_min >= np.iinfo(np.uint8).min\n                            and c_max <= np.iinfo(np.uint8).max\n                        ):\n                            df = df.with_columns(df[col].cast(pl.UInt8))\n                        elif (\n                            c_min >= np.iinfo(np.uint16).min\n                            and c_max <= np.iinfo(np.uint16).max\n                        ):\n                            df = df.with_columns(df[col].cast(pl.UInt16))\n                        elif (\n                            c_min >= np.iinfo(np.uint32).min\n                            and c_max <= np.iinfo(np.uint32).max\n                        ):\n                            df = df.with_columns(df[col].cast(pl.UInt32))\n                        elif (\n                            c_min >= np.iinfo(np.uint64).min\n                            and c_max <= np.iinfo(np.uint64).max\n                        ):\n                            df = df.with_columns(df[col].cast(pl.UInt64))\n                    else:\n                        if (\n                            c_min >= np.iinfo(np.int8).min\n                            and c_max <= np.iinfo(np.int8).max\n                        ):\n                            df = df.with_columns(df[col].cast(pl.Int8))\n                        elif (\n                            c_min >= np.iinfo(np.int16).min\n                            and c_max <= np.iinfo(np.int16).max\n                        ):\n                            df = df.with_columns(df[col].cast(pl.Int16))\n                        elif (\n                            c_min >= np.iinfo(np.int32).min\n                            and c_max <= np.iinfo(np.int32).max\n                        ):\n                            df = df.with_columns(df[col].cast(pl.Int32))\n                        elif (\n                            c_min >= np.iinfo(np.int64).min\n                            and c_max <= np.iinfo(np.int64).max\n                        ):\n                            df = df.with_columns(df[col].cast(pl.Int64))\n                elif col_type in float_types:\n                    if (\n                        c_min > np.finfo(np.float32).min\n                        and c_max < np.finfo(np.float32).max\n                    ):\n                        df = df.with_columns(df[col].cast(pl.Float32))\n\n    print(f\"Memory usage of dataframe \\\"{name}\\\" became {round(df.estimated_size('mb'), 4)} MB.\")\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:06:56.193888Z","iopub.execute_input":"2024-05-12T21:06:56.194478Z","iopub.status.idle":"2024-05-12T21:06:56.221011Z","shell.execute_reply.started":"2024-05-12T21:06:56.194441Z","shell.execute_reply":"2024-05-12T21:06:56.218650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_to_pandas(df, cat_cols=None):\n    df = df.to_pandas()\n\n    if cat_cols is None:\n        cat_cols = list(df.select_dtypes(\"object\").columns)\n\n    df[cat_cols] = df[cat_cols].astype(\"str\")\n\n    return df, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:06:59.822252Z","iopub.execute_input":"2024-05-12T21:06:59.822972Z","iopub.status.idle":"2024-05-12T21:06:59.829716Z","shell.execute_reply.started":"2024-05-12T21:06:59.822936Z","shell.execute_reply":"2024-05-12T21:06:59.828119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_aggregation_exprs(df):\n    # Выбираем столбцы для агрегации\n    aggregation_columns = [\n        col for col in df.columns\n        if (col[-1] in (\"P\", \"M\", \"A\", \"D\", \"T\", \"L\")) or (\"num_group\" in col)\n    ]\n    \n    # Выражения для максимальных значений\n    max_exprs = [pl.col(col).max().alias(f\"max_{col}\") for col in aggregation_columns]\n    \n    # Выражения для минимальных значений\n    min_exprs = [pl.col(col).min().alias(f\"min_{col}\") for col in aggregation_columns]\n    \n    # Выражения для средних значений\n    mean_exprs = [pl.col(col).mean().alias(f\"mean_{col}\") for col in df.columns if col.endswith((\"P\", \"A\", \"D\"))]\n    \n    # Выражения для дисперсии\n    var_exprs = [pl.col(col).var().alias(f\"var_{col}\") for col in df.columns if col.endswith((\"P\", \"A\", \"D\"))]\n    \n    # Выражения для моды\n    mode_exprs = [pl.col(col).drop_nulls().mode().first().alias(f\"mode_{col}\") for col in df.columns if col.endswith(\"M\")]\n\n    return max_exprs, min_exprs, mean_exprs, var_exprs, mode_exprs\n\ndef get_aggregation_exprs(df):\n    max_exprs, min_exprs, mean_exprs, var_exprs, mode_exprs = create_aggregation_exprs(df)\n    aggregation_exprs = max_exprs + min_exprs + var_exprs #+ mean_exprs + mode_exprs\n    return aggregation_exprs","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:07:03.183436Z","iopub.execute_input":"2024-05-12T21:07:03.184399Z","iopub.status.idle":"2024-05-12T21:07:03.196368Z","shell.execute_reply.started":"2024-05-12T21:07:03.184354Z","shell.execute_reply":"2024-05-12T21:07:03.194893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def change_dtypes(df):\n    for col in df.columns:\n        if col == \"case_id\":\n            df = df.with_columns(pl.col(col).cast(pl.UInt32).alias(col))\n        elif col in [\"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n            df = df.with_columns(pl.col(col).cast(pl.UInt16).alias(col))\n        elif col == \"date_decision\" or col[-1] == \"D\":\n            df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        elif col[-1] in [\"P\", \"A\"]:\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        elif col[-1] in (\"M\",):\n            df = df.with_columns(pl.col(col).cast(pl.String))\n    return df\n\ndef scan_files(glob_path, depth=None):\n    chunks = []\n    for path in glob(str(glob_path)):\n        df = pl.scan_parquet(path, low_memory=True, rechunk=True).pipe(change_dtypes)\n        print(f\"File {path} loaded into memory.\")\n        \n        if depth in (1, 2):\n            exprs = get_aggregation_exprs(df)\n            df = df.group_by(\"case_id\").agg(exprs)\n\n            del exprs\n            gc.collect()\n\n        chunks.append(df)\n\n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n\n    del chunks\n    gc.collect()\n\n    df = df.unique(subset=[\"case_id\"])\n\n    return df\n\ndef join_dataframes(df_base, depth_0, depth_1, depth_2):\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n\n    return df_base.collect().pipe(reduce_memory_usage, \"df_train\")","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:07:06.617022Z","iopub.execute_input":"2024-05-12T21:07:06.617436Z","iopub.status.idle":"2024-05-12T21:07:06.632431Z","shell.execute_reply.started":"2024-05-12T21:07:06.617405Z","shell.execute_reply":"2024-05-12T21:07:06.631151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_cols(df):\n    cols_to_drop = []\n\n    # Удаление столбцов с большим процентом отсутствующих значений\n    for col in df.columns:\n        if col not in [\"case_id\", \"year\", \"month\", \"week_num\", \"target\"]:\n            null_pct = df[col].is_null().mean()\n            if null_pct > 0.95:\n                cols_to_drop.append(col)\n\n    # Удаление строковых столбцов с слишком большим количеством уникальных значений или только одним уникальным значением\n    for col in df.columns:\n        if (col not in [\"case_id\", \"year\", \"month\", \"week_num\", \"target\"]) and (df[col].dtype == pl.String):\n            freq = df[col].n_unique()\n            if (freq > 200) or (freq == 1):\n                cols_to_drop.append(col)\n\n    # Удаление выбранных столбцов\n    df = df.drop(cols_to_drop)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:07:11.284299Z","iopub.execute_input":"2024-05-12T21:07:11.284684Z","iopub.status.idle":"2024-05-12T21:07:11.292976Z","shell.execute_reply.started":"2024-05-12T21:07:11.284655Z","shell.execute_reply":"2024-05-12T21:07:11.292113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform_cols(df):\n    if \"riskassesment_302T\" in df.columns:\n        if df[\"riskassesment_302T\"].dtype == pl.Null:\n            df = df.with_columns(\n                [\n                    pl.Series(\n                        \"riskassesment_302T_rng\", df[\"riskassesment_302T\"], pl.UInt8\n                    ),\n                    pl.Series(\n                        \"riskassesment_302T_mean\", df[\"riskassesment_302T\"], pl.UInt8\n                    ),\n                ]\n            )\n        else:\n            # Вычисление диапазона и среднего значения\n            pct_low = (\n                df[\"riskassesment_302T\"]\n                .str.split(\" - \")\n                .apply(lambda x: x[0].replace(\"%\", \"\"))\n                .cast(pl.UInt8)\n            )\n            pct_high = (\n                df[\"riskassesment_302T\"]\n                .str.split(\" - \")\n                .apply(lambda x: x[1].replace(\"%\", \"\"))\n                .cast(pl.UInt8)\n            )\n            diff = pct_high - pct_low\n            avg = ((pct_low + pct_high) / 2).cast(pl.Float32)\n\n            # Удаление переменных pct_low и pct_high\n            del pct_low, pct_high\n\n            # Создание новых столбцов\n            df = df.with_columns(\n                [\n                    diff.alias(\"riskassesment_302T_rng\"),\n                    avg.alias(\"riskassesment_302T_mean\"),\n                ]\n            )\n\n        # Удаление исходного столбца\n        df.drop(\"riskassesment_302T\")\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:07:13.688981Z","iopub.execute_input":"2024-05-12T21:07:13.689387Z","iopub.status.idle":"2024-05-12T21:07:13.702120Z","shell.execute_reply.started":"2024-05-12T21:07:13.689357Z","shell.execute_reply":"2024-05-12T21:07:13.700288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def handle_dates(df):\n    for col in df.columns:\n        if col.endswith(\"D\"):\n            df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n            df = df.with_columns(pl.col(col).dt.total_days().cast(pl.Int32))\n\n    # Переименование столбцов MONTH и WEEK_NUM\n    df = df.rename({\n        \"MONTH\": \"month\",\n        \"WEEK_NUM\": \"week_num\"\n    })\n\n    # Добавление новых столбцов year и day\n    df = df.with_columns(\n        [\n            pl.col(\"date_decision\").dt.year().alias(\"year\").cast(pl.Int16),\n            pl.col(\"date_decision\").dt.day().alias(\"day\").cast(pl.UInt8),\n        ]\n    )\n\n    # Удаление столбца date_decision\n    df = df.drop(\"date_decision\")\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:07:16.720105Z","iopub.execute_input":"2024-05-12T21:07:16.720495Z","iopub.status.idle":"2024-05-12T21:07:16.728882Z","shell.execute_reply.started":"2024-05-12T21:07:16.720466Z","shell.execute_reply":"2024-05-12T21:07:16.727902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/\"\ntest_dir = \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/\"","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:07:20.880328Z","iopub.execute_input":"2024-05-12T21:07:20.880698Z","iopub.status.idle":"2024-05-12T21:07:20.885767Z","shell.execute_reply.started":"2024-05-12T21:07:20.880671Z","shell.execute_reply":"2024-05-12T21:07:20.884577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train1 = pl.read_parquet('/kaggle/input/home-credit-2024/df_train1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:24:42.044595Z","iopub.execute_input":"2024-05-12T21:24:42.046424Z","iopub.status.idle":"2024-05-12T21:24:42.095955Z","shell.execute_reply.started":"2024-05-12T21:24:42.046374Z","shell.execute_reply":"2024-05-12T21:24:42.094819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_store = {\n    \"df_base\": scan_files(test_dir + \"test_base.parquet\"),\n    \"depth_0\": [\n        scan_files(test_dir + \"test_static_cb_0.parquet\"),\n        scan_files(test_dir + \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        scan_files(test_dir + \"test_applprev_1_*.parquet\", 1),\n        scan_files(test_dir + \"test_tax_registry_a_1.parquet\", 1),\n        scan_files(test_dir + \"test_tax_registry_b_1.parquet\", 1),\n        scan_files(test_dir + \"test_tax_registry_c_1.parquet\", 1),\n        scan_files(test_dir + \"test_credit_bureau_a_1_*.parquet\", 1),\n        scan_files(test_dir + \"test_credit_bureau_b_1.parquet\", 1),\n        scan_files(test_dir + \"test_other_1.parquet\", 1),\n        scan_files(test_dir + \"test_person_1.parquet\", 1),\n        scan_files(test_dir + \"test_deposit_1.parquet\", 1),\n        scan_files(test_dir + \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        scan_files(test_dir + \"test_credit_bureau_a_2_*.parquet\", 2),\n        scan_files(test_dir + \"test_credit_bureau_b_2.parquet\", 2),\n    ],\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:26:56.616773Z","iopub.execute_input":"2024-05-12T21:26:56.617170Z","iopub.status.idle":"2024-05-12T21:27:01.074178Z","shell.execute_reply.started":"2024-05-12T21:26:56.617137Z","shell.execute_reply":"2024-05-12T21:27:01.072904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test  = (\n    join_dataframes(**data_store)\n    .pipe(transform_cols)\n    .pipe(handle_dates)\n    .select([col for col in df_train1.columns if col != \"target\"])\n    .pipe(reduce_memory_usage, \"df_test\")\n)\n\ndel data_store","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:27:06.575933Z","iopub.execute_input":"2024-05-12T21:27:06.576361Z","iopub.status.idle":"2024-05-12T21:27:07.013829Z","shell.execute_reply.started":"2024-05-12T21:27:06.576327Z","shell.execute_reply":"2024-05-12T21:27:07.012016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pl.read_parquet('/kaggle/input/home-credit-2024/df_train.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:24:59.756420Z","iopub.execute_input":"2024-05-12T21:24:59.756864Z","iopub.status.idle":"2024-05-12T21:25:15.308151Z","shell.execute_reply.started":"2024-05-12T21:24:59.756826Z","shell.execute_reply":"2024-05-12T21:25:15.306610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, cat_cols = convert_to_pandas(df_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:25:41.008503Z","iopub.execute_input":"2024-05-12T21:25:41.008961Z","iopub.status.idle":"2024-05-12T21:26:25.945881Z","shell.execute_reply.started":"2024-05-12T21:25:41.008926Z","shell.execute_reply":"2024-05-12T21:26:25.944581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_train = df_train.iloc[:50000]\nX = df_train.drop(columns=[\"target\", \"case_id\", \"week_num\"])\ny = df_train[\"target\"]\n\nweeks = df_train[\"week_num\"]\n\ndel df_train","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:26:32.575857Z","iopub.execute_input":"2024-05-12T21:26:32.576261Z","iopub.status.idle":"2024-05-12T21:26:40.611626Z","shell.execute_reply.started":"2024-05-12T21:26:32.576230Z","shell.execute_reply":"2024-05-12T21:26:40.610411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\nfitted_models_cat = []\n\ncv_scores_cat = []\n\niter_cnt = 0\nfor idx_train, idx_valid in cv.split(X, y, groups=weeks):\n    X_train, y_train = X.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X.iloc[idx_valid], y.iloc[idx_valid]\n\n    train_pool = Pool(X_train, y_train, cat_features=cat_cols)\n    val_pool = Pool(X_valid, y_valid, cat_features=cat_cols)\n\n    model = CatBoostClassifier(\n        best_model_min_trees = 1000,\n        boosting_type = \"Plain\",\n        eval_metric = \"AUC\",\n        iterations = 500,\n        learning_rate = 0.05,\n        l2_leaf_reg = 10,\n        max_leaves = 64,\n        random_seed = 42,\n        task_type = \"GPU\",\n        use_best_model = True\n    )\n\n    model.fit(train_pool, eval_set=val_pool, verbose=False)\n    fitted_models_cat.append(model)\n\n    y_pred_valid = model.predict_proba(X_valid)[:, 1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_cat.append(auc_score)\n    \n    iter_cnt += 1\n    \nprint(f\"\\nCV AUC scores for CatBoost: {cv_scores_cat}\")\nprint(f\"Maximum CV AUC score for Catboost: {max(cv_scores_cat)}\", end=\"\\n\\n\")\n\ndel X, y","metadata":{"execution":{"iopub.status.busy":"2024-05-12T19:55:48.345622Z","iopub.execute_input":"2024-05-12T19:55:48.346338Z","iopub.status.idle":"2024-05-12T20:34:35.974909Z","shell.execute_reply.started":"2024-05-12T19:55:48.346307Z","shell.execute_reply":"2024-05-12T20:34:35.973893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Maximum CV AUC score for Catboost: 0.8494825347956171\nMaximum CV AUC score for Catboost: 0.849615783461656","metadata":{}},{"cell_type":"code","source":"df_test, cat_cols = convert_to_pandas(df_test, cat_cols)\n\nX_test = df_test.drop(columns=[\"week_num\"]).set_index(\"case_id\")\nX_test[cat_cols] = X_test[cat_cols].astype(\"category\")\n\ny_pred = model.predict_proba(X_test)[:, 1]\n\nsubmission = pd.DataFrame({\"case_id\": X_test.index, \"score\": y_pred}).set_index('case_id')\nsubmission.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:01:58.410677Z","iopub.execute_input":"2024-05-12T21:01:58.411518Z"},"trusted":true},"execution_count":null,"outputs":[]}]}