{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Reference \n(LGB + Cat ensemble) +Stacking https://www.kaggle.com/code/harrychan123/lgb-cat-ensemble-stacking","metadata":{"_uuid":"faa7d89a-6387-46a1-bfeb-e3f0f5099c91","_cell_guid":"d5a9924a-919a-4bc6-8979-ef4eedd8dbc1","trusted":true}},{"cell_type":"markdown","source":"# Let's get to work!","metadata":{"_uuid":"f3bf9fbe-081b-42ee-a738-cb5f0f66240b","_cell_guid":"12cf95ea-0637-4327-adb1-7875b8335483","trusted":true}},{"cell_type":"markdown","source":"## Imports","metadata":{"_uuid":"c8ac59e1-a139-43da-8fcc-9e7ce19ede80","_cell_guid":"82099699-1459-4244-b3ba-d53b0df5e9d3","trusted":true}},{"cell_type":"code","source":"import sys\nfrom pathlib import Path\nimport subprocess\nimport os\nimport gc\nfrom glob import glob\nimport time\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom datetime import datetime\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nROOT = '/kaggle/input/home-credit-credit-risk-model-stability'\n\nfrom sklearn.model_selection import TimeSeriesSplit, GroupKFold, StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\n\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.impute import KNNImputer\nfrom sklearn.preprocessing import OrdinalEncoder","metadata":{"_uuid":"6e0527b5-e744-4277-920f-dab2aa0dab66","_cell_guid":"ffeff0c4-83f4-4293-af9a-681cf87654d6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-04-29T11:24:54.415815Z","iopub.execute_input":"2024-04-29T11:24:54.416443Z","iopub.status.idle":"2024-04-29T11:25:00.900254Z","shell.execute_reply.started":"2024-04-29T11:24:54.416389Z","shell.execute_reply":"2024-04-29T11:25:00.899112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{"_uuid":"076ede6a-4289-46de-a0f5-598b3b5df26f","_cell_guid":"70eca6f3-9375-489a-b792-482cd960d612","trusted":true}},{"cell_type":"markdown","source":"## Processing functions","metadata":{"_uuid":"1f1fcb11-69c9-4165-b57d-3db3af8de66c","_cell_guid":"dae8d2e2-6cb9-4295-a63a-c18bcfbad630","trusted":true}},{"cell_type":"code","source":"class Pipeline:\n\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date,strict=False))\n        return df\n\n    def handle_dates(df):\n        for col in df.columns: \n                if col.endswith(\"D\"):\n                    # Calculate the difference in days between each date column and date_decision\n                    df = df.with_columns(\n                        (pl.col(\"date_decision\") - pl.col(col)).dt.total_days().alias(col)\n                    )\n                    df = df.with_columns(pl.col(col).fill_null(np.nan)) \n        # Drop date_decision column\n        df = df.drop(\"date_decision\")\n#         print(df.dtypes) # for Debugging\n        return df\n\n    def filter_cols(df,base_df = None,test=False):\n        #for test data\n            for col in df.columns:\n                if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                    isnull = df[col].is_null().mean()\n                    if isnull > 0.7:\n                        df = df.drop(col)\n            columns_to_drop = []\n            for col in df.columns:\n                if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                    freq = df[col].n_unique()\n                    if (freq == 1) or (freq > 100):\n                        columns_to_drop.append(col)\n\n            df = df.drop(columns_to_drop)\n            return df\n\n\nclass Aggregator:\n    \n    @staticmethod\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n        return exprs\n\ndef read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    if depth in [1,2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df)) \n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        chunks.append(df)\n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    return df\n\ndef feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n    df_base = df_base.pipe(Pipeline.set_table_dtypes)\n    df_base = df_base.pipe(Pipeline.handle_dates)\n    return df_base\n\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    return df_data, cat_cols\n\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                try:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        df[col] = df[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        df[col] = df[col].astype(np.float64)\n                except:\n                    continue\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"_uuid":"2e7936e7-0b04-4e01-ac7f-4b4108d1b10e","_cell_guid":"9f62963f-ed9b-4c6c-bad8-e23fc7ff55ff","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-04-29T11:25:00.902377Z","iopub.execute_input":"2024-04-29T11:25:00.903117Z","iopub.status.idle":"2024-04-29T11:25:00.943367Z","shell.execute_reply.started":"2024-04-29T11:25:00.903081Z","shell.execute_reply":"2024-04-29T11:25:00.942458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Little Testing","metadata":{"_uuid":"7e038d9f-4263-4f5b-9994-6350a1c24d36","_cell_guid":"e9f3a9b2-0e5b-4d4c-91b0-973d951fa825","trusted":true}},{"cell_type":"code","source":"data = {\n    'category1': ['type1', 'type2', 'type3', 'type1', 'type2', 'unknown'],\n    'category2': ['A', 'B', 'C', 'unknown', 'B', 'A'],\n    'target': [1, 0, 1, 0, 1, 0]\n}\n\na = pl.DataFrame(data)\na = a.pipe(Pipeline.filter_cols)\na","metadata":{"_uuid":"e222eed4-8d19-4f0a-8219-cb93cde520a2","_cell_guid":"2cb47925-5b91-4943-8556-9272d589a22c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-04-29T11:25:00.944558Z","iopub.execute_input":"2024-04-29T11:25:00.944842Z","iopub.status.idle":"2024-04-29T11:25:01.076247Z","shell.execute_reply.started":"2024-04-29T11:25:00.944819Z","shell.execute_reply":"2024-04-29T11:25:01.075286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train data","metadata":{"_uuid":"4b896d05-24e0-4f72-bfdf-b237bbd061da","_cell_guid":"31da3188-c204-4aab-9939-fa94d91136e3","trusted":true}},{"cell_type":"code","source":"ROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\n\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR        = ROOT / \"parquet_files\" / \"test\"","metadata":{"_uuid":"3523e5d2-2e63-48bb-a64c-d2b26f288f31","_cell_guid":"2b2892c7-5d06-48e3-ab9d-169430954ba0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-04-29T11:25:01.078479Z","iopub.execute_input":"2024-04-29T11:25:01.078760Z","iopub.status.idle":"2024-04-29T11:25:01.083390Z","shell.execute_reply.started":"2024-04-29T11:25:01.078736Z","shell.execute_reply":"2024-04-29T11:25:01.082453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_store = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n        read_file(TRAIN_DIR / \"train_applprev_2.parquet\", 2),\n        read_file(TRAIN_DIR / \"train_person_2.parquet\", 2)\n    ]\n}","metadata":{"_uuid":"51d5f86c-8aa6-468e-9ca4-d59dfc5c585c","_cell_guid":"7364a05e-c4a2-466a-8bb8-b10e5b8fecff","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-04-29T11:25:01.084526Z","iopub.execute_input":"2024-04-29T11:25:01.084777Z","iopub.status.idle":"2024-04-29T11:27:09.428522Z","shell.execute_reply.started":"2024-04-29T11:25:01.084757Z","shell.execute_reply":"2024-04-29T11:27:09.427497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Processing data","metadata":{"_uuid":"93b52652-9a66-46cf-8b97-b64e750e51d8","_cell_guid":"e8c4c56c-33e3-45e8-9eca-461008944f10","trusted":true}},{"cell_type":"code","source":"\ndf_train = feature_eng(**data_store) # import train data \nprint(\"train data shape:\\t\", df_train.shape)\n# gc.collect()\n# spamming gc.collect praying for memory to not full\ngc.collect()\ndf_train = df_train.pipe(Pipeline.filter_cols) # fillter column\ngc.collect()\ndf_train, cat_cols = to_pandas(df_train) # tranform to pandas dataframe, easier to work with\ngc.collect()\ndf_train = reduce_mem_usage(df_train) # as the name said\ngc.collect()\nprint(\"train data shape:\\t\", df_train.shape)\nnums=df_train.select_dtypes(exclude='category').columns\n# IDK what is going on for now\nfrom itertools import combinations, permutations\n#df_train=df_train[nums]\nnans_df = df_train[nums].isna()\nnans_groups={}\nfor col in nums:\n    cur_group = nans_df[col].sum()\n    try:\n        nans_groups[cur_group].append(col)\n    except:\n        nans_groups[cur_group]=[col]\ndel nans_df; x=gc.collect()\n\ndef reduce_group(grps):\n    use = []\n    for g in grps:\n        mx = 0; vx = g[0]\n        for gg in g:\n            n = df_train[gg].nunique()\n            if n>mx:\n                mx = n\n                vx = gg\n            #print(str(gg)+'-'+str(n),', ',end='')\n        use.append(vx)\n        #print()\n    print('Use these',use)\n    return use\n\ndef group_columns_by_correlation(matrix, threshold=0.8):\n    correlation_matrix = matrix.corr()\n    groups = []\n    remaining_cols = list(matrix.columns)\n    while remaining_cols:\n        col = remaining_cols.pop(0)\n        group = [col]\n        correlated_cols = [col]\n        for c in remaining_cols:\n            if correlation_matrix.loc[col, c] >= threshold:\n                group.append(c)\n                correlated_cols.append(c)\n        groups.append(group)\n        remaining_cols = [c for c in remaining_cols if c not in correlated_cols]\n    return groups\n\nuses=[]\nfor k,v in nans_groups.items():\n    if len(v)>1:\n            Vs = nans_groups[k]\n            #cross_features=list(combinations(Vs, 2))\n            #make_corr(Vs)\n            grps= group_columns_by_correlation(df_train[Vs], threshold=0.8)\n            use=reduce_group(grps)\n            uses=uses+use\n            #make_corr(use)\n    else:\n        uses=uses+v\n    print('####### NAN count =',k)\nprint(uses)\nprint(len(uses))\nuses=uses+list(df_train.select_dtypes(include='category').columns)\nprint(len(uses))\ndf_train=df_train[uses]","metadata":{"_uuid":"07a3b06a-24f2-4fe1-b346-017fc08ccb98","_cell_guid":"a73a013d-46f2-43ea-801b-062090937688","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-04-29T11:27:09.430320Z","iopub.execute_input":"2024-04-29T11:27:09.431106Z","iopub.status.idle":"2024-04-29T11:28:17.677514Z","shell.execute_reply.started":"2024-04-29T11:27:09.431067Z","shell.execute_reply":"2024-04-29T11:28:17.676469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/sample_submission.csv\")\ndevice='gpu'\n#n_samples=200000\nn_est=6000\nDRY_RUN = True if sample.shape[0] == 10 else False   \nif DRY_RUN:\n    device='cpu'\n    df_train = df_train.iloc[:600000]\n    #n_samples=10000\n    n_est=600\nprint(device)","metadata":{"_uuid":"d221dccd-89df-4499-a18e-0197ce036e3f","_cell_guid":"1140d4a2-806e-4c9b-9930-cef518219718","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-04-29T11:28:17.678786Z","iopub.execute_input":"2024-04-29T11:28:17.679097Z","iopub.status.idle":"2024-04-29T11:28:17.692448Z","shell.execute_reply.started":"2024-04-29T11:28:17.679072Z","shell.execute_reply":"2024-04-29T11:28:17.691355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Selection","metadata":{"_uuid":"7e0486b3-5aa1-4646-b7e5-d4135ae3f949","_cell_guid":"13ce4543-f01f-45d9-80ee-25fc22a27225","trusted":true}},{"cell_type":"code","source":"y = df_train[\"target\"]\nweeks = df_train[\"WEEK_NUM\"]\ndf_train= df_train.drop(columns=[\"target\", \"case_id\", \"WEEK_NUM\"])\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)","metadata":{"_uuid":"56c4adfe-2822-427e-b1b8-454809302918","_cell_guid":"c97de980-d95f-42b0-9963-4e634796d67f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-04-29T11:28:17.693768Z","iopub.execute_input":"2024-04-29T11:28:17.694414Z","iopub.status.idle":"2024-04-29T11:28:18.170334Z","shell.execute_reply.started":"2024-04-29T11:28:17.694378Z","shell.execute_reply":"2024-04-29T11:28:18.169280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Applying OrdinalEncoder to handle catagory columns","metadata":{"_uuid":"6ff7b7db-be32-4c9d-afcf-e9c519756f29","_cell_guid":"d5bc6b04-91cd-4052-a5b4-27e498b7cde0","trusted":true}},{"cell_type":"code","source":"df_train[cat_cols] = df_train[cat_cols].astype(str)\nimport polars as pl\nfrom sklearn.preprocessing import OrdinalEncoder\n\n\n# Fit Ordinal Encoder on Training Data\nencoder = OrdinalEncoder(handle_unknown=\"use_encoded_value\", unknown_value=np.nan)\nencoder.fit(df_train[cat_cols])\n\n# Transform Training Data\ndf_train[cat_cols] = encoder.transform(df_train[cat_cols])\ndf_train[cat_cols] = df_train[cat_cols].fillna(0)\ndf_train[cat_cols] = df_train[cat_cols].astype(int)\ndf_train=df_train.fillna(0)","metadata":{"_uuid":"52aae2ef-af43-4a07-8a61-bc499750808c","_cell_guid":"2c58c21e-3fa4-46a1-b964-c0c82c00e69c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-04-29T11:28:18.171725Z","iopub.execute_input":"2024-04-29T11:28:18.172036Z","iopub.status.idle":"2024-04-29T11:28:34.931348Z","shell.execute_reply.started":"2024-04-29T11:28:18.172011Z","shell.execute_reply":"2024-04-29T11:28:34.930349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Setting up parameters","metadata":{"_uuid":"4dc6697e-619b-4635-ae3f-3b580945edab","_cell_guid":"c1e24bff-74c2-4f4a-9169-696821e35e24","trusted":true}},{"cell_type":"code","source":"\n\nparams = {\n    \"max_bin\": 63,\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 10,  \n    \"learning_rate\": 0.05,\n    \"n_estimators\": 2000,  \n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\":True,\n    'num_leaves':64,\n    \"device\": \"gpu\", \n    \"verbose\": -1,\n}\nparams2 = {\n    \"booster\": \"gbtree\",           # Use tree-based boosting\n    \"objective\": \"binary:logistic\",# Binary classification problem\n    \"eval_metric\": \"auc\",          # Evaluation metric: AUC\n    \"max_depth\": 10,               # Maximum tree depth\n    \"learning_rate\": 0.05,         # Learning rate\n    \"n_estimators\": 1000,          # Number of boosting rounds\n    \"colsample_bytree\": 0.8,       # Subsample ratio of columns when constructing each tree\n    \"colsample_bynode\": 0.8,       # Subsample ratio of columns for each node\n    \"alpha\": 0.1,                  # L1 regularization term on weights\n    \"lambda\": 10,                  # L2 regularization term on weights\n    \"tree_method\": 'gpu_hist',     # Use GPU for training if available\n    \"random_state\": 42,            # Random seed for reproducibility\n    \"verbosity\": 0,                # Verbosity of printing messages\n    \"enable_categorical\": True    # Enable categorical features\n}","metadata":{"_uuid":"ce695e9a-5b08-4ea6-b882-a3a009686348","_cell_guid":"29a958f6-2884-4647-b3e8-8ebc242de06b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-04-29T11:28:34.935027Z","iopub.execute_input":"2024-04-29T11:28:34.935304Z","iopub.status.idle":"2024-04-29T11:28:34.942986Z","shell.execute_reply.started":"2024-04-29T11:28:34.935281Z","shell.execute_reply":"2024-04-29T11:28:34.942077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train.to_csv('df_train.csv')","metadata":{"_uuid":"f409436e-2e18-4f8c-9def-33d28b5403b5","_cell_guid":"b653fb17-2f40-406b-b3cc-a743c755a5c5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-04-29T11:28:34.944058Z","iopub.execute_input":"2024-04-29T11:28:34.944385Z","iopub.status.idle":"2024-04-29T11:28:34.954143Z","shell.execute_reply.started":"2024-04-29T11:28:34.944347Z","shell.execute_reply":"2024-04-29T11:28:34.953189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"9cbc7b11-2d13-443e-b03e-b8b02a39ffb0","_cell_guid":"094549ba-b6a7-4c1d-b843-69048c147491","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom sklearn.metrics import roc_auc_score, roc_curve\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"9bd5e97f-5b9a-4887-beaa-f9ad5c55d7e7","_cell_guid":"78acb151-1f6b-4de5-b0a8-0d0131c0e411","collapsed":false,"execution":{"iopub.status.busy":"2024-04-29T11:28:34.955309Z","iopub.execute_input":"2024-04-29T11:28:34.955914Z","iopub.status.idle":"2024-04-29T11:28:35.171583Z","shell.execute_reply.started":"2024-04-29T11:28:34.955882Z","shell.execute_reply":"2024-04-29T11:28:35.170812Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"GPT","metadata":{"_uuid":"11a3948e-ee70-44e0-aead-4c7d69dac3f1","_cell_guid":"efc91c2d-7ed6-4dc6-b338-6c8104d8fe1b","trusted":true}},{"cell_type":"code","source":"# from imblearn.under_sampling import RandomUnderSampler\n\n# # Before undersampling\n# count_class_1_before = y.sum()  # Count of target == 1 before undersampling\n# count_class_0_before = len(df_train) - count_class_1_before  # Count of target == 0 before undersampling\n\n# print(\"Before undersampling:\")\n# print(f\"Count of target == 1: {count_class_1_before}\")\n# print(f\"Count of target == 0: {count_class_0_before}\")\n\n# # Apply random undersampling\n# rus = RandomUnderSampler(random_state=42)\n# df_train, y = rus.fit_resample((df_train), y)\n\n# # After undersampling\n# count_class_1_after = y.sum()  # Count of target == 1 after undersampling\n# count_class_0_after = len(df_train) - count_class_1_after  # Count of target == 0 after undersampling\n\n# print(\"\\nAfter undersampling:\")\n# print(f\"Count of target == 1: {count_class_1_after}\")\n# print(f\"Count of target == 0: {count_class_0_after}\")","metadata":{"_uuid":"f28441b2-7f3a-4811-997c-15172587f2f3","_cell_guid":"2d5a072b-7a73-4b24-81c9-fc2308084a30","collapsed":false,"execution":{"iopub.status.busy":"2024-04-29T11:28:35.172556Z","iopub.execute_input":"2024-04-29T11:28:35.172797Z","iopub.status.idle":"2024-04-29T11:28:35.177403Z","shell.execute_reply.started":"2024-04-29T11:28:35.172776Z","shell.execute_reply":"2024-04-29T11:28:35.176461Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"_uuid":"eff8b112-69b6-415e-9f5f-dda27e05cbed","_cell_guid":"5499f96f-4940-4ec2-84b3-f64005f2bf5f","collapsed":false,"execution":{"iopub.status.busy":"2024-04-29T11:28:35.178788Z","iopub.execute_input":"2024-04-29T11:28:35.179073Z","iopub.status.idle":"2024-04-29T11:28:35.362895Z","shell.execute_reply.started":"2024-04-29T11:28:35.179051Z","shell.execute_reply":"2024-04-29T11:28:35.361885Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Finding best model for this dataset","metadata":{"_uuid":"6ffffe9f-0af1-40db-a78b-158411961d09","_cell_guid":"43306e8a-f2a1-400e-9f32-89cb0d6938e1","trusted":true}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report","metadata":{"_uuid":"7348f9da-eef3-4f1a-a8b0-0a6296dd7355","_cell_guid":"5ff6ca7e-b6fe-4494-875d-292e9842c96a","collapsed":false,"execution":{"iopub.status.busy":"2024-04-29T11:28:35.364507Z","iopub.execute_input":"2024-04-29T11:28:35.365268Z","iopub.status.idle":"2024-04-29T11:28:35.369940Z","shell.execute_reply.started":"2024-04-29T11:28:35.365228Z","shell.execute_reply":"2024-04-29T11:28:35.368984Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{"_uuid":"c2ae2c2d-dc4a-445b-bc9d-698cde410ffb","_cell_guid":"78257908-ad62-4d5e-bb5d-4a1690305f3d","trusted":true}},{"cell_type":"code","source":"","metadata":{"_uuid":"c64c5c60-e3e3-4504-9d40-fd66c6970047","_cell_guid":"eae0cc8a-1efa-4e5b-8be3-266dbdf5ee34","collapsed":false,"execution":{"iopub.status.busy":"2024-04-29T11:28:35.371476Z","iopub.execute_input":"2024-04-29T11:28:35.371839Z","iopub.status.idle":"2024-04-29T11:31:49.039194Z","shell.execute_reply.started":"2024-04-29T11:28:35.371802Z","shell.execute_reply":"2024-04-29T11:31:49.038302Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LightGBM vs RandomForest","metadata":{"_uuid":"62063372-0073-4505-853e-0e93b04a191a","_cell_guid":"114744e9-49db-4218-a077-f8699db05f14","trusted":true}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfitted_models = []\ncv_scores = []\nfitted_models_rf = []  # List to store fitted XGBoost models\ncv_scores_rf = []  # List to store CV scores\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)\nplt.figure(figsize=(10,6))\nfor idx_train, idx_valid in cv.split(df_train, y, groups=weeks):\n    X_train, y_train = df_train.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = df_train.iloc[idx_valid], y.iloc[idx_valid]\n        \n    # Random Forest model\n    model_rf = RandomForestClassifier()\n    start_time = time.time()\n    model_rf.fit(X_train, y_train)\n    end_time = time.time()\n    training_time = end_time - start_time\n    print(f\"Random Forest Training time: {training_time} seconds\")\n    fitted_models_rf.append( model_rf )\n    # Calculate ROC AUC score for Random Forest\n    \n    y_pred_valid_rf = model_rf.predict_proba(X_valid)[:, 1]\n    y_pred_valid_bin = model_rf.predict(X_valid)\n    auc_score_rf = roc_auc_score(y_valid, y_pred_valid_rf)\n    print(classification_report(y_valid, y_pred_valid_bin))\n    print(confusion_matrix(y_valid, y_pred_valid_bin))\n    cv_scores_rf.append(auc_score_rf)\n\n\n    # Plot ROC curve for RandomForest\n    fpr_rf, tpr_rf, _ = roc_curve(y_valid, y_pred_valid_rf)\n    plt.plot(fpr_rf, tpr_rf, label=f'RandomForest ROC curve (AUC = {auc_score_rf:.2f})')\n\nplt.plot([0, 1], [0, 1], 'k--', label='Random guess')\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver Operating Characteristic (ROC) Curve')\nplt.legend()\nplt.show()","metadata":{"_uuid":"2e3c908c-4fa6-454d-96f2-141966411991","_cell_guid":"9ffe251c-4777-4eb4-af76-713597bde873","collapsed":false,"execution":{"iopub.status.busy":"2024-04-29T11:31:49.040311Z","iopub.execute_input":"2024-04-29T11:31:49.040572Z","iopub.status.idle":"2024-04-29T12:08:08.381140Z","shell.execute_reply.started":"2024-04-29T11:31:49.040549Z","shell.execute_reply":"2024-04-29T12:08:08.380234Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LightGBM vs SVM","metadata":{"_uuid":"e256fb12-21e6-47ae-9794-ffa269c0d52b","_cell_guid":"01f61d66-1144-4c51-ae22-8c00f1703a34","trusted":true}},{"cell_type":"code","source":"fitted_models = fitted_models_rf","metadata":{"_uuid":"9adad7a0-24fd-4328-9423-d2712c7b372e","_cell_guid":"f302b3b8-8584-4928-b479-4c15236459bf","collapsed":false,"execution":{"iopub.status.busy":"2024-04-29T12:08:08.382359Z","iopub.execute_input":"2024-04-29T12:08:08.382641Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n    \n    def predict_proba(self, X):\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n\nmodel = VotingModel(fitted_models)","metadata":{"_uuid":"f3de6ea6-c976-4231-af92-58a03ac133bb","_cell_guid":"66aba23b-233e-41f2-9c9f-295aac1178f6","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lgb.plot_importance(fitted_models[2], importance_type=\"split\", figsize=(10,50))\n# plt.show()","metadata":{"_uuid":"a41cd066-8a7b-4eb9-9424-af73640f5955","_cell_guid":"47b8475b-e4f1-4770-8324-8b35d219265a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# features = X_train.columns\n# importances = fitted_models[2].feature_importances_\n# feature_importance = pd.DataFrame({'importance':importances,'features':features}).sort_values('importance', ascending=False).reset_index(drop=True)\n# feature_importance","metadata":{"_uuid":"d85c5cd6-66da-477e-bdbb-c29002884c9f","_cell_guid":"8d897ca4-665a-43c9-bdc6-6f7862923b8b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop_list = []\n# for i, f in feature_importance.iterrows():\n#     if f['importance']<80:\n#         drop_list.append(f['features'])\n# print(f\"Number of features which are not important: {len(drop_list)} \")","metadata":{"_uuid":"41eb90b7-95c2-4e97-9407-6e9cb33e62da","_cell_guid":"ddb301c3-344c-4b5b-9d16-cb947af10e4c","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(drop_list)","metadata":{"_uuid":"f9d135fb-e7a1-4668-ad41-67f0ba3a8a2d","_cell_guid":"38123ad3-3661-4f09-867a-b852574ef740","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train = df_train.drop(columns=drop_list)","metadata":{"_uuid":"b706187d-9132-42a0-8413-84ec9fad0092","_cell_guid":"ea537abe-b38b-447e-a243-3dd3ea76b039","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# params = {\n#     \"max_bin\": 63,\n#     \"boosting_type\": \"gbdt\",\n#     \"objective\": \"binary\",\n#     \"metric\": \"auc\",\n#     \"max_depth\": 10,  \n#     \"learning_rate\": 0.05,\n#     \"n_estimators\": 2000,  \n#     \"colsample_bytree\": 0.8,\n#     \"colsample_bynode\": 0.8,\n#     \"verbose\": -1,\n#     \"random_state\": 42,\n#     \"reg_alpha\": 0.1,\n#     \"reg_lambda\": 10,\n#     \"extra_trees\":True,\n#     'num_leaves':64,\n#     \"device\": \"gpu\", \n#     \"verbose\": -1,\n# }","metadata":{"_uuid":"be1be572-3108-4128-800f-2e35c9b0d3d1","_cell_guid":"8d9909ce-d6f4-4e38-be3a-83c519a89e81","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = lgb.LGBMClassifier(**params)\n# model = model.fit(\n#         df_train, y)","metadata":{"_uuid":"76b0f4d6-4c96-4f5c-a68c-8c6f6c6f1f29","_cell_guid":"5ea260f6-c623-48e8-b20c-bc389cbff7ea","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submision","metadata":{"_uuid":"7a822d32-6f0f-4275-ba12-2a026d15429d","_cell_guid":"e5a9ff6c-07cc-4933-9ba7-f453ad50b044","trusted":true}},{"cell_type":"code","source":"data_store_test = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n        read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n        read_file(TEST_DIR / \"test_applprev_2.parquet\", 2),\n        read_file(TEST_DIR / \"test_person_2.parquet\", 2)\n    ]\n}","metadata":{"_uuid":"48ff563b-dc1b-4363-be62-aca7ee119e64","_cell_guid":"7e98e9c3-b134-4e32-9b9c-44caf01cffb7","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_store_test)\nprint(\"test data shape:\\t\", df_test.shape)\nprint(df_test[\"case_id\"])\ngc.collect()\nprint(\"train data shape:\\t\", df_train.shape)\nprint(\"test data shape:\\t\", df_test.shape)\ndf_test, cat_cols = to_pandas(df_test, cat_cols)\ndf_test = df_test.set_index(\"case_id\")\ndf_test = reduce_mem_usage(df_test)\ndf_test[cat_cols] = df_test[cat_cols].astype(str)\ngc.collect()","metadata":{"_uuid":"adcfa5d1-0c93-4adb-99cf-b885fa77dc3d","_cell_guid":"e5564d4a-5f6f-4a42-a881-d2d66b3ec170","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test","metadata":{"_uuid":"612e2633-ff76-4daa-ba06-eb922f9c9f0b","_cell_guid":"cfeda6b8-7fe6-4889-97d3-acbe0d482c88","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transform Test Data using the same encoder instance to ensure consistency\ndf_test[cat_cols] = encoder.transform(df_test[cat_cols])\ndf_test[cat_cols] = df_test[cat_cols].fillna(-1)\ndf_test[cat_cols] = df_test[cat_cols].astype(int)\ndf_test=df_test.fillna(0)\ndf_test = df_test[df_train.columns]","metadata":{"_uuid":"97e65bff-8423-44ee-9f46-594e93a117ca","_cell_guid":"c4e0f0cd-5da1-4024-b4a2-7409b6013952","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test","metadata":{"_uuid":"115e6510-a44f-4208-aaec-50df5c9b59f8","_cell_guid":"35f64298-9884-47e7-b04f-ce3ab2c97952","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nlgb_pred = pd.Series(model.predict_proba(df_test)[:, 1],index=df_test.index)","metadata":{"_uuid":"663fb891-3fd2-42b1-9518-84f2a4498caa","_cell_guid":"283f0936-5de9-43d9-9aad-ec56ed24d72c","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_pred","metadata":{"_uuid":"a48738fc-88b8-4db2-b512-722de203ee31","_cell_guid":"9b4b73a2-26fe-4d08-ab4e-0189c6ab0926","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm = pd.read_csv(ROOT / \"sample_submission.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\n\ndf_subm[\"score\"] = lgb_pred","metadata":{"_uuid":"b2ba52b2-7595-43f6-b755-5b5a1e762fc7","_cell_guid":"5fb5ccda-be1c-4b9f-a404-2b8162a5b505","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm.to_csv('submission.csv')\ndf_subm","metadata":{"_uuid":"a7642d35-a37b-4064-b2e9-8d1995e7c47b","_cell_guid":"09c119b0-421f-4229-ba74-1dd2db7ecfea","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"b7e799af-6ad3-48ea-b535-0193a6a511cd","_cell_guid":"aa251538-76c9-4d20-9ca6-e35487df9d04","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}