{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:34:25.861032Z","iopub.execute_input":"2025-05-29T13:34:25.861311Z","iopub.status.idle":"2025-05-29T13:34:27.009297Z","shell.execute_reply.started":"2025-05-29T13:34:25.861287Z","shell.execute_reply":"2025-05-29T13:34:27.008584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Verify GPU availability\nimport torch\nprint(\"GPU Available:\", torch.cuda.is_available())\nprint(\"Number of GPUs:\", torch.cuda.device_count())\nprint(\"GPU Name:\", torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"No GPU\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T12:19:14.937559Z","iopub.execute_input":"2025-05-12T12:19:14.938327Z","iopub.status.idle":"2025-05-12T12:19:19.683941Z","shell.execute_reply.started":"2025-05-12T12:19:14.938302Z","shell.execute_reply":"2025-05-12T12:19:19.683314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import Libraries\nimport polars as pl\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\nimport torch\nimport gc\n\n%matplotlib inline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:35:29.751508Z","iopub.execute_input":"2025-05-29T13:35:29.752260Z","iopub.status.idle":"2025-05-29T13:35:38.938637Z","shell.execute_reply.started":"2025-05-29T13:35:29.752234Z","shell.execute_reply":"2025-05-29T13:35:38.938051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ## 1. Load Data\n# Load and merge the raw data files with relational structure, using lazy loading to manage memory.\ndef load_raw_data(data_dir: Path, train=True) -> pl.DataFrame:\n    \"\"\"Load and merge raw data files with relational structure, optimized for memory.\"\"\"\n    base_path = data_dir / \"parquet_files\" / (\"train\" if train else \"test\")\n    \n    # Load base table lazily\n    base = pl.scan_parquet(base_path / f\"{'train' if train else 'test'}_base.parquet\")\n    \n    # Load and merge static tables (limit to first file to reduce memory usage)\n    static_files = list(base_path.glob(f\"{'train' if train else 'test'}_static_0_*.parquet\"))[:1] + \\\n                   [base_path / f\"{'train' if train else 'test'}_static_cb_0.parquet\"]\n    df = base\n    for static_file in static_files:\n        static_df = pl.scan_parquet(static_file)\n        print(f\"Merging with {static_file.name}\")\n        df = df.join(static_df, on=\"case_id\", how=\"left\")\n        gc.collect()  # Free memory after each merge\n\n    # Load and aggregate credit bureau data (limit to first file to reduce memory usage)\n    cb_files = list(base_path.glob(f\"{'train' if train else 'test'}_credit_bureau_a_1_*.parquet\"))[:1]\n    cb_dfs = [pl.scan_parquet(f) for f in cb_files]\n    cb_data = pl.concat(cb_dfs, how=\"vertical_relaxed\")\n    # Collect a small sample to inspect columns (minimal memory usage)\n    cb_sample = cb_data.head(10).collect()\n    print(f\"Credit Bureau columns: {cb_sample.columns}\")\n    numeric_cols = [col for col in cb_sample.columns if cb_sample[col].dtype in [pl.Float64, pl.Int64]]\n    agg_col = numeric_cols[0] if numeric_cols else None\n    cb_agg = cb_data.group_by(\"case_id\").agg(\n        num_records=pl.len().alias(\"cb_num_records\"),\n        **({f\"{agg_col}_mean\": pl.col(agg_col).mean().alias(f\"{agg_col}_mean\")} if agg_col else {})\n    )\n    df = df.join(cb_agg, on=\"case_id\", how=\"left\")\n    gc.collect()\n\n    # Load and aggregate previous application data with schema alignment (limit to first file to reduce memory usage)\n    applprev_files = list(base_path.glob(f\"{'train' if train else 'test'}_applprev_*.parquet\"))[:1]\n    applprev_dfs = [pl.scan_parquet(f) for f in applprev_files]\n    # Align schemas\n    all_columns = set().union(*[set(df.collect_schema().names()) for df in applprev_dfs])\n    aligned_applprev_dfs = []\n    for df in applprev_dfs:\n        missing_cols = all_columns - set(df.collect_schema().names())\n        if missing_cols:\n            for col in missing_cols:\n                df = df.with_columns(pl.lit(None).alias(col))\n        # Reorder columns to match the first DataFrame's schema\n        df = df.select(sorted(all_columns, key=lambda x: list(applprev_dfs[0].collect_schema().names()).index(x) if x in applprev_dfs[0].collect_schema().names() else len(applprev_dfs[0].collect_schema().names())))\n        aligned_applprev_dfs.append(df)\n    applprev_data = pl.concat(aligned_applprev_dfs, how=\"vertical_relaxed\")\n    # Collect a small sample to inspect columns\n    applprev_sample = applprev_data.head(10).collect()\n    print(f\"Previous Application columns: {applprev_sample.columns}\")\n    applprev_agg = applprev_data.group_by(\"case_id\").agg(\n        num_prev_apps=pl.len().alias(\"num_prev_apps\"),\n        **({\"prev_credamount_mean\": pl.col(\"credamount_590A\").mean().alias(\"prev_credamount_mean\")} if \"credamount_590A\" in applprev_sample.columns else {})\n    )\n    df = df.join(applprev_agg, on=\"case_id\", how=\"left\")\n    gc.collect()\n\n    # Collect the lazy DataFrame\n    df = df.collect()\n    return df\n\n# Load train and test data\ndata_path = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\ntrain_df_full = load_raw_data(data_path, train=True)\nprint(f\"Shape of train_df_full before sampling: {train_df_full.shape}\")\ntest_df = load_raw_data(data_path, train=False)\n\n# Sample the training data to reduce memory usage\nsample_size = 0.3  # Increase to 30% of data\ntrain_df = train_df_full.sample(fraction=sample_size, seed=42)\nprint(f\"Shape of train_df after sampling: {train_df.shape}\")\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:35:42.077621Z","iopub.execute_input":"2025-05-29T13:35:42.078586Z","iopub.status.idle":"2025-05-29T13:35:46.050697Z","shell.execute_reply.started":"2025-05-29T13:35:42.078561Z","shell.execute_reply":"2025-05-29T13:35:46.049981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.write_csv(\"train_df.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:36:14.504515Z","iopub.execute_input":"2025-05-29T13:36:14.505275Z","iopub.status.idle":"2025-05-29T13:36:15.256057Z","shell.execute_reply.started":"2025-05-29T13:36:14.505241Z","shell.execute_reply":"2025-05-29T13:36:15.255266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head(30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:36:17.857113Z","iopub.execute_input":"2025-05-29T13:36:17.857392Z","iopub.status.idle":"2025-05-29T13:36:17.870483Z","shell.execute_reply.started":"2025-05-29T13:36:17.857371Z","shell.execute_reply":"2025-05-29T13:36:17.869699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Columns in train_df:\", train_df.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:36:25.447305Z","iopub.execute_input":"2025-05-29T13:36:25.447746Z","iopub.status.idle":"2025-05-29T13:36:25.451784Z","shell.execute_reply.started":"2025-05-29T13:36:25.447721Z","shell.execute_reply":"2025-05-29T13:36:25.451057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:36:29.186380Z","iopub.execute_input":"2025-05-29T13:36:29.186873Z","iopub.status.idle":"2025-05-29T13:36:29.424258Z","shell.execute_reply.started":"2025-05-29T13:36:29.186848Z","shell.execute_reply":"2025-05-29T13:36:29.423540Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport gc\n\n# Assume train_df and test_df are already loaded as Polars DataFrames\n\ndef preprocess_data(df: pl.DataFrame) -> pl.DataFrame:\n    \"\"\"Preprocess the data by handling missing values, outliers, and encoding.\"\"\"\n    df_processed = df.clone() # Work on a clone\n\n    # Impute numerical columns with median and handle outliers\n    numerical_cols = [col for col, dtype in df_processed.schema.items() if dtype.is_numeric()]\n    \n    for col in numerical_cols:\n        # Impute with median\n        col_median = df_processed.select(pl.col(col).median()).item()\n        \n        if col_median is None:\n            fill_value_for_num = 0.0\n            df_processed = df_processed.with_columns(pl.col(col).fill_null(fill_value_for_num))\n        else:\n            df_processed = df_processed.with_columns(pl.col(col).fill_null(col_median))\n            \n        # Cap outliers at 99th percentile (only for specified columns)\n        if col in [\"actualdpd_943P\", \"childnum_21L\", \"credamount_590A\", \"mainoccupationinc_437A\"]:\n            if df_processed[col].null_count() < len(df_processed): # Check if column is not all nulls\n                percentile_99_series = df_processed.select(pl.col(col).quantile(0.99, \"midpoint\")).get_column(col)\n                if not percentile_99_series.is_empty() and percentile_99_series[0] is not None:\n                    percentile_99 = percentile_99_series[0]\n                    \n                    # --- THIS IS THE CORRECTED SYNTAX ---\n                    # Ensure upper_bound is not less than lower_bound.\n                    # For these columns, percentile_99 should be >= 0.\n                    actual_upper_bound = max(0.0, percentile_99) \n                    \n                    df_processed = df_processed.with_columns(\n                        pl.col(col).clip(lower_bound=0.0, upper_bound=actual_upper_bound) # Use 0.0 for float consistency\n                    )\n                # else: percentile_99 could not be computed, consider skipping clipping or using a default.\n\n        # Log-transform for skewed columns\n        if col in [\"credamount_590A\", \"mainoccupationinc_437A\", \"annuity_853A\"]:\n            df_processed = df_processed.with_columns(\n                (pl.when(pl.col(col) >= 0).then(pl.col(col)).otherwise(0) + 1).log().alias(f\"log_{col}\")\n            )\n\n    # Impute existing categorical columns with mode\n    categorical_cols = [col for col, dtype in df_processed.schema.items() if dtype == pl.Categorical]\n    for col in categorical_cols:\n        mode_series = df_processed.select(pl.col(col).mode().first()).get_column(col)\n        \n        if mode_series.is_empty() or mode_series[0] is None:\n            actual_mode_to_fill = \"Unknown_Mode_Cat\"\n        else:\n            actual_mode_to_fill = mode_series[0]\n            \n        df_processed = df_processed.with_columns(pl.col(col).fill_null(actual_mode_to_fill))\n\n    # Handle string columns (cast to categorical after imputation)\n    string_cols = [\"education_1138M\", \"familystate_726L\", \"credtype_587L\", \"profession_152M\", \n                   \"cancelreason_3545846M\", \"district_544M\", \"postype_4733339M\", \n                   \"rejectreason_755M\", \"rejectreasonclient_4145042M\", \"status_219L\"]\n    for col in string_cols:\n        if col in df_processed.columns:\n            if df_processed.schema[col] == pl.Categorical:\n                continue\n\n            mode_series = df_processed.select(pl.col(col).mode().first()).get_column(col)\n\n            if mode_series.is_empty() or mode_series[0] is None:\n                actual_mode_to_fill = \"Unknown_Mode_Str\"\n            else:\n                actual_mode_to_fill = mode_series[0]\n            \n            df_processed = df_processed.with_columns(\n                pl.col(col).fill_null(actual_mode_to_fill).cast(pl.Categorical)\n            )\n\n    # Convert time columns\n    time_cols = [\"employedfrom_700D\", \"dtlastpmt_581D\", \"creationdate_885D\", \"dtlastpmtallstes_3545839D\", \n                 \"firstnonzeroinstldate_307D\", \"dateactivated_425D\", \"approvaldate_319D\"]\n    for col in time_cols:\n        if col in df_processed.columns:\n            if df_processed.schema[col] == pl.Utf8: # If it's a string that should be a number\n                 df_processed = df_processed.with_columns(\n                    pl.col(col).cast(pl.Float64, strict=False) # Try to cast to float, nullify if fails\n                 )\n\n            # Now assume it's a numeric type (or became one)\n            if df_processed.schema[col].is_numeric():\n                df_processed = df_processed.with_columns(\n                    (pl.col(col).fill_null(0) / -365.25).alias(f\"{col}_years\")\n                )\n\n    return df_processed\n\n# # Example Usage:\n# # Assuming train_df and test_df are Polars DataFrames loaded elsewhere\nif 'train_df' in locals() and isinstance(train_df, pl.DataFrame):\n    train_processed = preprocess_data(train_df) # Pass the DataFrame directly\n    print(\"Train data processed.\")\n    print(train_processed.head())\nelse:\n    print(\"train_df is not a Polars DataFrame or not defined.\")\n\nif 'test_df' in locals() and isinstance(test_df, pl.DataFrame):\n     test_processed = preprocess_data(test_df) # Pass the DataFrame directly\n     print(\"Test data processed.\")\n     # print(test_processed.head())\nelse:\n     print(\"test_df is not a Polars DataFrame or not defined.\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T12:26:10.781886Z","iopub.execute_input":"2025-05-12T12:26:10.782147Z","iopub.status.idle":"2025-05-12T12:26:11.847601Z","shell.execute_reply.started":"2025-05-12T12:26:10.782127Z","shell.execute_reply":"2025-05-12T12:26:11.846884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_processed.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T12:26:20.375848Z","iopub.execute_input":"2025-05-12T12:26:20.376122Z","iopub.status.idle":"2025-05-12T12:26:20.578422Z","shell.execute_reply.started":"2025-05-12T12:26:20.376102Z","shell.execute_reply":"2025-05-12T12:26:20.577705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}