{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":35332,"databundleVersionId":3723648,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 📊 American Express - Default Prediction \n\n**Group:**\n**Course:** SC4000 Machine Learning &nbsp;•&nbsp; **Platform:** Kaggle   \n**Goal:** Predict customer default using a fast, memory-safe pipeline and GPU-accelerated machine learning models.\n","metadata":{}},{"cell_type":"markdown","source":"### Importing Libraries","metadata":{}},{"cell_type":"markdown","source":"We use NumPy/Pandas for data, a two-pass memory-safe feature builder, and GPU-accelerated\nLightGBM for modeling. The AMEX competition metric is implemented as a custom callback.\n","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, (CSV file I/O)\nimport os, gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nprint('Using TensorFlow version',tf.__version__)\nimport cupy as cp\nimport cudf\nfrom cuml.ensemble import RandomForestClassifier as cuRF\nfrom sklearn.ensemble import RandomForestClassifier as skRF\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nimport math, sys\nimport random\nimport matplotlib.pyplot as plt\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:17:33.539850Z","iopub.execute_input":"2025-11-13T14:17:33.540119Z","iopub.status.idle":"2025-11-13T14:17:33.547158Z","shell.execute_reply.started":"2025-11-13T14:17:33.540099Z","shell.execute_reply":"2025-11-13T14:17:33.546514Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Loading Dataset","metadata":{}},{"cell_type":"code","source":"csv_path = '/kaggle/input/amex-default-prediction/train_labels.csv'\ndf = pd.read_csv(csv_path)\nprint(df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:17:40.857001Z","iopub.execute_input":"2025-11-13T14:17:40.857271Z","iopub.status.idle":"2025-11-13T14:17:41.935749Z","shell.execute_reply.started":"2025-11-13T14:17:40.857251Z","shell.execute_reply":"2025-11-13T14:17:41.934998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n\n# TENSORFLOW : 8GB RAM\n# RAPIDS: 7GB RAM\nLIMIT = 8\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n  try:\n    tf.config.experimental.set_virtual_device_configuration(\n        gpus[0],\n        [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024*LIMIT)])\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n  except RuntimeError as e:\n    print(e)\nprint('Restrict TensorFlow to max %iGB GPU RAM'%LIMIT)\nprint('RAPIDS to use %iGB GPU RAM'%(15-LIMIT))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:17:50.479236Z","iopub.execute_input":"2025-11-13T14:17:50.479805Z","iopub.status.idle":"2025-11-13T14:17:50.748551Z","shell.execute_reply.started":"2025-11-13T14:17:50.479780Z","shell.execute_reply":"2025-11-13T14:17:50.747747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PATH_TO_CUSTOMER_HASHES = None\nPROCESS_DATA = True\nPATH_TO_DATA = '/kaggle/working/data/'\nTRAIN_MODEL = True\nPATH_TO_MODEL = '/kaggle/working/model/'\nINFER_TEST = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:17:53.728964Z","iopub.execute_input":"2025-11-13T14:17:53.729520Z","iopub.status.idle":"2025-11-13T14:17:53.733043Z","shell.execute_reply.started":"2025-11-13T14:17:53.729497Z","shell.execute_reply":"2025-11-13T14:17:53.732193Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"# read only the header to get column names\ncols = pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', nrows=0).columns\nprint(f\"{len(cols)} columns detected\")\n\n# just peek at dtypes from first few rows (cheap)\nsample = pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', nrows=1000)\nprint(sample.dtypes)\n\n# count total rows (fast line count)\nimport subprocess, shlex\nn = int(subprocess.check_output(shlex.split(\"wc -l /kaggle/input/amex-default-prediction/train_data.csv\")).split()[0]) - 1\nprint(f\"Total rows: {n:,}\")\nsummary = sample.dtypes.value_counts()\nprint(summary)","metadata":{"trusted":true,"scrolled":true,"execution":{"iopub.status.busy":"2025-11-13T14:17:56.297864Z","iopub.execute_input":"2025-11-13T14:17:56.298407Z","iopub.status.idle":"2025-11-13T14:20:31.944244Z","shell.execute_reply.started":"2025-11-13T14:17:56.298385Z","shell.execute_reply":"2025-11-13T14:20:31.943281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.random.seed(42); random.seed(42)\npd.set_option(\"display.max_columns\", 120)\npd.set_option(\"display.width\", 160)\n\nDATA_DIR = \"/kaggle/input/amex-default-prediction\"\nWORK_DIR = \"/kaggle/working\"\n\n# knobs for chunked scans (tune based on RAM)\nEDA_CHUNKSIZE = 2_000_000\nEDA_SAMPLE_ROWS = 1_000_000  # for quick sample-based EDA (set None to scan full)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:20:34.766807Z","iopub.execute_input":"2025-11-13T14:20:34.767355Z","iopub.status.idle":"2025-11-13T14:20:34.771469Z","shell.execute_reply.started":"2025-11-13T14:20:34.767328Z","shell.execute_reply":"2025-11-13T14:20:34.770713Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for fname in [\"train_data.csv\", \"test_data.csv\", \"train_labels.csv\", \"sample_submission.csv\"]:\n    fpath = os.path.join(DATA_DIR, fname)\n    size_gb = os.path.getsize(fpath) / (1024**3)\n    print(f\"{fname:22s}  {size_gb:6.2f} GB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:20:36.974552Z","iopub.execute_input":"2025-11-13T14:20:36.975252Z","iopub.status.idle":"2025-11-13T14:20:36.980609Z","shell.execute_reply.started":"2025-11-13T14:20:36.975225Z","shell.execute_reply":"2025-11-13T14:20:36.980023Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels = pd.read_csv(f\"{DATA_DIR}/train_labels.csv\")\npos_rate = labels[\"target\"].mean()\nprint(f\"Train customers: {len(labels):,}\")\nprint(f\"Positive rate (target=1): {pos_rate:.4f}\")\ndisplay(labels[\"target\"].value_counts().rename_axis(\"target\").to_frame(\"count\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:20:39.888921Z","iopub.execute_input":"2025-11-13T14:20:39.889605Z","iopub.status.idle":"2025-11-13T14:20:40.391595Z","shell.execute_reply.started":"2025-11-13T14:20:39.889580Z","shell.execute_reply":"2025-11-13T14:20:40.390963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EDA-3 — Per-customer statement counts + date range (chunked)\ncust_counts = {}\nmin_date, max_date = None, None\n\nreader = pd.read_csv(f\"{DATA_DIR}/train_data.csv\", usecols=[\"customer_ID\", \"S_2\"], chunksize=EDA_CHUNKSIZE)\nfor i, chunk in enumerate(reader, 1):\n    # track date range\n    chunk[\"S_2\"] = pd.to_datetime(chunk[\"S_2\"])\n    cmin, cmax = chunk[\"S_2\"].min(), chunk[\"S_2\"].max()\n    min_date = cmin if min_date is None or cmin < min_date else min_date\n    max_date = cmax if max_date is None or cmax > max_date else max_date\n\n    # counts within chunk\n    cnt = chunk[\"customer_ID\"].value_counts()\n    for cid, n in cnt.items():\n        cust_counts[cid] = cust_counts.get(cid, 0) + int(n)\n\n    if i % 2 == 0:\n        print(f\"[EDA-3] chunks processed: {i}\")\n    del chunk, cnt\n    gc.collect()\n\ncounts = np.array(list(cust_counts.values()))\nprint(f\"Customers seen: {len(counts):,}\")\nprint(f\"Statements per customer — min/median/mean/max: {counts.min()} / {np.median(counts):.1f} / {counts.mean():.2f} / {counts.max()}\")\nprint(f\"Date range (S_2): {min_date.date()} → {max_date.date()}\")\n\n# histogram of statement counts\nplt.figure(figsize=(6,3.5))\nplt.hist(counts, bins=range(1, counts.max()+2))\nplt.title(\"Statements per Customer (train)\")\nplt.xlabel(\"Statements\"); plt.ylabel(\"Customers\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.548410Z","iopub.status.idle":"2025-11-13T14:12:38.548718Z","shell.execute_reply.started":"2025-11-13T14:12:38.548586Z","shell.execute_reply":"2025-11-13T14:12:38.548595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EDA-4 — Schema + rough missingness on a sample\nsample = pd.read_csv(f\"{DATA_DIR}/train_data.csv\", nrows=EDA_SAMPLE_ROWS)\nprint(\"Sample shape:\", sample.shape)\n\n# dtypes summary\ndtype_counts = sample.dtypes.value_counts()\nprint(\"\\nDtype counts:\\n\", dtype_counts)\n\n# rough missingness %\nna_pct = (sample.isna().sum() / len(sample) * 100).sort_values(ascending=False).head(25)\ndisplay(na_pct.to_frame(\"NA% (approx from sample)\").round(2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.549591Z","iopub.status.idle":"2025-11-13T14:12:38.549902Z","shell.execute_reply.started":"2025-11-13T14:12:38.549798Z","shell.execute_reply":"2025-11-13T14:12:38.549809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EDA-5 — Categorical value counts (common: D_63, D_64)\ncat_cols_to_peek = [c for c in [\"D_63\", \"D_64\"] if c in sample.columns]\nfor c in cat_cols_to_peek:\n    vc = sample[c].value_counts(dropna=False).head(10)\n    print(f\"\\nTop values for {c}:\")\n    display(vc)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.550465Z","iopub.status.idle":"2025-11-13T14:12:38.550715Z","shell.execute_reply.started":"2025-11-13T14:12:38.550571Z","shell.execute_reply":"2025-11-13T14:12:38.550581Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EDA-6 — Histograms for a few numeric columns (from sample)\nnum_cols = [c for c in sample.columns if c not in (\"customer_ID\",\"S_2\") and np.issubdtype(sample[c].dtype, np.number)]\npick = num_cols[:6]  # first few numerics; change to your favorites\n\nfor col in pick:\n    s = sample[col].dropna()\n    if len(s) == 0:\n        continue\n    plt.figure(figsize=(6,3.5))\n    plt.hist(s.values, bins=50)\n    plt.title(f\"{col} (sample)\")\n    plt.xlabel(col); plt.ylabel(\"Frequency\")\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.552200Z","iopub.status.idle":"2025-11-13T14:12:38.552467Z","shell.execute_reply.started":"2025-11-13T14:12:38.552333Z","shell.execute_reply":"2025-11-13T14:12:38.552344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EDA-7 — Correlation snapshot (Spearman) on a small numeric subset\nsub = sample[num_cols].select_dtypes(include=[np.number]).copy()\nsub = sub.iloc[:, :30]  # first 30 numeric cols to keep it light\nsub = sub.fillna(sub.median(numeric_only=True))\n\ncorr = sub.corr(method=\"spearman\").values\nplt.figure(figsize=(5,4.5))\nplt.imshow(corr, interpolation=\"nearest\")\nplt.title(\"Spearman Correlation (sample subset)\")\nplt.colorbar(fraction=0.046, pad=0.04)\nplt.xticks([]); plt.yticks([])\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.553043Z","iopub.status.idle":"2025-11-13T14:12:38.553206Z","shell.execute_reply.started":"2025-11-13T14:12:38.553126Z","shell.execute_reply":"2025-11-13T14:12:38.553133Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. LIGHT GBM (MODEL)","metadata":{}},{"cell_type":"markdown","source":"Importing the model","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\nprint(\"LightGBM version:\", lgb.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:20:46.969818Z","iopub.execute_input":"2025-11-13T14:20:46.970512Z","iopub.status.idle":"2025-11-13T14:20:47.673474Z","shell.execute_reply.started":"2025-11-13T14:20:46.970486Z","shell.execute_reply":"2025-11-13T14:20:47.672727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 2 — AMEX metric + LightGBM callback\nimport numpy as np\n\ndef amex_metric(y_true, y_pred):\n    \"\"\"Official AMEX metric (vectorized).\"\"\"\n    y_true = np.asarray(y_true).astype(int)\n    y_pred = np.asarray(y_pred).astype(float)\n    assert y_true.shape == y_pred.shape\n\n    # Sort by prediction (desc)\n    order = np.argsort(-y_pred, kind=\"mergesort\")\n    y_true = y_true[order]\n\n    # Top 4% by weighted count (0 -> weight 20, 1 -> weight 1)\n    weights = np.where(y_true == 0, 20, 1)\n    cum_w = np.cumsum(weights)\n    cutoff = int(0.04 * cum_w[-1])\n    top_mask = cum_w <= cutoff\n    top_four = y_true[top_mask].sum() / max(1, y_true.sum())\n\n    # Normalized Gini\n    def gini(a, p, w):\n        order = np.argsort(-p, kind=\"mergesort\")\n        a, w = a[order], w[order]\n        cum_w = np.cumsum(w)\n        cum_y = np.cumsum(a * w)\n        return (cum_y / cum_y[-1] - cum_w / cum_w[-1]).sum()\n\n    den = gini(y_true, y_true, weights)\n    g = gini(y_true, y_pred[order], weights) / (den if abs(den) > 1e-12 else 1e-12)\n\n    return 0.5 * (g + top_four)\n\ndef lgb_amex_metric(preds, train_data):\n    y = train_data.get_label()\n    return (\"amex\", amex_metric(y, preds), True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:20:52.831413Z","iopub.execute_input":"2025-11-13T14:20:52.831679Z","iopub.status.idle":"2025-11-13T14:20:52.838910Z","shell.execute_reply.started":"2025-11-13T14:20:52.831660Z","shell.execute_reply":"2025-11-13T14:20:52.838087Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Engineering — Last Statement per Customer\n\n**Why last-statement?**  \nThe dataset has multiple monthly statements per customer. A strong, fast baseline is to keep only\nthe most recent row per `customer_ID` (by `S_2`) and hash-encode any string categoricals to small integers.\n\n**Pipeline (two passes, chunked):**\n1. **Pass 1:** scan only `customer_ID,S_2` → get max date per customer.  \n2. **Pass 2:** keep rows where `S_2 == max_date(customer_ID)`, break ties by last occurrence.  \n3. Hash-encode categoricals → compact `int32`.  \n4. Output: `train_df` (with `target`) and `test_agg` (same features).\n\nThis avoids GPU/CPU OOM while staying close to leaderboard-solid baselines.\n","metadata":{}},{"cell_type":"code","source":"# Cell 2 — Two-pass, chunked “last row per customer” build (memory-safe)\nimport os, gc, numpy as np, pandas as pd\n\nDATA_DIR = \"/kaggle/input/amex-default-prediction\"\nWORK_DIR = \"/kaggle/working\"\nos.makedirs(WORK_DIR, exist_ok=True)\n\n# ---------- helpers ----------\ndef _hash_series_to_int32_pandas(s: pd.Series, modulo: int = 2048) -> pd.Series:\n    s = s.astype(\"string\").fillna(\"__NA__\")\n    h = pd.util.hash_pandas_object(s, index=False).astype(\"int64\")\n    h = (h & 0x7FFFFFFF) % modulo\n    return h.astype(\"int32\")\n\ndef _pass1_max_date(csv_path: str, chunksize: int = 2_000_000):\n    \"\"\"\n    Pass 1: read only customer_ID,S_2 to find the max S_2 per customer.\n    Returns a dict: cid -> max_date (pd.Timestamp)\n    \"\"\"\n    cid_to_max = {}\n    for chunk in pd.read_csv(csv_path, usecols=[\"customer_ID\",\"S_2\"], chunksize=chunksize):\n        chunk[\"S_2\"] = pd.to_datetime(chunk[\"S_2\"])\n        # reduce within chunk\n        grp = chunk.groupby(\"customer_ID\", sort=False)[\"S_2\"].max()\n        for cid, dt in grp.items():\n            if (cid not in cid_to_max) or (dt > cid_to_max[cid]):\n                cid_to_max[cid] = dt\n        del chunk, grp\n        gc.collect()\n    return cid_to_max\n\ndef _pass2_collect_last_rows(csv_path: str, cid_to_max: dict, chunksize: int = 1_000_000):\n    \"\"\"\n    Pass 2: scan full CSV; keep rows where S_2 == cid_to_max[cid].\n    If multiple rows per (cid, S_2_max), keep the LAST (tail(1)) in that chunk.\n    Returns a pandas DataFrame with one row per customer (may still need final groupby).\n    \"\"\"\n    # probe full schema to keep all feature columns\n    sample = pd.read_csv(csv_path, nrows=2000)\n    cols = list(sample.columns)\n\n    kept_parts = []\n    for chunk in pd.read_csv(csv_path, usecols=cols, chunksize=chunksize):\n        chunk[\"S_2\"] = pd.to_datetime(chunk[\"S_2\"])\n        # filter to rows that match that customer's max date\n        # map lookup (vectorized): create a Series of max dates aligned to chunk cids\n        max_dates = chunk[\"customer_ID\"].map(cid_to_max)\n        mask = (max_dates.notna()) & (chunk[\"S_2\"].values == max_dates.values)\n        if not mask.any():\n            del chunk, max_dates\n            gc.collect()\n            continue\n        sub = chunk.loc[mask].copy()\n        # within this sub-chunk, keep last row per customer_ID\n        sub = sub.sort_values([\"customer_ID\", \"S_2\"]).groupby(\"customer_ID\", sort=False).tail(1)\n        kept_parts.append(sub[cols])  # keep all columns\n        del chunk, max_dates, sub\n        gc.collect()\n\n    if not kept_parts:\n        return pd.DataFrame(columns=cols)\n\n    df_last = pd.concat(kept_parts, axis=0, ignore_index=True)\n    # In case same customer got split across chunks with same max date, finalize with tail(1)\n    df_last = df_last.sort_values([\"customer_ID\", \"S_2\"]).groupby(\"customer_ID\", sort=False).tail(1).reset_index(drop=True)\n    return df_last\n\ndef _finalize_features_last(df_last: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"\n    Hash any non-numeric columns (except customer_ID, S_2), drop S_2.\n    \"\"\"\n    for c in df_last.columns:\n        if c in (\"customer_ID\", \"S_2\"):\n            continue\n        if not np.issubdtype(df_last[c].dtype, np.number):\n            df_last[c] = _hash_series_to_int32_pandas(df_last[c], modulo=2048)\n    return df_last.drop(columns=[\"S_2\"])\n\ndef build_last_table(csv_path: str, label_df: pd.DataFrame | None = None, role: str = \"train\"):\n    \"\"\"\n    role='train' → returns train_df (merged with labels)\n    role='test'  → returns test_agg (customer_ID + features)\n    \"\"\"\n    print(f\"[{role}] pass 1: scanning max dates ...\")\n    cid_to_max = _pass1_max_date(csv_path)\n    print(f\"[{role}] unique customers found:\", len(cid_to_max))\n\n    print(f\"[{role}] pass 2: collecting last rows ...\")\n    last_rows = _pass2_collect_last_rows(csv_path, cid_to_max)\n    print(f\"[{role}] last_rows shape:\", last_rows.shape)\n\n    print(f\"[{role}] hashing categoricals & finalizing ...\")\n    last_feats = _finalize_features_last(last_rows)\n    print(f\"[{role}] finalized feature table:\", last_feats.shape)\n\n    if role == \"train\":\n        assert label_df is not None, \"label_df must be provided for role='train'\"\n        out = label_df.merge(last_feats, on=\"customer_ID\", how=\"left\")\n        return out\n    else:\n        return last_feats\n\n# ---------- TRAIN ----------\nlabels = pd.read_csv(f\"{DATA_DIR}/train_labels.csv\")  # customer_ID, target\ntrain_df = build_last_table(f\"{DATA_DIR}/train_data.csv\", label_df=labels, role=\"train\")\n\n# ---------- TEST ----------\ntest_last = build_last_table(f\"{DATA_DIR}/test_data.csv\", label_df=None, role=\"test\")\n\n# Align columns to training features\nfeature_cols = [c for c in train_df.columns if c not in (\"customer_ID\",\"target\")]\nfor col in feature_cols:\n    if col not in test_last.columns:\n        test_last[col] = np.nan\ntest_agg = test_last[[\"customer_ID\"] + feature_cols]\n\nprint(\"Final shapes → train_df:\", train_df.shape, \" | test_agg:\", test_agg.shape)\ndisplay(train_df.head())\ndisplay(test_agg.head())\n\n# Medians for imputation in Cells 3 & 4\nTRAIN_FEATURES_ONLY = train_df[feature_cols].replace([np.inf, -np.inf], np.nan).astype(np.float32)\ntrain_medians = TRAIN_FEATURES_ONLY.median(numeric_only=True)\nprint(f\"Medians computed for {len(train_medians)} features.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:21:00.735681Z","iopub.execute_input":"2025-11-13T14:21:00.736417Z","iopub.status.idle":"2025-11-13T14:48:03.718724Z","shell.execute_reply.started":"2025-11-13T14:21:00.736392Z","shell.execute_reply":"2025-11-13T14:48:03.717978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"train_df: {train_df.shape}  |  test_agg: {test_agg.shape}\")\ndisplay(train_df.head(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:48:55.776530Z","iopub.execute_input":"2025-11-13T14:48:55.777260Z","iopub.status.idle":"2025-11-13T14:48:55.828890Z","shell.execute_reply.started":"2025-11-13T14:48:55.777235Z","shell.execute_reply":"2025-11-13T14:48:55.828153Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training — LightGBM (GPU) + AMEX Metric\n\n**Setup**\n- Loss: `binary_logloss`\n- Metric shown: AMEX (custom callback)\n- Early stopping via callbacks (patience = 200)\n- Regularization & speed: `num_leaves=128`, `min_data_in_leaf=100`, `feature_fraction=0.4`, `bagging_fraction=0.8`, `max_bin=255`\n- Device: **GPU**\n\nWe split 80/20 stratified, impute medians, and let early stopping find the best iteration.\n","metadata":{}},{"cell_type":"code","source":"# Cell 3 — Train LightGBM (GPU) with callbacks early stopping\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\n\nprint(\"LightGBM version:\", lgb.__version__)\n\nassert \"train_df\" in globals(), \"Expected a DataFrame named train_df from your preprocessing step.\"\n\nFEATURES = [c for c in train_df.columns if c not in (\"customer_ID\", \"target\")]\nTARGET = \"target\"\n\nX = train_df[FEATURES].replace([np.inf, -np.inf], np.nan).astype(np.float32)\n# Use medians computed in Cell 2 if available; otherwise compute now\nif \"train_medians\" not in globals():\n    train_medians = X.median(numeric_only=True)\nX = X.fillna(train_medians)\ny = train_df[TARGET].astype(np.int8)\n\nX_train, X_valid, y_train, y_valid = train_test_split(\n    X, y, test_size=0.20, random_state=42, stratify=y\n)\n\nprint(f\"Train: {X_train.shape}, Valid: {X_valid.shape}, Features: {len(FEATURES)}\")\n\ntrain_set = lgb.Dataset(X_train, label=y_train, free_raw_data=False)\nvalid_set = lgb.Dataset(X_valid, label=y_valid, reference=train_set, free_raw_data=False)\n\nparams = {\n    \"objective\": \"binary\",\n    \"metric\": \"binary_logloss\",     # AMEX reported via feval\n    \"boosting_type\": \"gbdt\",\n    \"learning_rate\": 0.05,\n    \"num_leaves\": 128,\n    \"min_data_in_leaf\": 100,\n    \"feature_fraction\": 0.4,\n    \"bagging_fraction\": 0.8,\n    \"bagging_freq\": 1,\n    \"max_depth\": -1,\n    \"lambda_l1\": 1.0,\n    \"lambda_l2\": 2.0,\n    \"max_bin\": 255,\n    \"bin_construct_sample_cnt\": 200_000,\n    \"device\": \"gpu\",                # GPU\n    \"gpu_platform_id\": 0,\n    \"gpu_device_id\": 0,\n    # If you hit OOM on GPU, try:\n    # \"force_col_wise\": True,\n}\n\n# Use callbacks for early stopping & logging (compatible across LGBM versions)\ncallbacks = [\n    lgb.early_stopping(stopping_rounds=200, verbose=True),\n    lgb.log_evaluation(period=200),\n]\n\nmodel = lgb.train(\n    params=params,\n    train_set=train_set,\n    num_boost_round=5000,\n    valid_sets=[train_set, valid_set],\n    valid_names=[\"train\", \"valid\"],\n    feval=lgb_amex_metric,          # from Cell 2\n    callbacks=callbacks\n)\n\n# Validation AMEX\nvalid_pred = model.predict(X_valid, num_iteration=model.best_iteration or model.current_iteration())\nprint(\"AMEX (valid):\", amex_metric(y_valid.values.astype(int), valid_pred))\nprint(\"Best iteration:\", model.best_iteration or model.current_iteration())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:48:59.596753Z","iopub.execute_input":"2025-11-13T14:48:59.597418Z","iopub.status.idle":"2025-11-13T14:50:47.201511Z","shell.execute_reply.started":"2025-11-13T14:48:59.597394Z","shell.execute_reply":"2025-11-13T14:50:47.200805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Best iteration: {getattr(model, 'best_iteration', None) or model.current_iteration()}\")\n# Optional: quick top-20 feature importances\nimport pandas as pd\nimp = pd.DataFrame({\n    \"feature\": FEATURES,\n    \"gain\": model.feature_importance(importance_type=\"gain\")\n}).sort_values(\"gain\", ascending=False).head(20)\ndisplay(imp)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:50:57.168443Z","iopub.execute_input":"2025-11-13T14:50:57.168984Z","iopub.status.idle":"2025-11-13T14:50:57.180494Z","shell.execute_reply.started":"2025-11-13T14:50:57.168964Z","shell.execute_reply":"2025-11-13T14:50:57.179443Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Inference & Submission\n\nWe align predictions to `sample_submission.csv` order by `customer_ID` to avoid any ID/order mismatch,\nthen write the final CSV to `/kaggle/working/`.\n","metadata":{}},{"cell_type":"code","source":"# Cell 4 — Inference & submission (GPU-trained model)\n\nimport numpy as np\nimport pandas as pd\n\n# Sanity checks\nassert \"test_agg\" in globals(), \"Expected test_agg from Cell 2.\"\nassert \"FEATURES\" in globals(), \"Run Cell 3 first to define FEATURES.\"\nassert \"model\" in globals(), \"Run Cell 3 to train the model.\"\nassert \"train_medians\" in globals(), \"Expected train_medians from Cell 2/3.\"\n\n# Align test columns to training FEATURES\nfor col in FEATURES:\n    if col not in test_agg.columns:\n        test_agg[col] = np.nan\n\nX_test = test_agg[FEATURES].replace([np.inf, -np.inf], np.nan).astype(np.float32)\nX_test = X_test.fillna(train_medians)\n\n# Use best iteration if available, else current\nbest_it = getattr(model, \"best_iteration\", None) or getattr(model, \"current_iteration\", lambda: None)()\ntest_pred = model.predict(X_test, num_iteration=best_it)\n\n# Build and save submission\nDATA_DIR = \"/kaggle/input/amex-default-prediction\"\nsub = pd.read_csv(f\"{DATA_DIR}/sample_submission.csv\", usecols=[\"customer_ID\"])\nsub[\"prediction\"] = test_pred\n\nout_path = \"/kaggle/working/submission.csv\"\nsub.to_csv(out_path, index=False)\nprint(\"Saved:\", out_path)\n\n# Quick peek\ndisplay(sub.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:54:51.304854Z","iopub.execute_input":"2025-11-13T14:54:51.305589Z","iopub.status.idle":"2025-11-13T14:55:19.814063Z","shell.execute_reply.started":"2025-11-13T14:54:51.305563Z","shell.execute_reply":"2025-11-13T14:55:19.813381Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Submission created ✅**  \nUpload `/kaggle/working/submission_lgbm_gpu.csv` to Kaggle to score on the leaderboard.\n","metadata":{}},{"cell_type":"markdown","source":"## 2. Random Forest (Model)","metadata":{}},{"cell_type":"markdown","source":"## Processing Train Data ##\n- Split Train data to 20 parts\n- Split Test data to 20 parts\n- Prevent memory error during processing\n- Use GPU","metadata":{}},{"cell_type":"code","source":"# # CALCULATE SIZE OF EACH SEPARATE FILE\n# def get_rows(customers, train, NUM_FILES = 20, verbose = ''):\n#     chunk = len(customers)//NUM_FILES\n#     if verbose != '':\n#         print(f'We will split {verbose} data into {NUM_FILES} separate files.')\n#         print(f'There will be {chunk} customers in each file (except the last file).')\n#         print('Below are number of rows in each file:')\n#     rows = []\n\n#     for k in range(NUM_FILES):\n#         if k==NUM_FILES-1: cc = customers[k*chunk:]\n#         else: cc = customers[k*chunk:(k+1)*chunk]\n#         s = train.loc[train.customer_ID.isin(cc)].shape[0]\n#         rows.append(s)\n#     if verbose != '': print( rows )\n#     return rows\n\n# if PROCESS_DATA:\n#     NUM_FILES = 20\n#     rows = get_rows(customers, train, NUM_FILES = NUM_FILES, verbose = 'train')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.562510Z","iopub.status.idle":"2025-11-13T14:12:38.562784Z","shell.execute_reply.started":"2025-11-13T14:12:38.562624Z","shell.execute_reply":"2025-11-13T14:12:38.562635Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1. customer_ID\n- Original: 64 bytes (string of length 64)\n- Reduced: 8 bytes (int64)\n\n3. S_2 (Date with time)\n- Original: 10 bytes (string)\n- Reduced: 3 bytes (datetime object)\n\n4. 11 Categorical Columns\n- Original: 8 bytes (int64)\n- Reduced: 1 bytes (int8)\n- Columns: ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n\n\n5. 177 Numeric Columns\n- Original: 1416 bytes (float64)\n- Reduced: 708 bytes (float32)\n\n6. Specific column (B_31): Convert to int8 (1 byte) since it has only 2 values.","metadata":{}},{"cell_type":"code","source":"# def feature_engineer(train, PAD_CUSTOMER_TO_13_ROWS = True, targets = None):\n        \n#     # REDUCE STRING COLUMNS \n#     # from 64 bytes to 8 bytes, and 10 bytes to 3 bytes respectively\n#     train['customer_ID'] = train['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n#     train.S_2 = cudf.to_datetime( train.S_2 )\n#     train['year'] = (train.S_2.dt.year-2000).astype('int8')\n#     train['month'] = (train.S_2.dt.month).astype('int8')\n#     train['day'] = (train.S_2.dt.day).astype('int8')\n#     del train['S_2']\n        \n#     # LABEL ENCODE CAT COLUMNS (and reduce to 1 byte)\n#     # with 0: padding, 1: nan, 2,3,4,etc: values\n#     d_63_map = {'CL':2, 'CO':3, 'CR':4, 'XL':5, 'XM':6, 'XZ':7}\n#     train['D_63'] = train.D_63.map(d_63_map).fillna(1).astype('int8')\n\n#     d_64_map = {'-1':2,'O':3, 'R':4, 'U':5}\n#     train['D_64'] = train.D_64.map(d_64_map).fillna(1).astype('int8')\n    \n#     CATS = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_66', 'D_68']\n#     OFFSETS = [2,1,2,2,3,2,3,2,2] #2 minus minimal value in full train csv\n#     # then 0 will be padding, 1 will be NAN, 2,3,4,etc will be values\n#     for c,s in zip(CATS,OFFSETS):\n#         train[c] = train[c] + s\n#         train[c] = train[c].fillna(1).astype('int8')\n#     CATS += ['D_63','D_64']\n    \n#     # ADD NEW FEATURES HERE\n#     # EXAMPLE: train['feature_189'] = etc etc etc\n#     # EXAMPLE: train['feature_190'] = etc etc etc\n#     # IF CATEGORICAL, THEN ADD TO CATS WITH: CATS += ['feaure_190'] etc etc etc\n    \n#     # REDUCE MEMORY DTYPE\n#     SKIP = ['customer_ID','year','month','day']\n#     for c in train.columns:\n#         if c in SKIP: continue\n#         if str( train[c].dtype )=='int64':\n#             train[c] = train[c].astype('int32')\n#         if str( train[c].dtype )=='float64':\n#             train[c] = train[c].astype('float32')\n            \n#     # PAD ROWS SO EACH CUSTOMER HAS 13 ROWS\n#     if PAD_CUSTOMER_TO_13_ROWS:\n#         tmp = train[['customer_ID']].groupby('customer_ID').customer_ID.agg('count')\n#         more = cupy.array([],dtype='int64') \n#         for j in range(1,13):\n#             i = tmp.loc[tmp==j].index.values\n#             more = cupy.concatenate([more,cupy.repeat(i,13-j)])\n#         df = train.iloc[:len(more)].copy().fillna(0)\n#         df = df * 0 - 1 #pad numerical columns with -1\n#         df[CATS] = (df[CATS] * 0).astype('int8') #pad categorical columns with 0\n#         df['customer_ID'] = more\n#         train = cudf.concat([train,df],axis=0,ignore_index=True)\n        \n#     # ADD TARGETS (and reduce to 1 byte)\n#     if targets is not None:\n#         train = train.merge(targets,on='customer_ID',how='left')\n#         train.target = train.target.astype('int8')\n        \n#     # FILL NAN\n#     train = train.fillna(-0.5) #this applies to numerical columns\n    \n#     # SORT BY CUSTOMER THEN DATE\n#     train = train.sort_values(['customer_ID','year','month','day']).reset_index(drop=True)\n#     train = train.drop(['year','month','day'],axis=1)\n    \n#     # REARRANGE COLUMNS WITH 11 CATS FIRST\n#     COLS = list(train.columns[1:])\n#     COLS = ['customer_ID'] + CATS + [c for c in COLS if c not in CATS]\n#     train = train[COLS]\n    \n#     return train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.564004Z","iopub.status.idle":"2025-11-13T14:12:38.564266Z","shell.execute_reply.started":"2025-11-13T14:12:38.564128Z","shell.execute_reply":"2025-11-13T14:12:38.564140Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training — Random Forest (GPU/CPU)\n\nWe train a Random Forest as a strong non-boosting baseline.\n- **GPU option (cuML RandomForestClassifier)** — fast on Kaggle GPU for larger tabular sets.\n- **CPU option (sklearn RandomForestClassifier)** — portable and simple.\n\nFor evaluation we use the **AMEX metric** on the same validation split as LightGBM.\n","metadata":{}},{"cell_type":"code","source":"PROCESS_DATA = True\nUSE_GPU = True             # True: cuML RF on GPU, False: sklearn RF on CPU\nMAKE_FEATURES = 'agg'      # 'last', 'agg', or 'flat13'\nRANDOM_STATE = 26\nTEST_SIZE = 0.25","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.565137Z","iopub.status.idle":"2025-11-13T14:12:38.565383Z","shell.execute_reply.started":"2025-11-13T14:12:38.565259Z","shell.execute_reply":"2025-11-13T14:12:38.565270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_dataset_from_engineered(train_engineered, make=MAKE_FEATURES):\n    has_target = 'target' in train_engineered.columns\n    COLS = list(train_engineered.columns)\n    CATS = COLS[1:1+11]\n    ALL = COLS[1:]\n    NUMS = [c for c in ALL if c not in CATS + (['target'] if has_target else [])]\n\n    g = train_engineered.groupby('customer_ID')\n\n    if make == 'last':\n        df = g.tail(1)\n        y = df['target'].values if has_target else None\n        X = df.drop(columns=['target']) if has_target else df\n        return X.set_index('customer_ID'), y, list(X.columns)\n\n    elif make == 'agg':\n        num_agg = g[NUMS].agg(['last','mean','std','min','max'])\n        cat_last = g[CATS].agg('last')\n        X = cat_last.join(num_agg)\n        X.columns = [f\"{a}__{b}\" if isinstance(a, tuple) else a for a in X.columns]\n        y = g['target'].agg('last').values if has_target else None\n        return X, y, list(X.columns)\n\n    elif make == 'flat13':\n        df = train_engineered.copy()\n        if has_target:\n            df_target = df[['customer_ID','target']].groupby('customer_ID').tail(1).drop_duplicates().set_index('customer_ID')\n            df = df.drop(columns=['target'])\n        df['_row'] = (df.groupby('customer_ID').cumcount()).astype('int8')\n        piv = df.pivot(index='customer_ID', columns='_row')\n        piv.columns = [f'{c[0]}@t{int(c[1])}' for c in piv.columns]\n        X = piv\n        y = df_target.loc[X.index,'target'].values if has_target else None\n        return X, y, list(X.columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.566026Z","iopub.status.idle":"2025-11-13T14:12:38.566234Z","shell.execute_reply.started":"2025-11-13T14:12:38.566143Z","shell.execute_reply":"2025-11-13T14:12:38.566152Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport os, gc, sys, glob, warnings, json\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nimport joblib\n\n# ---------------- CONFIG ----------------\nINPUT_DIR  = \"/kaggle/input/amex-default-prediction\"\nWORK_DIR   = \"/kaggle/working\"\nTMP_DIR    = os.path.join(WORK_DIR, \"lastsnap-temp\")\nCACHE_DIR  = os.path.join(WORK_DIR, \"lastsnap-cache\")\nos.makedirs(TMP_DIR, exist_ok=True)\nos.makedirs(CACHE_DIR, exist_ok=True)\n\nTRAIN_CSV  = os.path.join(INPUT_DIR, \"train_data.csv\")\nTEST_CSV   = os.path.join(INPUT_DIR,  \"test_data.csv\")\nLABELS_CSV = os.path.join(INPUT_DIR, \"train_labels.csv\")\n\n# Cache files (reused across runs)\nTRAIN_LAST_PQT = os.path.join(CACHE_DIR, \"train_last_snapshot.parquet\")\nTEST_LAST_PQT  = os.path.join(CACHE_DIR, \"test_last_snapshot.parquet\")\n\n# Chunking + split\nCHUNKSIZE   = 1_000_000      # lower if you still see restarts: 700_000 or 500_000\nVALID_SIZE  = 0.25\nRANDOM_STATE = 26\n\n# RandomForest (kept modest to stay RAM-safe)\nRF_PARAMS = dict(\n    n_estimators=400,\n    max_depth=28,\n    max_features=\"sqrt\",\n    min_samples_split=4,\n    min_samples_leaf=1,\n    bootstrap=True,\n    n_jobs=-1,\n    class_weight=\"balanced\",\n    random_state=RANDOM_STATE,\n)\n\n# Categorical maps / encodes (same meaning as your GPU pipeline)\nCATS_BASE   = ['B_30','B_38','D_114','D_116','D_117','D_120','D_126','D_66','D_68']\nOFFSETS     = [2,      1,      2,      2,      3,      2,      3,      2,      2]\nD63_MAP     = {'CL':2, 'CO':3, 'CR':4, 'XL':5, 'XM':6, 'XZ':7}\nD64_MAP     = {'-1':2, 'O':3, 'R':4, 'U':5}\n\n# ---------------- Helpers ----------------\ndef _id_hex_to_int16(s):\n    return int(str(s)[-16:], 16)\n\ndef amex_metric(y_true, y_prob):\n    \"\"\"\n    Official AMEX metric implementation (vectorized, RAM-safe for our split sizes).\n    \"\"\"\n    y_true = np.asarray(y_true).astype(int)\n    y_pred = np.asarray(y_prob)\n\n    # normalize ranks and weights\n    w = np.where(y_true == 0, 20, 1)\n\n    # top 4% by prediction\n    order = np.argsort(-y_pred)\n    w_cum = np.cumsum(w[order]) / np.sum(w)\n    top4 = y_true[order][w_cum <= 0.04]\n    d = np.sum(top4) / np.sum(y_true)\n\n    # gini\n    def _gini(a, p, w):\n        order = np.argsort(-p)\n        a, w = a[order], w[order]\n        cum_w = np.cumsum(w)\n        cum_a = np.cumsum(a * w)\n        sum_a = np.sum(a * w)\n        g = np.sum(cum_a / sum_a * w) / np.sum(w) - (np.sum(cum_w / np.sum(w) * w) / np.sum(w))\n        return g\n\n    g = _gini(y_true, y_pred, w)\n    gmax = _gini(y_true, y_true, w)\n    return 0.5 * (g / gmax + d)\n\ndef feature_engineer_chunk(df, keep_time=True):\n    \"\"\"\n    Lean CPU version of your feature_engineer WITHOUT padding.\n    \"\"\"\n    df['customer_ID'] = df['customer_ID'].astype(str).str[-16:].apply(_id_hex_to_int16).astype('int64')\n\n    df['S_2'] = pd.to_datetime(df['S_2'])\n    df['year']  = (df['S_2'].dt.year - 2000).astype('int16')\n    df['month'] = df['S_2'].dt.month.astype('int8')\n    df['day']   = df['S_2'].dt.day.astype('int8')\n\n    if 'D_63' in df.columns:\n        df['D_63'] = df['D_63'].map(D63_MAP).fillna(1).astype('int16')\n    if 'D_64' in df.columns:\n        df['D_64'] = df['D_64'].map(D64_MAP).fillna(1).astype('int16')\n\n    for c, s in zip(CATS_BASE, OFFSETS):\n        if c in df.columns:\n            x = pd.to_numeric(df[c], errors='coerce')\n            x = (x + s).astype('float32')\n            df[c] = x.fillna(1).astype('int16')\n\n    skip = {'customer_ID','S_2','year','month','day','target'}\n    for c in df.columns:\n        if c in skip: continue\n        if pd.api.types.is_integer_dtype(df[c]):\n            df[c] = pd.to_numeric(df[c], downcast='integer')\n        elif pd.api.types.is_float_dtype(df[c]):\n            df[c] = pd.to_numeric(df[c], downcast='float')\n\n    num_cols = [c for c in df.columns if c not in ['customer_ID','S_2','year','month','day','target']]\n    df[num_cols] = df[num_cols].fillna(-0.5)\n\n    if not keep_time:\n        df = df.drop(columns=['S_2','year','month','day'])\n\n    return df\n\ndef write_chunk_last_rows(csv_path, prefix, chunksize=CHUNKSIZE):\n    # clean any previous temp parts for this prefix\n    for p in glob.glob(os.path.join(TMP_DIR, f\"{prefix}_last_part_*.parquet\")):\n        try: os.remove(p)\n        except: pass\n\n    i = 0\n    for chunk in pd.read_csv(csv_path, chunksize=chunksize, low_memory=False):\n        fe = feature_engineer_chunk(chunk, keep_time=True)\n        idx = fe.groupby('customer_ID')['S_2'].idxmax()\n        last_rows = fe.loc[idx].copy()\n        outp = os.path.join(TMP_DIR, f\"{prefix}_last_part_{i}.parquet\")\n        last_rows.to_parquet(outp, index=False)\n        print(f\"[{prefix}] wrote {os.path.basename(outp)}  (rows={len(last_rows):,})\")\n        i += 1\n        del chunk, fe, last_rows\n        gc.collect()\n    return i\n\ndef build_global_last(prefix):\n    parts = sorted(glob.glob(os.path.join(TMP_DIR, f\"{prefix}_last_part_*.parquet\")))\n    if not parts:\n        raise FileNotFoundError(f\"No parts found for prefix={prefix}\")\n\n    dfs = []\n    for p in parts:\n        dfp = pd.read_parquet(p)\n        dfs.append(dfp)\n    big = pd.concat(dfs, ignore_index=True)\n    del dfs; gc.collect()\n\n    idx = big.groupby('customer_ID')['S_2'].idxmax()\n    last_global = big.loc[idx].copy()\n\n    last_global = last_global.sort_values('customer_ID')\n    last_global = last_global.drop(columns=['S_2','year','month','day'])\n    last_global = last_global.set_index('customer_ID')\n\n    # optional: clean temp\n    for p in parts:\n        try: os.remove(p)\n        except: pass\n\n    return last_global\n\n# ---------------- Main ----------------\ndef main():\n    # TRAIN last snapshot cache\n    if os.path.exists(TRAIN_LAST_PQT):\n        print(\">> Loading cached TRAIN last snapshot…\")\n        X_train = pd.read_parquet(TRAIN_LAST_PQT).set_index('customer_ID')\n    else:\n        print(\">> Processing TRAIN chunks to LAST rows…\")\n        write_chunk_last_rows(TRAIN_CSV, prefix=\"train\")\n        print(\">> Building global TRAIN last snapshot…\")\n        X_train = build_global_last(prefix=\"train\")\n        X_train.reset_index().to_parquet(TRAIN_LAST_PQT, index=False)\n    print(\"   train per-customer shape:\", X_train.shape)\n\n    # Labels\n    labels = pd.read_csv(LABELS_CSV)\n    labels['customer_ID'] = labels['customer_ID'].astype(str).str[-16:].apply(_id_hex_to_int16).astype('int64')\n    labels = labels.set_index('customer_ID')\n    y = labels.reindex(X_train.index)['target'].astype('int8').values\n\n    # Split\n    X_tr, X_va, y_tr, y_va = train_test_split(\n        X_train, y, test_size=VALID_SIZE, random_state=RANDOM_STATE, stratify=y\n    )\n\n    # Train RF\n    model = RandomForestClassifier(**RF_PARAMS)\n    model.fit(X_tr, y_tr)\n    va_pred = model.predict_proba(X_va)[:, 1]\n    auc = roc_auc_score(y_va, va_pred)\n    amex = amex_metric(y_va, va_pred)\n    print(f\"[Model] Validation ROC AUC = {auc:.5f} | AMEX = {amex:.5f}\")\n\n    # Save model + columns\n    joblib.dump(model, os.path.join(WORK_DIR, \"rf_model.joblib\"))\n    X_cols = X_train.columns.tolist()\n    with open(os.path.join(WORK_DIR, \"rf_columns.json\"), \"w\") as f:\n        json.dump(X_cols, f)\n    print(\"Saved rf_model.joblib and rf_columns.json\")\n\n    # Save feature importances\n    fi = pd.DataFrame({\n        \"feature\": X_cols,\n        \"importance\": model.feature_importances_.astype(np.float32)\n    }).sort_values(\"importance\", ascending=False)\n    fi.to_csv(os.path.join(WORK_DIR, \"feature_importances.csv\"), index=False)\n    print(\"Saved feature_importances.csv (top 10):\")\n    print(fi.head(10))\n\n    # TEST last snapshot cache\n    if os.path.exists(TEST_LAST_PQT):\n        print(\"\\n>> Loading cached TEST last snapshot…\")\n        X_test = pd.read_parquet(TEST_LAST_PQT).set_index('customer_ID')\n    else:\n        print(\"\\n>> Processing TEST chunks to LAST rows…\")\n        write_chunk_last_rows(TEST_CSV, prefix=\"test\")\n        print(\">> Building global TEST last snapshot…\")\n        X_test = build_global_last(prefix=\"test\")\n        X_test.reset_index().to_parquet(TEST_LAST_PQT, index=False)\n    print(\"   test per-customer shape:\", X_test.shape)\n\n    # Align columns\n    missing = [c for c in X_cols if c not in X_test.columns]\n    if missing:\n        for c in missing:\n            X_test[c] = 1 if c in (CATS_BASE + ['D_63','D_64']) else 0.0\n    extra = [c for c in X_test.columns if c not in X_cols]\n    if extra:\n        X_test = X_test.drop(columns=extra)\n    X_test = X_test[X_cols]\n\n    # Predict test\n    y_pred = model.predict_proba(X_test)[:, 1]\n    # Convert numeric IDs back to 64-char hex strings (pad with zeros)\n    customer_hex = [format(int(x), '064x') for x in X_test.index.values]\n    \n    submission = pd.DataFrame({\n        \"customer_ID\": customer_hex,\n        \"prediction\": y_pred.astype(np.float32)\n    })\n\n    submission.to_csv(\"submission.csv\", index=False)\n    print(\"\\n✅ Saved submission.csv\")\n    print(\"Artifacts:\")\n    print(\" - rf_model.joblib\")\n    print(\" - rf_columns.json\")\n    print(\" - feature_importances.csv\")\n    print(\" - submission.csv\")\n    print(\" - Cached last snapshots in lastsnap-cache/\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.568605Z","iopub.status.idle":"2025-11-13T14:12:38.568976Z","shell.execute_reply.started":"2025-11-13T14:12:38.568835Z","shell.execute_reply":"2025-11-13T14:12:38.568845Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Results and Comparison\n\nWe summarize AMEX on the same validation split for a fair comparison.\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\nresults = []\n\n# LightGBM (if present)\nif 'valid_pred' in globals():\n    results.append({\n        \"Model\": \"LightGBM (GPU)\",\n        \"Valid AMEX\": float(amex_metric(y_valid.values.astype(int), valid_pred)),\n        \"Notes\": f\"best_iteration={getattr(model,'best_iteration', None) or model.current_iteration()}\"\n    })\n\n# Random Forest (if present)\nif 'valid_pred_rf' in globals():\n    results.append({\n        \"Model\": \"Random Forest\" + (\" (GPU/cuML)\" if 'rf_gpu' in globals() else \" (CPU/sklearn)\"),\n        \"Valid AMEX\": float(rf_valid_amex),\n        \"Notes\": \"\"\n    })\n\npd.DataFrame(results).sort_values(\"Valid AMEX\", ascending=False).style.format({\"Valid AMEX\": \"{:.6f}\"})\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.569709Z","iopub.status.idle":"2025-11-13T14:12:38.569907Z","shell.execute_reply.started":"2025-11-13T14:12:38.569811Z","shell.execute_reply":"2025-11-13T14:12:38.569821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nlabels = [r[\"Model\"] for r in results]\nscores = [r[\"Valid AMEX\"] for r in results]\n\nplt.figure(figsize=(6,3.5))\nplt.bar(labels, scores)\nplt.ylabel(\"Valid AMEX\")\nplt.title(\"Model Comparison\")\nplt.xticks(rotation=10)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T14:12:38.571554Z","iopub.status.idle":"2025-11-13T14:12:38.571911Z","shell.execute_reply.started":"2025-11-13T14:12:38.571773Z","shell.execute_reply":"2025-11-13T14:12:38.571784Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Conclusion ","metadata":{}},{"cell_type":"markdown","source":"**Submission choice:** We use **LightGBM (GPU)** for the final predictions as it achieved the best AMEX on validation.  \n(Both andom Forest and LightGBM results are kept for reference and ablations.)\n","metadata":{}}]}