{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":35332,"databundleVersionId":3723648,"sourceType":"competition"},{"sourceId":3739819,"sourceType":"datasetVersion","datasetId":2231132},{"sourceId":11516877,"sourceType":"datasetVersion","datasetId":7222569}],"dockerImageVersionId":31011,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview","metadata":{}},{"cell_type":"markdown","source":"This is the notebook for our SC4000 Group Project.","metadata":{}},{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"# Standard library\nimport gc\nimport os\n\n# Data handling\nimport cudf\nimport cupy\nimport numpy as np\nimport pandas as pd\n\n# Machine learning & preprocessing\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\n\n# Deep learning (TensorFlow / Keras)\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom tensorflow.keras.layers import BatchNormalization, Dense, Dropout, Input\nfrom tensorflow.keras.models import Sequential, load_model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-25T07:07:13.441162Z","iopub.execute_input":"2025-04-25T07:07:13.441377Z","iopub.status.idle":"2025-04-25T07:07:35.088971Z","shell.execute_reply.started":"2025-04-25T07:07:13.441354Z","shell.execute_reply":"2025-04-25T07:07:35.088380Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Reading Data and Feature Engineering","metadata":{}},{"cell_type":"code","source":"# Reading Data\ndef read_data(path):\n    df = pd.read_parquet(path)\n    df['customer_ID'] = df['customer_ID'].str[-16:].apply(lambda x: int(x, 16)).astype('int64')\n    df['S_2'] = pd.to_datetime(df['S_2'])\n    df = df.fillna(0)\n    print(f\"Data shape: {df.shape}\")\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-25T07:07:35.089726Z","iopub.execute_input":"2025-04-25T07:07:35.090304Z","iopub.status.idle":"2025-04-25T07:07:35.094824Z","shell.execute_reply.started":"2025-04-25T07:07:35.090277Z","shell.execute_reply":"2025-04-25T07:07:35.094095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_eng(df: pd.DataFrame) -> pd.DataFrame:\n    # ————————————————————————————\n    # 1) Prep\n    all_f = [c for c in df.columns if c not in ['customer_ID','S_2']]\n    cat   = [\n        \"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\n        \"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\"\n    ]\n    num   = [c for c in all_f if c not in cat]\n    \n    # Sort once to enable group.tail(k) without re-sorting\n    df.sort_values(['customer_ID','S_2'], inplace=True)\n    \n    # GroupBy object\n    g = df.groupby('customer_ID')\n    \n    # ————————————————————————————\n    # 2a) numeric aggregations\n    print('2a')\n    num_agg = g[num].agg(['mean','std','min','max','last'])\n    num_agg.columns = [f'{c}_{agg}' for c, agg in num_agg.columns]\n    num_agg = num_agg.astype('float32')\n    full_feats = num_agg\n    del num_agg; gc.collect()\n\n    # ————————————————————————————\n    # 2b) categorical aggregations\n    print('2b')\n    cat_agg = g[cat].agg(['count','last','nunique'])\n    cat_agg.columns = [f'{c}_{agg}' for c, agg in cat_agg.columns]\n    cat_agg = cat_agg.astype('float32')\n    full_feats = full_feats.join(cat_agg, how='left')\n    del cat_agg; gc.collect()\n\n    # ————————————————————————————\n    # 2c) first/last differences and percent change\n    print('2c')\n    first_vals = g[num].first().astype('float32')\n    last_vals  = g[num].last().astype('float32')\n    \n    diff = last_vals - first_vals\n    diff.columns = [f'diff_{c}' for c in diff.columns]\n    diff = diff.astype('float32')\n    full_feats = full_feats.join(diff, how='left')\n    del diff; gc.collect()\n\n    pctchg = (last_vals - first_vals).div(first_vals.replace(0, np.nan))\n    pctchg.columns = [f'pctchg_{c}' for c in pctchg.columns]\n    pctchg = pctchg.fillna(0).astype('float32')\n    full_feats = full_feats.join(pctchg, how='left')\n    del first_vals, last_vals, pctchg; gc.collect()\n\n    # ————————————————————————————\n    # 2d) time span & recency\n    print('2d')\n    max_time = g['S_2'].max()\n    min_time = g['S_2'].min()\n    span     = (max_time - min_time).dt.days.to_frame('days_span').astype('int16')\n    recency  = (pd.Timestamp('today') - max_time).dt.days.to_frame('days_since_last').astype('int16')\n    del max_time, min_time; gc.collect()\n\n    full_feats = full_feats.join(span, how='left')\n    del span; gc.collect()\n    full_feats = full_feats.join(recency, how='left')\n    del recency; gc.collect()\n    # 3) Memory-friendly Last-k windows by customer batches\n    ids = df['customer_ID'].unique()\n    batch_size = 50_000    # tune this to taste\n    count = 0\n\n    for batch_ids in np.array_split(ids, len(ids) // batch_size + 1):\n        print(f'batch {count}')\n        count += 1\n        sub = df[df['customer_ID'].isin(batch_ids)]\n        # since df is already sorted, cumcount(descending) gives “reverse rank”\n        rev_rank = sub.groupby('customer_ID').cumcount(ascending=False)\n\n        # ---- k=3 aggregates ----\n        sub3 = sub[rev_rank < 3]\n        # existing numeric & categorical aggs\n        num3 = (\n            sub3.groupby('customer_ID')[num]\n                .agg(['mean','std','min','max'])\n                .astype('float32')\n        )\n        num3.columns = [f'last3_{agg}_{c}' for c, agg in num3.columns]\n    \n        cat3 = (\n            sub3.groupby('customer_ID')[cat]\n                .agg(['count','nunique'])\n                .astype('float32')\n        )\n        cat3.columns = [f'last3_{agg}_{c}' for c, agg in cat3.columns]\n    \n        # —— new: first/last diff for k=3 —— \n        first3 = sub3.groupby('customer_ID')[num].first().astype('float32')\n        last3  = sub3.groupby('customer_ID')[num].last().astype('float32')\n        diff3  = (last3 - first3)\n        diff3.columns = [f'last3_diff_{c}' for c in diff3.columns]\n    \n        # combine all last-3 features\n        part3 = pd.concat([num3, cat3, diff3], axis=1)\n\n        # inject into full_feats\n        full_feats.loc[part3.index, part3.columns] = part3\n    \n        # cleanup for this batch\n        del sub3, num3, cat3, first3, last3, diff3, part3\n        gc.collect()\n    \n    \n        # # —— repeat the same for k=6 —— (gave up due to RAM limitation)\n    \n        # sub6 = sub[rev_rank < 6]\n        # num6 = (\n        #     sub6.groupby('customer_ID')[num]\n        #         .agg(['mean','std','min','max'])\n        #         .astype('float32')\n        # )\n        # num6.columns = [f'last6_{agg}_{c}' for c, agg in num6.columns]\n    \n        # cat6 = (\n        #     sub6.groupby('customer_ID')[cat]\n        #         .agg(['count','nunique'])\n        #         .astype('float32')\n        # )\n        # cat6.columns = [f'last6_{agg}_{c}' for c, agg in cat6.columns]\n    \n        # # —— first/last diff for k=6 —— \n        # first6 = sub6.groupby('customer_ID')[num].first().astype('float32')\n        # last6  = sub6.groupby('customer_ID')[num].last().astype('float32')\n        # diff6  = (last6 - first6)\n        # diff6.columns = [f'last6_diff_{c}' for c in diff6.columns]\n    \n        # part6 = pd.concat([num6, cat6, diff6], axis=1)\n\n        # full_feats.loc[part6.index, part6.columns] = part6\n    \n        # del sub6, num6, cat6, first6, last6, diff6, part6, sub, rev_rank\n        del sub, rev_rank\n        gc.collect()\n\n\n    # ————————————————————————————\n    # 4) Final clean and return\n    full_feats = full_feats.fillna(0)\n    print('Feature Engineering Done — shape', full_feats.shape)\n    return full_feats\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-25T07:07:35.096488Z","iopub.execute_input":"2025-04-25T07:07:35.096723Z","iopub.status.idle":"2025-04-25T07:07:35.120523Z","shell.execute_reply.started":"2025-04-25T07:07:35.096707Z","shell.execute_reply":"2025-04-25T07:07:35.119926Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Read in Train Data","metadata":{}},{"cell_type":"code","source":"train = read_data('../input/amex-data-integer-dtypes-parquet-format/train.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-25T07:07:35.121300Z","iopub.execute_input":"2025-04-25T07:07:35.121600Z","execution_failed":"2025-04-25T07:07:45.949Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineer the Train Data","metadata":{}},{"cell_type":"code","source":"train = feature_eng(train)\nfeatures = train.columns[1:-1]\ntrain.head()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.949Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Adding Label to the Train Data","metadata":{}},{"cell_type":"code","source":"labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\nlabels['customer_ID'] = labels['customer_ID'].str[-16:].apply(lambda x: int(x, 16)).astype('int64')\nlabels = labels.set_index('customer_ID')\ntrain = train.merge(labels, left_index=True, right_index=True, how='left')\ntrain.target = train.target.astype('int8')\ndel labels\ngc.collect()\n\ntrain = train.sort_index().reset_index()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.949Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Evaluation for Valid Set","metadata":{}},{"cell_type":"code","source":"# Evaluation for Valid\ndef amex_metric_mod(y_true, y_pred):\n\n    y_pred = np.squeeze(np.array(y_pred))\n    y_true = np.squeeze(np.array(y_true))\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 0.5 * (gini[1]/gini[0] + top_four)\n\ndef lgb_amex_eval(y_pred, dtrain):\n    y_true = dtrain.get_label()\n    score  = amex_metric_mod(y_true, y_pred)\n    # name must be unique among metrics\n    return 'amex_metric', score, True","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.949Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Build LightGB Model","metadata":{}},{"cell_type":"code","source":"# Transform the training data\nX = train.drop(columns=['customer_ID', 'target']).astype('float32').to_numpy()\ny = train['target'].astype('float32').to_numpy()\ndel train\ngc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Stratified K-Fold for training\nNUM_FOLDS = 5\nskf = StratifiedKFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# params = {\n#     # task and device\n#     'objective': 'binary',\n#     'metric': ['binary_logloss', 'auc'],\n#     'boosting': 'gbdt',\n#     'device': 'gpu',\n#     'seed': 42,\n#     'verbosity': -1,\n\n#     # learning\n#     'learning_rate': 0.03,\n#     'lambda_l1': 0.1,\n#     'lambda_l2': 30,\n\n#     # tree complexity\n#     'num_leaves': 64,\n#     'max_depth': -1,\n#     'min_data_in_leaf': 256,\n#     'min_data_in_bin': 256,\n#     'max_bin': 63,\n\n#     # sampling\n#     'feature_fraction': 0.10,\n#     'bagging_fraction': 0.75,\n#     'bagging_freq': 5,\n\n#     # disable boosting from global average\n#     'boost_from_average': False,\n# }\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Train for LightGBM\n\n# oof_preds = np.zeros(X.shape[0], dtype=np.float32)\n\n# for fold, (train_index, valid_index) in enumerate(skf.split(X, y)):\n#     print(f\"Starting LGB Fold #{fold}\")\n#     train_data = lgb.Dataset(X[train_index], label=y[train_index])\n#     valid_data = lgb.Dataset(X[valid_index], label=y[valid_index])\n    \n#     model = lgb.train(\n#         params,\n#         train_data,\n#         num_boost_round=5000,\n#         valid_sets=[valid_data],\n#         callbacks=[\n#             lgb.early_stopping(100),\n#             lgb.log_evaluation(50),\n#         ]\n#     )\n#     model.save_model(f'lgb_model_{fold}.txt', num_iteration=model.best_iteration)\n\n\n#     # predict on the validation fold\n#     y_preds = model.predict(\n#         X[valid_index],\n#         num_iteration=model.best_iteration\n#     )\n#     oof_preds[valid_index] = y_preds\n    \n#     acc = amex_metric_mod(y[valid_index], y_preds)\n#     print('Kaggle Metric =',acc,'\\n')\n\n#     # clean up\n#     del train_data, valid_data, y_preds, model\n#     gc.collect()\n\n# np.save('oof_preds_train.npy', oof_preds)\n# del oof_preds\n# gc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.950Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Concat the data with oof results","metadata":{}},{"cell_type":"code","source":"# oof_preds = np.load('oof_preds_train.npy').astype('float32')\n# X = np.hstack([X, oof_preds.reshape(-1, 1)])\n# del oof_preds","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.950Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Using this as new dataframe to train LGBM","metadata":{}},{"cell_type":"code","source":"# params = {\n#     # task and device\n#     'objective': 'binary',\n#     'metric': ['binary_logloss', 'auc'],\n#     'boosting': 'gbdt',\n#     'device': 'gpu',\n#     'seed': 42,\n#     'verbosity': -1,\n\n#     # learning\n#     'learning_rate': 0.03,\n#     'lambda_l1': 0.1,\n#     'lambda_l2': 30,\n    \n\n#     # tree complexity\n#     'num_leaves': 64,\n#     'max_depth': -1,\n#     'min_data_in_leaf': 256,\n#     'min_data_in_bin': 256,\n#     'max_bin': 63,\n\n#     # sampling\n#     'feature_fraction': 0.10,\n#     'bagging_fraction': 0.75,\n#     'bagging_freq': 5,\n\n#     # disable boosting from global average\n#     'boost_from_average': False,\n# }\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Train for LightGBM 2\n# for fold, (train_index, valid_index) in enumerate(skf.split(X, y)):\n#     print(f\"Starting LGB-2 Fold #{fold}\")\n#     train_data = lgb.Dataset(X[train_index], label=y[train_index])\n#     valid_data = lgb.Dataset(X[valid_index], label=y[valid_index])\n    \n#     model = lgb.train(\n#         params,\n#         train_data,\n#         num_boost_round=5000,\n#         valid_sets=[valid_data],\n#         callbacks=[\n#             lgb.early_stopping(100),\n#             lgb.log_evaluation(50),\n#         ]\n#     )\n#     model.save_model(f'lgb_model2_{fold}.txt', num_iteration=model.best_iteration)\n\n\n#     # predict on the validation fold\n#     y_preds = model.predict(\n#         X[valid_index],\n#         num_iteration=model.best_iteration\n#     )\n    \n#     acc = amex_metric_mod(y[valid_index], y_preds)\n#     print('Kaggle Metric =',acc,'\\n')\n\n#     # clean up\n#     del train_data, valid_data, y_preds, model\n#     gc.collect()\n\n# del params\n# gc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.950Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Neural Network Model","metadata":{}},{"cell_type":"code","source":"# Build NN Model\ndef build_model(input_dim):\n    model = Sequential([\n        Input(shape=(input_dim,)),\n        Dense(2*input_dim, activation='relu', kernel_initializer='he_normal'),\n        BatchNormalization(),\n        Dropout(0.3),\n        Dense(input_dim, activation='relu', kernel_initializer='he_normal'),\n        BatchNormalization(),\n        Dropout(0.3),\n        Dense(64, activation='relu', kernel_initializer='he_normal'),\n        BatchNormalization(),\n        Dropout(0.3),\n        Dense(1, activation='sigmoid')\n    ])\n    \n    # Use a lower learning rate to prevent exploding gradients\n    optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5)\n    model.compile(\n        optimizer=optimizer,\n        loss='binary_crossentropy',\n        metrics=['accuracy']\n    )\n    return model","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Transforming Test Data\nscaler = StandardScaler()\nX = X.astype('float32')   \nchunk = 20_000                           # ~20 k rows at a time\nfor i in range(0, X.shape[0], chunk):\n    print(f\"Chunk #{i//chunk}\")\n    j = min(i+chunk, X.shape[0])\n    X[i:j] = scaler.fit_transform(X[i:j])\n    gc.collect()\ngc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_dataset(X, y, batch_size=128, shuffle=True):\n    ds = tf.data.Dataset.from_tensor_slices((X, y))\n    if shuffle:\n        # shuffle buffer can be much smaller than full dataset\n        ds = ds.shuffle(buffer_size=10_000)  \n    return ds.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n\n\nfor fold, (train_index, valid_index) in enumerate(skf.split(X, y)):\n    print(f\"Starting NN Fold #{fold}\")\n    \n    K.clear_session()\n    X_train, X_valid = X[train_index], X[valid_index]\n    y_train, y_valid = y[train_index], y[valid_index]\n    \n    # Build the model using the number of features in X_train\n    model = build_model(X_train.shape[1])\n    \n    # Set up early stopping to avoid overfitting\n    early_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n\n    ds_train = make_dataset(X_train, y_train, batch_size=128, shuffle=True)\n    ds_val   = make_dataset(X_valid, y_valid, batch_size=128, shuffle=False)\n    \n    # Train the model\n    history = model.fit(\n        ds_train,\n        validation_data=ds_val,\n        epochs=64,\n        batch_size=128,\n        callbacks=[early_stop],\n        verbose=1\n    )\n    model.save(f'NN_model_{fold}.h5')\n\n    preds = model.predict(X_valid)\n    preds = preds.flatten()  \n    \n    acc = amex_metric_mod(y_valid, preds)\n    print('Kaggle Metric =',acc,'\\n')\n\n    del X_train, X_valid\n    del y_train, y_valid\n    del model\n    gc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Delete training data to free RAM\ndel X, y\ngc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Predict the testing data","metadata":{}},{"cell_type":"markdown","source":"## Clean everything up to start predict","metadata":{}},{"cell_type":"code","source":"# for name in list(globals().keys()):\n#     if name.startswith('_'):\n#         continue\n#     del globals()[name]","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Reload the function needed","metadata":{}},{"cell_type":"code","source":"# # Standard library\n# import gc\n\n# import numpy as np\n# import pandas as pd\n\n# # Machine learning & preprocessing\n# import lightgbm as lgb\n# from sklearn.preprocessing import StandardScaler\n\n# # Deep learning (TensorFlow / Keras)\n# import tensorflow as tf\n# from tensorflow.keras.models import load_model\n\n# gc.collect()\n\n# # Reading Data\n# def read_data(path):\n#     df = pd.read_parquet(path)\n#     df['customer_ID'] = df['customer_ID'].str[-16:].apply(lambda x: int(x, 16)).astype('int64')\n#     df['S_2'] = pd.to_datetime(df['S_2'])\n#     df = df.fillna(0)\n#     print(f\"Data shape: {df.shape}\")\n#     return df\n\n# def feature_eng(df: pd.DataFrame) -> pd.DataFrame:\n#     # ————————————————————————————\n#     # 1) Prep\n#     all_f = [c for c in df.columns if c not in ['customer_ID','S_2']]\n#     cat   = [\n#         \"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\n#         \"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\"\n#     ]\n#     num   = [c for c in all_f if c not in cat]\n    \n#     # Sort once to enable group.tail(k) without re-sorting\n#     df.sort_values(['customer_ID','S_2'], inplace=True)\n    \n#     # GroupBy object\n#     g = df.groupby('customer_ID')\n    \n#     # ————————————————————————————\n#     # 2a) numeric aggregations\n#     print('2a')\n#     num_agg = g[num].agg(['mean','std','min','max','last'])\n#     num_agg.columns = [f'{c}_{agg}' for c, agg in num_agg.columns]\n#     num_agg = num_agg.astype('float32')\n#     full_feats = num_agg\n#     del num_agg; gc.collect()\n\n#     # ————————————————————————————\n#     # 2b) categorical aggregations\n#     print('2b')\n#     cat_agg = g[cat].agg(['count','last','nunique'])\n#     cat_agg.columns = [f'{c}_{agg}' for c, agg in cat_agg.columns]\n#     cat_agg = cat_agg.astype('float32')\n#     full_feats = full_feats.join(cat_agg, how='left')\n#     del cat_agg; gc.collect()\n\n#     # ————————————————————————————\n#     # 2c) first/last differences and percent change\n#     print('2c')\n#     first_vals = g[num].first().astype('float32')\n#     last_vals  = g[num].last().astype('float32')\n    \n#     diff = last_vals - first_vals\n#     diff.columns = [f'diff_{c}' for c in diff.columns]\n#     diff = diff.astype('float32')\n#     full_feats = full_feats.join(diff, how='left')\n#     del diff; gc.collect()\n\n#     pctchg = (last_vals - first_vals).div(first_vals.replace(0, np.nan))\n#     pctchg.columns = [f'pctchg_{c}' for c in pctchg.columns]\n#     pctchg = pctchg.fillna(0).astype('float32')\n#     full_feats = full_feats.join(pctchg, how='left')\n#     del first_vals, last_vals, pctchg; gc.collect()\n\n#     # ————————————————————————————\n#     # 2d) time span & recency\n#     print('2d')\n#     max_time = g['S_2'].max()\n#     min_time = g['S_2'].min()\n#     span     = (max_time - min_time).dt.days.to_frame('days_span').astype('int16')\n#     recency  = (pd.Timestamp('today') - max_time).dt.days.to_frame('days_since_last').astype('int16')\n#     del max_time, min_time; gc.collect()\n\n#     full_feats = full_feats.join(span, how='left')\n#     del span; gc.collect()\n#     full_feats = full_feats.join(recency, how='left')\n#     del recency; gc.collect()\n#     # 3) Memory-friendly Last-k windows by customer batches\n#     ids = df['customer_ID'].unique()\n#     batch_size = 50_000    # tune this to taste\n\n#     for batch_ids in np.array_split(ids, len(ids) // batch_size + 1):\n#         sub = df[df['customer_ID'].isin(batch_ids)]\n#         # since df is already sorted, cumcount(descending) gives “reverse rank”\n#         rev_rank = sub.groupby('customer_ID').cumcount(ascending=False)\n    \n#         # pick last 3, last 6\n#         sub3 = sub[rev_rank < 3]\n    \n#         # ---- k=3 aggregates ----\n#         num3 = (\n#             sub3.groupby('customer_ID')[num]\n#                 .agg(['mean','std','min','max'])\n#                 .astype('float32')\n#         )\n#         num3.columns = [f'last3_{agg}_{c}' for c, agg in num3.columns]\n    \n#         cat3 = (\n#             sub3.groupby('customer_ID')[cat]\n#                 .agg(['count','nunique'])\n#                 .astype('float32')\n#         )\n#         cat3.columns = [f'last3_{agg}_{c}' for c, agg in cat3.columns]\n    \n#         part3 = pd.concat([num3, cat3], axis=1)\n    \n#         # inject into full_feats **in place**\n#         # this will create the columns if they don’t exist yet,\n#         # but only update rows in `batch_ids`\n#         full_feats.loc[part3.index, part3.columns] = part3\n    \n#         # cleanup for this batch\n#         del sub, rev_rank, sub3, num3, cat3, part3\n#         gc.collect()\n\n#     # ————————————————————————————\n#     # 4) Final clean and return\n#     full_feats = full_feats.fillna(0)\n#     print('Feature Engineering Done — shape', full_feats.shape)\n#     return full_feats","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Read in test data","metadata":{}},{"cell_type":"code","source":"# test = read_data('../input/amex-data-integer-dtypes-parquet-format/test.parquet')\n# test = feature_eng(test)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Preds with LGBM","metadata":{}},{"cell_type":"code","source":"# NUM_FOLDS = 5\n# # Initialize an array to accumulate predictions\n# SHAPE = test.shape\n# preds_sum = np.zeros(SHAPE[0], dtype=np.float32)\n\n# for fold in range(NUM_FOLDS):\n#     print(f\"Fold #{fold}\")\n#     model = lgb.Booster(model_file=f'lgb_model_{fold}.txt')\n    \n#     preds = model.predict(test, num_iteration=model.best_iteration)\n    \n#     preds_sum += preds\n#     del model, preds\n#     gc.collect()\n\n# preds_sum /= NUM_FOLDS\n# np.save('lgb_preds.npy', preds_sum)\n# test['oof_pred'] = preds_sum\n# del preds_sum\n# gc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Preds with LGB Model 2","metadata":{}},{"cell_type":"code","source":"# # Initialize an array to accumulate predictions\n# preds_sum = np.zeros(SHAPE[0], dtype=np.float32)\n\n# for fold in range(NUM_FOLDS):\n#     print(f\"Fold #{fold}\")\n#     model = lgb.Booster(model_file=f'lgb_model2_{fold}.txt')\n    \n#     preds = model.predict(test, num_iteration=model.best_iteration)\n    \n#     preds_sum += preds\n#     del model, preds\n#     # gc.collect()\n\n# preds_sum /= NUM_FOLDS\n# np.save('lgb_preds2.npy', preds_sum)\n# del preds_sum\n# gc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Preds with NN","metadata":{}},{"cell_type":"code","source":"# # Transforming Test Data\n# scaler = StandardScaler()\n# test = test.astype('float32')   \n# chunk = 20_000                           # ~20 k rows at a time\n# for i in range(0, test.shape[0], chunk):\n#     print(f\"Chunk #{i/chunk}\")\n#     j = min(i+chunk, test.shape[0])\n#     test[i:j] = scaler.fit_transform(test[i:j])\n#     gc.collect()\n# gc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# preds_sum = np.zeros(SHAPE[0], dtype=np.float32)\n\n# for fold in range(NUM_FOLDS):\n#     print(f\"Fold #{fold}\")\n#     model = load_model(f'NN_model_{fold}.h5')\n    \n#     for start in range(0, SHAPE[0], chunk):\n#         end = min(start + chunk, SHAPE[0])\n#         chunk_preds = model.predict(\n#             test[start:end],\n#             verbose=0\n#         ).ravel()\n        \n#         preds_sum[start:end] += chunk_preds\n#         del chunk_preds\n#         gc.collect()\n    \n#     del model\n#     gc.collect()\n\n# preds_sum /= NUM_FOLDS\n# np.save('nn_preds.npy', preds_sum)\n# del test, preds_sum\n# gc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensemble two predictions","metadata":{}},{"cell_type":"code","source":"# # Average the predictions across all folds (Soft Voting)\n# lgb_preds = np.load('lgb_preds.npy')\n# lgb_preds2 = np.load('lgb_preds2.npy')\n# nn_preds  = np.load('nn_preds.npy')\n\n# final_pred = (0.4 * nn_preds + 0.5 * lgb_preds + 0.1 * lgb_preds2)\n# del nn_preds\n# # gc.collect()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # WRITE SUBMISSION FILE\n# sub = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')[['customer_ID']]\n# sub['customer_ID_hash'] = sub['customer_ID'].str[-16:].apply(lambda x: int(x, 16)).astype('int64')\n# sub = sub.set_index('customer_ID_hash')\n# sub = sub.sort_index()\n# sub['prediction'] = final_pred  # test_preds should be a 1-D array or list of predictions\n\n# sub[['customer_ID', 'prediction']].to_csv('submission.csv', index=False)\n# sub = sub.reset_index(drop=True)\n\n# sub.to_csv(f'submission.csv', index=False)\n# print('Submission file shape is', sub.shape )\n# sub.head()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-04-25T07:07:45.955Z"}},"outputs":[],"execution_count":null}]}