{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"},{"sourceId":162605422,"sourceType":"kernelVersion"}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{}},{"cell_type":"markdown","source":"**Based on:**\n\nhttps://www.kaggle.com/code/greysky/home-credit-baseline\n\n**Related notebooks**\n\nUtility script notebook (with addtional functions, aggregators):\n\nhttps://www.kaggle.com/andreynesterov/home-credit-baseline-data\n\nInference notebook:\n\nhttps://www.kaggle.com/andreynesterov/home-credit-baseline-inference","metadata":{}},{"cell_type":"markdown","source":"# Dependencies","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport joblib\nimport lightgbm as lgb\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.preprocessing import LabelEncoder\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-12T11:45:21.222430Z","iopub.execute_input":"2024-02-12T11:45:21.222690Z","iopub.status.idle":"2024-02-12T11:45:27.892853Z","shell.execute_reply.started":"2024-02-12T11:45:21.222665Z","shell.execute_reply":"2024-02-12T11:45:27.892029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data collection","metadata":{}},{"cell_type":"markdown","source":"Import utility script notebook with data collection functions. See notebook for details: [Home Credit: baseline - DATA](https://www.kaggle.com/code/andreynesterov/home-credit-baseline-data)","metadata":{}},{"cell_type":"code","source":"import home_credit_baseline_data as data_nb","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:45:27.894478Z","iopub.execute_input":"2024-02-12T11:45:27.894774Z","iopub.status.idle":"2024-02-12T11:45:27.909230Z","shell.execute_reply.started":"2024-02-12T11:45:27.894749Z","shell.execute_reply":"2024-02-12T11:45:27.908539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# data_nb.Aggregator.group_aggregators = [pl.n_unique] #pl.max, pl.min, \n# data_nb.Aggregator.str_aggregators = [pl.max, pl.min, pl.first, pl.last, pl.n_unique]\ntrain_df = data_nb.prepare_df(data_nb.CFG.train_dir)\n# train_df = pd.read_parquet('/kaggle/usr/lib/home_credit_baseline_data/train_full.parquet')\ncat_cols = list(train_df.select_dtypes(\"category\").columns)\ndisplay(train_df)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-12T11:45:27.914737Z","iopub.execute_input":"2024-02-12T11:45:27.915041Z","iopub.status.idle":"2024-02-12T11:47:48.132582Z","shell.execute_reply.started":"2024-02-12T11:45:27.915015Z","shell.execute_reply":"2024-02-12T11:47:48.131696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(cat_cols)","metadata":{"scrolled":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-12T11:47:48.133630Z","iopub.execute_input":"2024-02-12T11:47:48.133908Z","iopub.status.idle":"2024-02-12T11:47:48.142170Z","shell.execute_reply.started":"2024-02-12T11:47:48.133884Z","shell.execute_reply":"2024-02-12T11:47:48.141299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = data_nb.prepare_df(data_nb.CFG.test_dir, cat_cols=cat_cols, mode=\"test\", train_cols=train_df.columns)\ndisplay(test_df)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-12T11:47:48.143231Z","iopub.execute_input":"2024-02-12T11:47:48.143558Z","iopub.status.idle":"2024-02-12T11:47:49.147249Z","shell.execute_reply.started":"2024-02-12T11:47:48.143531Z","shell.execute_reply":"2024-02-12T11:47:49.146364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Drop columns","metadata":{}},{"cell_type":"code","source":"drop_cols = []\n# drop_cols_startwith = [\"std_\"]\n# for name_prefix in drop_cols_startwith:\n#     cols_names = train_df.columns[train_df.columns.str.startswith(name_prefix)]\n#     drop_cols += cols_names.to_list()\n# display(drop_cols)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-02-12T11:47:49.148446Z","iopub.execute_input":"2024-02-12T11:47:49.148723Z","iopub.status.idle":"2024-02-12T11:47:49.152747Z","shell.execute_reply.started":"2024-02-12T11:47:49.148699Z","shell.execute_reply":"2024-02-12T11:47:49.151895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"### from https://www.kaggle.com/code/batprem/home-credit-risk-mode-utility-scripts\n\ndef gini_stability(base, score_col=\"score\", w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", score_col]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", score_col]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[score_col])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:47:49.153995Z","iopub.execute_input":"2024-02-12T11:47:49.154250Z","iopub.status.idle":"2024-02-12T11:47:49.164038Z","shell.execute_reply.started":"2024-02-12T11:47:49.154228Z","shell.execute_reply":"2024-02-12T11:47:49.163114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nX = train_df.drop(columns=[\"target\", \"case_id\", \"WEEK_NUM\"] + drop_cols)\nprint(\"X shape: \", X.shape)\ny = train_df[\"target\"]\nweeks = train_df[\"WEEK_NUM\"]\n\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 8,\n    \"max_bin\": 255,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 1000,\n    \"colsample_bytree\": 0.8, \n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    \"device\": \"gpu\",\n    \"gpu_platform_id\": 0,\n    \"gpu_device_id\": 0,\n    \"num_thread\": 2,\n}\n\nfitted_models = []\noof_pred = np.zeros(X.shape[0])\n\nfor idx_train, idx_valid in cv.split(X, y, groups=weeks):\n    X_train, y_train = X.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X.iloc[idx_valid], y.iloc[idx_valid]\n\n    model = lgb.LGBMClassifier(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        callbacks=[lgb.log_evaluation(100), lgb.early_stopping(100)]\n    )\n    fitted_models.append(model)\n    val_pred = model.predict_proba(X_valid)[:, 1]\n    oof_pred[idx_valid] = val_pred\n    gc.collect()","metadata":{"_kg_hide-output":true,"scrolled":true,"execution":{"iopub.status.busy":"2024-02-12T11:47:49.165014Z","iopub.execute_input":"2024-02-12T11:47:49.165267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc_oof = roc_auc_score(y, oof_pred)\nprint(\"CV roc_auc_oof: \", roc_auc_oof)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df = train_df[[\"WEEK_NUM\", \"target\"]].copy()\noof_df[\"pred_oof\"] = oof_pred\ngini_score = gini_stability(oof_df, score_col=\"pred_oof\")\nprint(\"gini_score:\\t\", gini_score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_models_dict = [(str(i), model) for i, model in enumerate(fitted_models)]\n\nmodel = VotingClassifier(\n    estimators=oof_models_dict,\n    voting='soft',\n)\nmodel.estimators_ = fitted_models\nmodel.le_ = LabelEncoder().fit(y)\nmodel.classes_ = model.le_.classes_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(model, \"oof_model_1.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump((train_df.columns, cat_cols, drop_cols), \"train_cat_columns.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(oof_pred, \"oof_pred.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X\ndel train_df\ndel oof_pred\ndel oof_df\ngc.collect()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"def predict_proba_in_batches(model, data, batch_size=100000):\n    num_samples = len(data)\n    num_batches = int(np.ceil(num_samples / batch_size))\n    probabilities = np.zeros((num_samples,))\n\n    for batch_idx in range(num_batches):\n        print(f\"Processing batch: {batch_idx+1}/{num_batches}\")\n        start_idx = batch_idx * batch_size\n        end_idx = min((batch_idx + 1) * batch_size, num_samples)\n        X_batch = data.iloc[start_idx:end_idx]\n        batch_probs = model.predict_proba(X_batch)[:, 1]\n        probabilities[start_idx:end_idx] = batch_probs\n        gc.collect()\n\n    return probabilities","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_df.drop(columns=[\"WEEK_NUM\"] + drop_cols)\nX_test = X_test.set_index(\"case_id\")\nprint(\"X_test shape: \", X_test.shape)\n\ny_pred = pd.Series(predict_proba_in_batches(model, X_test), index=X_test.index)\ny_pred[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"subm_df = pd.read_csv(data_nb.CFG.root_dir / \"sample_submission.csv\")\nsubm_df = subm_df.set_index(\"case_id\")\n\nsubm_df[\"score\"] = y_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Check null: \", subm_df[\"score\"].isnull().any())\n\nsubm_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm_df.to_csv(\"submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}