{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"},{"sourceId":162338974,"sourceType":"kernelVersion"}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":1450.149075,"end_time":"2024-02-09T20:41:54.384237","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-02-09T20:17:44.235162","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{"papermill":{"duration":0.006849,"end_time":"2024-02-09T20:17:46.933113","exception":false,"start_time":"2024-02-09T20:17:46.926264","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Based on:**\n\nhttps://www.kaggle.com/code/greysky/home-credit-baseline\n\n**Related notebooks**\n\nUtility script notebook (with addtional functions, aggregators):\n\nhttps://www.kaggle.com/andreynesterov/home-credit-baseline-data\n\nInference notebook:\n\nhttps://www.kaggle.com/andreynesterov/home-credit-baseline-inference","metadata":{"papermill":{"duration":0.005813,"end_time":"2024-02-09T20:17:46.945111","exception":false,"start_time":"2024-02-09T20:17:46.939298","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Dependencies","metadata":{"papermill":{"duration":0.005895,"end_time":"2024-02-09T20:17:46.957434","exception":false,"start_time":"2024-02-09T20:17:46.951539","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport joblib\nimport lightgbm as lgb\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.preprocessing import LabelEncoder\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":6.124386,"end_time":"2024-02-09T20:17:53.087910","exception":false,"start_time":"2024-02-09T20:17:46.963524","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-12T10:39:52.823552Z","iopub.execute_input":"2024-02-12T10:39:52.823944Z","iopub.status.idle":"2024-02-12T10:39:57.954222Z","shell.execute_reply.started":"2024-02-12T10:39:52.823889Z","shell.execute_reply":"2024-02-12T10:39:57.953246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data collection","metadata":{"papermill":{"duration":0.00635,"end_time":"2024-02-09T20:17:53.100686","exception":false,"start_time":"2024-02-09T20:17:53.094336","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Import utility script notebook with data collection functions. See notebook for details: [Home Credit: baseline - DATA](https://www.kaggle.com/code/andreynesterov/home-credit-baseline-data)","metadata":{"papermill":{"duration":0.005936,"end_time":"2024-02-09T20:17:53.112760","exception":false,"start_time":"2024-02-09T20:17:53.106824","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import home_credit_baseline_data as data_nb","metadata":{"papermill":{"duration":0.023909,"end_time":"2024-02-09T20:17:53.142845","exception":false,"start_time":"2024-02-09T20:17:53.118936","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-12T10:39:57.955707Z","iopub.execute_input":"2024-02-12T10:39:57.956031Z","iopub.status.idle":"2024-02-12T10:39:57.979433Z","shell.execute_reply.started":"2024-02-12T10:39:57.956005Z","shell.execute_reply":"2024-02-12T10:39:57.978729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# data_nb.Aggregator.group_aggregators = [pl.n_unique]\ntrain_df = data_nb.prepare_df(data_nb.CFG.train_dir)\ncat_cols = list(train_df.select_dtypes(\"category\").columns)\ndisplay(train_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":143.621672,"end_time":"2024-02-09T20:20:16.771827","exception":false,"start_time":"2024-02-09T20:17:53.150155","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-12T10:39:57.980404Z","iopub.execute_input":"2024-02-12T10:39:57.980668Z","iopub.status.idle":"2024-02-12T10:42:20.254981Z","shell.execute_reply.started":"2024-02-12T10:39:57.980646Z","shell.execute_reply":"2024-02-12T10:42:20.254035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(cat_cols)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.028373,"end_time":"2024-02-09T20:20:16.816716","exception":false,"start_time":"2024-02-09T20:20:16.788343","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-02-12T10:42:20.257778Z","iopub.execute_input":"2024-02-12T10:42:20.258130Z","iopub.status.idle":"2024-02-12T10:42:20.267544Z","shell.execute_reply.started":"2024-02-12T10:42:20.258101Z","shell.execute_reply":"2024-02-12T10:42:20.266617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = data_nb.prepare_df(data_nb.CFG.test_dir, cat_cols=cat_cols, mode=\"test\", train_cols=train_df.columns)\ndisplay(test_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.978397,"end_time":"2024-02-09T20:20:17.811801","exception":false,"start_time":"2024-02-09T20:20:16.833404","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-12T10:42:20.268688Z","iopub.execute_input":"2024-02-12T10:42:20.268971Z","iopub.status.idle":"2024-02-12T10:42:21.176000Z","shell.execute_reply.started":"2024-02-12T10:42:20.268949Z","shell.execute_reply":"2024-02-12T10:42:21.175089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Drop columns","metadata":{"papermill":{"duration":0.025739,"end_time":"2024-02-09T20:20:17.863257","exception":false,"start_time":"2024-02-09T20:20:17.837518","status":"completed"},"tags":[]}},{"cell_type":"code","source":"list(train_df.columns)","metadata":{"scrolled":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-12T10:42:21.177299Z","iopub.execute_input":"2024-02-12T10:42:21.177631Z","iopub.status.idle":"2024-02-12T10:42:21.193284Z","shell.execute_reply.started":"2024-02-12T10:42:21.177602Z","shell.execute_reply":"2024-02-12T10:42:21.192310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Drop date features","metadata":{}},{"cell_type":"code","source":"drop_cols = []\ndrop_cols_endwith = [\"D\"]\nfor name_prefix in drop_cols_endwith:\n    cols_names = train_df.columns[train_df.columns.str.endswith(name_prefix)]\n    drop_cols += cols_names.to_list()\ndisplay(drop_cols)\ndisplay(len(drop_cols))","metadata":{"papermill":{"duration":0.03341,"end_time":"2024-02-09T20:20:17.922246","exception":false,"start_time":"2024-02-09T20:20:17.888836","status":"completed"},"scrolled":true,"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-12T10:42:21.194445Z","iopub.execute_input":"2024-02-12T10:42:21.194804Z","iopub.status.idle":"2024-02-12T10:42:21.207645Z","shell.execute_reply.started":"2024-02-12T10:42:21.194772Z","shell.execute_reply":"2024-02-12T10:42:21.206686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.025442,"end_time":"2024-02-09T20:20:17.972884","exception":false,"start_time":"2024-02-09T20:20:17.947442","status":"completed"},"tags":[]}},{"cell_type":"code","source":"### from https://www.kaggle.com/code/batprem/home-credit-risk-mode-utility-scripts\n\ndef gini_stability(base, score_col=\"score\", w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", score_col]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", score_col]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[score_col])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nX = train_df.drop(columns=[\"target\", \"case_id\", \"WEEK_NUM\"] + drop_cols)\nprint(\"X shape: \", X.shape)\ny = train_df[\"target\"]\nweeks = train_df[\"WEEK_NUM\"]\n\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 8,\n    \"max_bin\": 128,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 1000,\n    \"colsample_bytree\": 0.8, \n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    \"device\": \"gpu\",\n    \"gpu_platform_id\": 0,\n    \"gpu_device_id\": 0,\n    \"num_thread\": 2,\n#     \"gpu_use_dp\": False,\n}\n\nfitted_models = []\noof_pred = np.zeros(X.shape[0])\n\nfor idx_train, idx_valid in cv.split(X, y, groups=weeks):\n    X_train, y_train = X.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X.iloc[idx_valid], y.iloc[idx_valid]\n\n    model = lgb.LGBMClassifier(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        callbacks=[lgb.log_evaluation(100), lgb.early_stopping(100)]\n    )\n    fitted_models.append(model)\n    val_pred = model.predict_proba(X_valid)[:, 1]\n    oof_pred[idx_valid] = val_pred\n    gc.collect()","metadata":{"_kg_hide-output":true,"papermill":{"duration":1291.921212,"end_time":"2024-02-09T20:41:50.162021","exception":false,"start_time":"2024-02-09T20:20:18.240809","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-02-12T10:42:21.209094Z","iopub.execute_input":"2024-02-12T10:42:21.209427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc_oof = roc_auc_score(y, oof_pred)\nprint(\"CV roc_auc_oof: \", roc_auc_oof)","metadata":{"papermill":{"duration":0.769099,"end_time":"2024-02-09T20:41:50.962939","exception":false,"start_time":"2024-02-09T20:41:50.193840","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df = train_df[[\"WEEK_NUM\", \"target\"]].copy()\noof_df[\"pred_oof\"] = oof_pred\ngini_score = gini_stability(oof_df, score_col=\"pred_oof\")\nprint(\"gini_score:\\t\", gini_score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_models_dict = [(str(i), model) for i, model in enumerate(fitted_models)]\n\nmodel = VotingClassifier(\n    estimators=oof_models_dict,\n    voting='soft',\n)\nmodel.estimators_ = fitted_models\nmodel.le_ = LabelEncoder().fit(y)\nmodel.classes_ = model.le_.classes_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(model, \"oof_model.pkl\")","metadata":{"papermill":{"duration":0.617175,"end_time":"2024-02-09T20:41:51.611456","exception":false,"start_time":"2024-02-09T20:41:50.994281","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump((train_df.columns, cat_cols, drop_cols), \"train_cat_columns.pkl\")","metadata":{"papermill":{"duration":0.04376,"end_time":"2024-02-09T20:41:51.687256","exception":false,"start_time":"2024-02-09T20:41:51.643496","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(oof_pred, \"oof_pred.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X\ndel train_df\ndel oof_pred\ngc.collect()","metadata":{"papermill":{"duration":0.166807,"end_time":"2024-02-09T20:41:51.934896","exception":false,"start_time":"2024-02-09T20:41:51.768089","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{"papermill":{"duration":0.031089,"end_time":"2024-02-09T20:41:51.998070","exception":false,"start_time":"2024-02-09T20:41:51.966981","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def predict_proba_in_batches(model, data, batch_size=100000):\n    num_samples = len(data)\n    num_batches = int(np.ceil(num_samples / batch_size))\n    probabilities = np.zeros((num_samples,))\n\n    for batch_idx in range(num_batches):\n        print(f\"Processing batch: {batch_idx+1}/{num_batches}\")\n        start_idx = batch_idx * batch_size\n        end_idx = min((batch_idx + 1) * batch_size, num_samples)\n        X_batch = data.iloc[start_idx:end_idx]\n        batch_probs = model.predict_proba(X_batch)[:, 1]\n        probabilities[start_idx:end_idx] = batch_probs\n        gc.collect()\n\n    return probabilities","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_df.drop(columns=[\"WEEK_NUM\"] + drop_cols)\nX_test = X_test.set_index(\"case_id\")\nprint(\"X_test shape: \", X_test.shape)\n\ny_pred = pd.Series(predict_proba_in_batches(model, X_test), index=X_test.index)\ny_pred[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.03177,"end_time":"2024-02-09T20:41:52.680734","exception":false,"start_time":"2024-02-09T20:41:52.648964","status":"completed"},"tags":[]}},{"cell_type":"code","source":"subm_df = pd.read_csv(data_nb.CFG.root_dir / \"sample_submission.csv\")\nsubm_df = subm_df.set_index(\"case_id\")\n\nsubm_df[\"score\"] = y_pred","metadata":{"papermill":{"duration":0.054343,"end_time":"2024-02-09T20:41:52.766247","exception":false,"start_time":"2024-02-09T20:41:52.711904","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Check null: \", subm_df[\"score\"].isnull().any())\n\nsubm_df.head()","metadata":{"papermill":{"duration":0.046577,"end_time":"2024-02-09T20:41:52.848167","exception":false,"start_time":"2024-02-09T20:41:52.801590","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm_df.to_csv(\"submission.csv\")","metadata":{"papermill":{"duration":0.044206,"end_time":"2024-02-09T20:41:52.925405","exception":false,"start_time":"2024-02-09T20:41:52.881199","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}