{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"},{"sourceId":162363627,"sourceType":"kernelVersion"},{"sourceId":162364554,"sourceType":"kernelVersion"},{"sourceId":162403520,"sourceType":"kernelVersion"},{"sourceId":162438049,"sourceType":"kernelVersion"},{"sourceId":162470947,"sourceType":"kernelVersion"}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":281.877528,"end_time":"2024-02-11T08:41:12.310691","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-02-11T08:36:30.433163","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{"papermill":{"duration":0.012452,"end_time":"2024-02-11T08:36:33.110261","exception":false,"start_time":"2024-02-11T08:36:33.097809","status":"completed"},"tags":[]}},{"cell_type":"code","source":"display = print","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Related notebooks**\n\nUtility script notebook (with addtional functions, aggregators; data collection):\n\nhttps://www.kaggle.com/andreynesterov/home-credit-baseline-data\n\nTraining models notebooks:\n\nhttps://www.kaggle.com/andreynesterov/home-credit-baseline-training\n\nhttps://www.kaggle.com/code/andreynesterov/home-credit-baseline-training-no-dates\n\nhttps://www.kaggle.com/code/andreynesterov/home-credit-baseline-training-lightautoml","metadata":{"papermill":{"duration":0.011039,"end_time":"2024-02-11T08:36:33.133055","exception":false,"start_time":"2024-02-11T08:36:33.122016","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Dependencies","metadata":{"papermill":{"duration":0.011294,"end_time":"2024-02-11T08:36:33.155630","exception":false,"start_time":"2024-02-11T08:36:33.144336","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install --no-index -Uq --find-links=/kaggle/input/lightautoml-038-dependencies pandas==2.0.3","metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2024-02-11T08:36:33.180251Z","iopub.status.busy":"2024-02-11T08:36:33.179883Z","iopub.status.idle":"2024-02-11T08:36:57.220796Z","shell.execute_reply":"2024-02-11T08:36:57.219687Z"},"papermill":{"duration":24.056457,"end_time":"2024-02-11T08:36:57.223293","exception":false,"start_time":"2024-02-11T08:36:33.166836","status":"completed"},"scrolled":true,"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.preprocessing import LabelEncoder\nfrom scipy.optimize import minimize\n\nimport joblib\nimport lightgbm as lgb\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2024-02-11T08:36:57.248534Z","iopub.status.busy":"2024-02-11T08:36:57.247696Z","iopub.status.idle":"2024-02-11T08:37:02.279560Z","shell.execute_reply":"2024-02-11T08:37:02.278783Z"},"papermill":{"duration":5.046717,"end_time":"2024-02-11T08:37:02.281893","exception":false,"start_time":"2024-02-11T08:36:57.235176","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions","metadata":{"papermill":{"duration":0.011787,"end_time":"2024-02-11T08:37:02.305835","exception":false,"start_time":"2024-02-11T08:37:02.294048","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import home_credit_baseline_data as data_nb","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:02.330582Z","iopub.status.busy":"2024-02-11T08:37:02.329852Z","iopub.status.idle":"2024-02-11T08:37:02.505920Z","shell.execute_reply":"2024-02-11T08:37:02.505196Z"},"papermill":{"duration":0.190797,"end_time":"2024-02-11T08:37:02.508178","exception":false,"start_time":"2024-02-11T08:37:02.317381","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### from https://www.kaggle.com/code/batprem/home-credit-risk-mode-utility-scripts\n\ndef gini_stability(base, score_col=\"score\", w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", score_col]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", score_col]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[score_col])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:02.532900Z","iopub.status.busy":"2024-02-11T08:37:02.532419Z","iopub.status.idle":"2024-02-11T08:37:02.539602Z","shell.execute_reply":"2024-02-11T08:37:02.538771Z"},"papermill":{"duration":0.021654,"end_time":"2024-02-11T08:37:02.541502","exception":false,"start_time":"2024-02-11T08:37:02.519848","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_proba_in_batches(model, data, batch_size=100000, predict_mode=\"base\"):\n    num_samples = len(data)\n    num_batches = int(np.ceil(num_samples / batch_size))\n    probabilities = np.zeros((num_samples,))\n\n    for batch_idx in range(num_batches):\n        print(f\"Processing batch: {batch_idx+1}/{num_batches}\")\n        start_idx = batch_idx * batch_size\n        end_idx = min((batch_idx + 1) * batch_size, num_samples)\n        X_batch = data.iloc[start_idx:end_idx]\n        if predict_mode == \"base\":\n            batch_probs = model.predict_proba(X_batch)[:, 1]\n        elif predict_mode == \"lightautoml\":\n            batch_probs = model.predict(X_batch).data.squeeze()\n        probabilities[start_idx:end_idx] = batch_probs\n        gc.collect()\n\n    return probabilities","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:02.565840Z","iopub.status.busy":"2024-02-11T08:37:02.565575Z","iopub.status.idle":"2024-02-11T08:37:02.572455Z","shell.execute_reply":"2024-02-11T08:37:02.571678Z"},"papermill":{"duration":0.021075,"end_time":"2024-02-11T08:37:02.574365","exception":false,"start_time":"2024-02-11T08:37:02.553290","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_base_df = pd.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_base.csv\")\ny_train = train_base_df[\"target\"]\noof_df = train_base_df\nmodels_score_df = pd.DataFrame()\n# model_names = []\ntest_preds_df = pd.DataFrame()","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:02.598323Z","iopub.status.busy":"2024-02-11T08:37:02.598064Z","iopub.status.idle":"2024-02-11T08:37:03.732466Z","shell.execute_reply":"2024-02-11T08:37:03.731469Z"},"papermill":{"duration":1.149191,"end_time":"2024-02-11T08:37:03.735005","exception":false,"start_time":"2024-02-11T08:37:02.585814","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model 1","metadata":{"papermill":{"duration":0.011674,"end_time":"2024-02-11T08:37:03.758479","exception":false,"start_time":"2024-02-11T08:37:03.746805","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model_name = \"model_1\"\nmodel_1 = joblib.load(\"/kaggle/input/home-credit-baseline-training/oof_model_1.pkl\")\nmodel_1\n# model_names.append(model_name)","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:03.784284Z","iopub.status.busy":"2024-02-11T08:37:03.783604Z","iopub.status.idle":"2024-02-11T08:37:04.158298Z","shell.execute_reply":"2024-02-11T08:37:04.157349Z"},"papermill":{"duration":0.389462,"end_time":"2024-02-11T08:37:04.160532","exception":false,"start_time":"2024-02-11T08:37:03.771070","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cols, cat_cols, drop_cols = joblib.load(\"/kaggle/input/home-credit-baseline-training/train_cat_columns.pkl\")\nprint(\"train_cols:\\t\", len(train_cols))\nprint(\"cat_cols:\\t\", len(cat_cols))\nprint(\"drop_cols:\\t\", len(drop_cols))","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:04.185937Z","iopub.status.busy":"2024-02-11T08:37:04.185653Z","iopub.status.idle":"2024-02-11T08:37:04.195481Z","shell.execute_reply":"2024-02-11T08:37:04.194424Z"},"papermill":{"duration":0.024579,"end_time":"2024-02-11T08:37:04.197310","exception":false,"start_time":"2024-02-11T08:37:04.172731","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = data_nb.prepare_df(data_nb.CFG.test_dir, cat_cols=cat_cols, mode=\"test\", train_cols=train_cols)\ndisplay(test_df)","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:04.222439Z","iopub.status.busy":"2024-02-11T08:37:04.222183Z","iopub.status.idle":"2024-02-11T08:37:05.364504Z","shell.execute_reply":"2024-02-11T08:37:05.363470Z"},"papermill":{"duration":1.164742,"end_time":"2024-02-11T08:37:05.374180","exception":false,"start_time":"2024-02-11T08:37:04.209438","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds_df['case_id'] = test_df['case_id']\ntest_preds_df.set_index('case_id', inplace=True)","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:05.417282Z","iopub.status.busy":"2024-02-11T08:37:05.416932Z","iopub.status.idle":"2024-02-11T08:37:05.424078Z","shell.execute_reply":"2024-02-11T08:37:05.423250Z"},"papermill":{"duration":0.03076,"end_time":"2024-02-11T08:37:05.425946","exception":false,"start_time":"2024-02-11T08:37:05.395186","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_df.drop(columns=[\"WEEK_NUM\"] + drop_cols)\nX_test = X_test.set_index(\"case_id\")\nprint(\"X_test shape: \", X_test.shape)\n\ny_pred_1 = pd.Series(predict_proba_in_batches(model_1, X_test), index=X_test.index)\ntest_preds_df[f\"pred_{model_name}\"] = y_pred_1","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:05.468687Z","iopub.status.busy":"2024-02-11T08:37:05.468385Z","iopub.status.idle":"2024-02-11T08:37:06.170684Z","shell.execute_reply":"2024-02-11T08:37:06.169870Z"},"papermill":{"duration":0.726296,"end_time":"2024-02-11T08:37:06.173058","exception":false,"start_time":"2024-02-11T08:37:05.446762","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df[f\"pred_{model_name}\"] = joblib.load(\"/kaggle/input/home-credit-baseline-training/oof_pred.pkl\")","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:06.218011Z","iopub.status.busy":"2024-02-11T08:37:06.217645Z","iopub.status.idle":"2024-02-11T08:37:06.342778Z","shell.execute_reply":"2024-02-11T08:37:06.341995Z"},"papermill":{"duration":0.149864,"end_time":"2024-02-11T08:37:06.345017","exception":false,"start_time":"2024-02-11T08:37:06.195153","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gini_score = gini_stability(oof_df, score_col=f\"pred_{model_name}\")\nmodels_score_df.loc[model_name, [\"gini_score\"]] = gini_score\nprint(\"gini_score:\\t\", gini_score)","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:06.388660Z","iopub.status.busy":"2024-02-11T08:37:06.388354Z","iopub.status.idle":"2024-02-11T08:37:07.180150Z","shell.execute_reply":"2024-02-11T08:37:07.179141Z"},"papermill":{"duration":0.815926,"end_time":"2024-02-11T08:37:07.182523","exception":false,"start_time":"2024-02-11T08:37:06.366597","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model 2","metadata":{"papermill":{"duration":0.020706,"end_time":"2024-02-11T08:37:07.225279","exception":false,"start_time":"2024-02-11T08:37:07.204573","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model_name = \"model_2\"\nmodel_2 = joblib.load(\"/kaggle/input/home-credit-baseline-training-model-2/oof_model.pkl\")\nmodel_2","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:07.268591Z","iopub.status.busy":"2024-02-11T08:37:07.268260Z","iopub.status.idle":"2024-02-11T08:37:07.624882Z","shell.execute_reply":"2024-02-11T08:37:07.623929Z"},"papermill":{"duration":0.38103,"end_time":"2024-02-11T08:37:07.627074","exception":false,"start_time":"2024-02-11T08:37:07.246044","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cols, cat_cols, drop_cols = joblib.load(\"/kaggle/input/home-credit-baseline-training-model-2/train_cat_columns.pkl\")\nprint(\"train_cols:\\t\", len(train_cols))\nprint(\"cat_cols:\\t\", len(cat_cols))\nprint(\"drop_cols:\\t\", len(drop_cols))","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:07.673992Z","iopub.status.busy":"2024-02-11T08:37:07.673677Z","iopub.status.idle":"2024-02-11T08:37:07.684173Z","shell.execute_reply":"2024-02-11T08:37:07.683159Z"},"papermill":{"duration":0.036148,"end_time":"2024-02-11T08:37:07.686262","exception":false,"start_time":"2024-02-11T08:37:07.650114","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_df.drop(columns=[\"WEEK_NUM\"] + drop_cols)\nX_test = X_test.set_index(\"case_id\")\nprint(\"X_test shape: \", X_test.shape)\n\ny_pred_2 = pd.Series(predict_proba_in_batches(model_2, X_test), index=X_test.index)\ntest_preds_df[f\"pred_{model_name}\"] = y_pred_2","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:07.731448Z","iopub.status.busy":"2024-02-11T08:37:07.730722Z","iopub.status.idle":"2024-02-11T08:37:08.436117Z","shell.execute_reply":"2024-02-11T08:37:08.435163Z"},"papermill":{"duration":0.730494,"end_time":"2024-02-11T08:37:08.438552","exception":false,"start_time":"2024-02-11T08:37:07.708058","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df[f\"pred_{model_name}\"] = joblib.load(\"/kaggle/input/home-credit-baseline-training-model-2/oof_pred.pkl\")","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:08.483629Z","iopub.status.busy":"2024-02-11T08:37:08.483322Z","iopub.status.idle":"2024-02-11T08:37:08.614598Z","shell.execute_reply":"2024-02-11T08:37:08.613676Z"},"papermill":{"duration":0.156226,"end_time":"2024-02-11T08:37:08.617051","exception":false,"start_time":"2024-02-11T08:37:08.460825","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gini_score = gini_stability(oof_df, score_col=f\"pred_{model_name}\")\nmodels_score_df.loc[model_name, [\"gini_score\"]] = gini_score\nprint(\"gini_score:\\t\", gini_score)","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:37:08.667725Z","iopub.status.busy":"2024-02-11T08:37:08.667389Z","iopub.status.idle":"2024-02-11T08:37:09.518422Z","shell.execute_reply":"2024-02-11T08:37:09.517420Z"},"papermill":{"duration":0.880853,"end_time":"2024-02-11T08:37:09.520550","exception":false,"start_time":"2024-02-11T08:37:08.639697","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model 3","metadata":{"papermill":{"duration":0.021667,"end_time":"2024-02-11T08:37:09.564498","exception":false,"start_time":"2024-02-11T08:37:09.542831","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install --no-index -Uq --find-links=/kaggle/input/lightautoml-038-dependencies lightautoml==0.3.8","metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2024-02-11T08:37:09.649515Z","iopub.status.busy":"2024-02-11T08:37:09.649162Z","iopub.status.idle":"2024-02-11T08:39:27.218839Z","shell.execute_reply":"2024-02-11T08:39:27.217445Z"},"papermill":{"duration":137.595643,"end_time":"2024-02-11T08:39:27.221759","exception":false,"start_time":"2024-02-11T08:37:09.626116","status":"completed"},"scrolled":true,"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightautoml.automl.presets.tabular_presets import TabularAutoML\nfrom lightautoml.tasks import Task","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:39:27.267769Z","iopub.status.busy":"2024-02-11T08:39:27.267399Z","iopub.status.idle":"2024-02-11T08:39:55.230514Z","shell.execute_reply":"2024-02-11T08:39:55.229666Z"},"papermill":{"duration":27.988395,"end_time":"2024-02-11T08:39:55.232891","exception":false,"start_time":"2024-02-11T08:39:27.244496","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = \"denselight_model\"\nmodel_3 = joblib.load(\"/kaggle/input/home-credit-baseline-training-lightautoml/denselight_model.pkl\")\nmodel_3","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:39:55.279700Z","iopub.status.busy":"2024-02-11T08:39:55.278469Z","iopub.status.idle":"2024-02-11T08:39:56.227424Z","shell.execute_reply":"2024-02-11T08:39:56.226476Z"},"papermill":{"duration":0.974078,"end_time":"2024-02-11T08:39:56.229609","exception":false,"start_time":"2024-02-11T08:39:55.255531","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cols, cat_cols, drop_cols = joblib.load(\"/kaggle/input/home-credit-baseline-training-lightautoml/train_cat_columns.pkl\")\nprint(\"train_cols:\\t\", len(train_cols))\nprint(\"cat_cols:\\t\", len(cat_cols))\nprint(\"drop_cols:\\t\", len(drop_cols))","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:39:56.275286Z","iopub.status.busy":"2024-02-11T08:39:56.274931Z","iopub.status.idle":"2024-02-11T08:39:56.285234Z","shell.execute_reply":"2024-02-11T08:39:56.284301Z"},"papermill":{"duration":0.035223,"end_time":"2024-02-11T08:39:56.287195","exception":false,"start_time":"2024-02-11T08:39:56.251972","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_df.drop(columns=[\"WEEK_NUM\"] + drop_cols)\nX_test = X_test.set_index(\"case_id\")\nprint(\"X_test shape: \", X_test.shape)\n\ny_pred_3 = pd.Series(\n    predict_proba_in_batches(model_3, X_test, predict_mode = \"lightautoml\"),\n    index=X_test.index)\ntest_preds_df[f\"pred_{model_name}\"] = y_pred_3","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:39:56.333826Z","iopub.status.busy":"2024-02-11T08:39:56.333014Z","iopub.status.idle":"2024-02-11T08:39:59.783298Z","shell.execute_reply":"2024-02-11T08:39:59.782452Z"},"papermill":{"duration":3.475767,"end_time":"2024-02-11T08:39:59.785596","exception":false,"start_time":"2024-02-11T08:39:56.309829","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df[f\"pred_{model_name}\"] = joblib.load(\"/kaggle/input/home-credit-baseline-training-lightautoml/denselight_oof_preds.pkl\")","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:39:59.832366Z","iopub.status.busy":"2024-02-11T08:39:59.832034Z","iopub.status.idle":"2024-02-11T08:39:59.884896Z","shell.execute_reply":"2024-02-11T08:39:59.884091Z"},"papermill":{"duration":0.078549,"end_time":"2024-02-11T08:39:59.887256","exception":false,"start_time":"2024-02-11T08:39:59.808707","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gini_score = gini_stability(oof_df, score_col=f\"pred_{model_name}\")\nmodels_score_df.loc[model_name, [\"gini_score\"]] = gini_score\nprint(\"gini_score:\\t\", gini_score)","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:39:59.933565Z","iopub.status.busy":"2024-02-11T08:39:59.933229Z","iopub.status.idle":"2024-02-11T08:40:00.696909Z","shell.execute_reply":"2024-02-11T08:40:00.696003Z"},"papermill":{"duration":0.789404,"end_time":"2024-02-11T08:40:00.699370","exception":false,"start_time":"2024-02-11T08:39:59.909966","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Estimation Results","metadata":{"papermill":{"duration":0.022794,"end_time":"2024-02-11T08:40:00.747726","exception":false,"start_time":"2024-02-11T08:40:00.724932","status":"completed"},"tags":[]}},{"cell_type":"code","source":"models_score_df","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:40:00.794372Z","iopub.status.busy":"2024-02-11T08:40:00.794025Z","iopub.status.idle":"2024-02-11T08:40:00.802300Z","shell.execute_reply":"2024-02-11T08:40:00.801410Z"},"papermill":{"duration":0.03477,"end_time":"2024-02-11T08:40:00.804523","exception":false,"start_time":"2024-02-11T08:40:00.769753","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:40:00.851080Z","iopub.status.busy":"2024-02-11T08:40:00.850729Z","iopub.status.idle":"2024-02-11T08:40:00.867407Z","shell.execute_reply":"2024-02-11T08:40:00.866525Z"},"papermill":{"duration":0.041416,"end_time":"2024-02-11T08:40:00.869237","exception":false,"start_time":"2024-02-11T08:40:00.827821","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_df\ngc.collect()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.execute_input":"2024-02-11T08:40:00.915492Z","iopub.status.busy":"2024-02-11T08:40:00.915204Z","iopub.status.idle":"2024-02-11T08:40:01.179508Z","shell.execute_reply":"2024-02-11T08:40:01.178543Z"},"papermill":{"duration":0.289345,"end_time":"2024-02-11T08:40:01.181330","exception":false,"start_time":"2024-02-11T08:40:00.891985","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Blending","metadata":{"papermill":{"duration":0.022646,"end_time":"2024-02-11T08:40:01.227251","exception":false,"start_time":"2024-02-11T08:40:01.204605","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def gini_wrapper(base_df):\n    base_df = base_df[[\"WEEK_NUM\", \"target\"]].copy()\n    def gini_wrapper_inner(target, scores):\n        base_df[\"score\"] = scores\n        gini_score = gini_stability(base_df, score_col=\"score\")\n        return 1 - gini_score\n    return gini_wrapper_inner","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:40:01.273977Z","iopub.status.busy":"2024-02-11T08:40:01.273243Z","iopub.status.idle":"2024-02-11T08:40:01.278619Z","shell.execute_reply":"2024-02-11T08:40:01.277777Z"},"papermill":{"duration":0.03077,"end_time":"2024-02-11T08:40:01.280454","exception":false,"start_time":"2024-02-11T08:40:01.249684","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hill climbing using minimize","metadata":{"papermill":{"duration":0.022225,"end_time":"2024-02-11T08:40:01.325493","exception":false,"start_time":"2024-02-11T08:40:01.303268","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class WeightsSearcher:\n    def __init__(self, loss_fn, bounds=[], mode=\"min\", method='SLSQP'):\n        self.loss_fn = loss_fn\n        self.bounds = bounds\n        self.mode = mode\n        self.method = method # Nelder-Mead - for not smooth functions\n        \n    def _objective_function_wrapper(self, pred_values, true_targets, obj_fn):\n        def objective_function(weights):\n            pred_weighted = (pred_values * weights).sum(axis=1)\n            score = obj_fn(true_targets, pred_weighted)\n            return score\n        return objective_function\n    \n    def find_weights(self, val_preds, true_targets):\n        len_models = len(self.bounds)\n        bounds = [0,1] * len_models if len(self.bounds) == 0 else self.bounds\n        initial_weights = np.ones(len_models) / len_models\n        objective_function = self._objective_function_wrapper(val_preds, true_targets, self.loss_fn)\n        result = minimize(\n            objective_function, \n            initial_weights, \n            bounds=bounds, \n            method=self.method,\n        )\n        optimized_weights = result.x\n        optimized_weights /= np.sum(optimized_weights)\n        return optimized_weights","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:40:01.371765Z","iopub.status.busy":"2024-02-11T08:40:01.371410Z","iopub.status.idle":"2024-02-11T08:40:01.380157Z","shell.execute_reply":"2024-02-11T08:40:01.379270Z"},"papermill":{"duration":0.033967,"end_time":"2024-02-11T08:40:01.382049","exception":false,"start_time":"2024-02-11T08:40:01.348082","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_names = models_score_df.index.to_list()\npred_cols = [f\"pred_{name}\" for name in model_names]\nmodel_names","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:40:01.427989Z","iopub.status.busy":"2024-02-11T08:40:01.427500Z","iopub.status.idle":"2024-02-11T08:40:01.433561Z","shell.execute_reply":"2024-02-11T08:40:01.432640Z"},"papermill":{"duration":0.031101,"end_time":"2024-02-11T08:40:01.435449","exception":false,"start_time":"2024-02-11T08:40:01.404348","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bounds = [(0, 1)] * len(pred_cols)\nroc_auc_fn = lambda y_true, y_pred: 1 - roc_auc_score(y_true, y_pred)\ngini_score_fn = gini_wrapper(oof_df)\nw_searcher = WeightsSearcher(gini_score_fn, bounds, method='Nelder-Mead') # log_loss, gini_stability roc_auc_fn\noptimized_weights = w_searcher.find_weights(\n    oof_df[pred_cols].to_numpy(), \n    y_train\n)\noptimized_weights_df = pd.DataFrame(zip(model_names, optimized_weights), columns=['model', 'weight'])\ndisplay(optimized_weights_df)\nprint(\"sum: \", np.sum(optimized_weights))","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:40:01.481587Z","iopub.status.busy":"2024-02-11T08:40:01.481316Z","iopub.status.idle":"2024-02-11T08:41:08.169813Z","shell.execute_reply":"2024-02-11T08:41:08.168986Z"},"papermill":{"duration":66.737809,"end_time":"2024-02-11T08:41:08.195833","exception":false,"start_time":"2024-02-11T08:40:01.458024","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_pred_optimized = (oof_df[pred_cols] * optimized_weights).sum(axis=1).to_numpy()\noof_df[\"pred_optimized\"] = oof_pred_optimized\nroc_auc_oof = roc_auc_score(y_train, oof_pred_optimized)\ngini_score = gini_stability(oof_df, score_col=\"pred_optimized\")\nprint(\"CV roc_auc_oof optimized:\\t\", roc_auc_oof)\nprint(\"CV gini_score:\\t\\t\\t\", gini_score)","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:41:08.243916Z","iopub.status.busy":"2024-02-11T08:41:08.243295Z","iopub.status.idle":"2024-02-11T08:41:10.133000Z","shell.execute_reply":"2024-02-11T08:41:10.132024Z"},"papermill":{"duration":1.916425,"end_time":"2024-02-11T08:41:10.135305","exception":false,"start_time":"2024-02-11T08:41:08.218880","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_all = (optimized_weights * test_preds_df[pred_cols]).sum(axis=1).to_numpy()\ny_pred_all[:10]","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:41:10.185435Z","iopub.status.busy":"2024-02-11T08:41:10.185105Z","iopub.status.idle":"2024-02-11T08:41:10.194406Z","shell.execute_reply":"2024-02-11T08:41:10.193578Z"},"papermill":{"duration":0.035874,"end_time":"2024-02-11T08:41:10.196297","exception":false,"start_time":"2024-02-11T08:41:10.160423","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.022678,"end_time":"2024-02-11T08:41:10.242291","exception":false,"start_time":"2024-02-11T08:41:10.219613","status":"completed"},"tags":[]}},{"cell_type":"code","source":"subm_df = pd.read_csv(data_nb.CFG.root_dir / \"sample_submission.csv\")\nsubm_df = subm_df.set_index(\"case_id\")\nsubm_df[\"score\"] = y_pred_all\ndisplay(subm_df.head())\nprint(\"Check null: \", subm_df[\"score\"].isnull().any())","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:41:10.289691Z","iopub.status.busy":"2024-02-11T08:41:10.289398Z","iopub.status.idle":"2024-02-11T08:41:10.305761Z","shell.execute_reply":"2024-02-11T08:41:10.304819Z"},"papermill":{"duration":0.042526,"end_time":"2024-02-11T08:41:10.307975","exception":false,"start_time":"2024-02-11T08:41:10.265449","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm_df.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.execute_input":"2024-02-11T08:41:10.356579Z","iopub.status.busy":"2024-02-11T08:41:10.356307Z","iopub.status.idle":"2024-02-11T08:41:10.362111Z","shell.execute_reply":"2024-02-11T08:41:10.361202Z"},"papermill":{"duration":0.032394,"end_time":"2024-02-11T08:41:10.364033","exception":false,"start_time":"2024-02-11T08:41:10.331639","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}