{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8405978,"sourceType":"datasetVersion","datasetId":4976625,"isSourceIdPinned":true},{"sourceId":162470947,"sourceType":"kernelVersion"},{"sourceId":177639500,"sourceType":"kernelVersion"}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --no-index -Uq --find-links=/kaggle/input/lightautoml-038-dependencies lightautoml==0.3.8","metadata":{"execution":{"iopub.status.busy":"2024-05-22T08:46:10.513389Z","iopub.execute_input":"2024-05-22T08:46:10.514182Z","iopub.status.idle":"2024-05-22T08:48:49.826003Z","shell.execute_reply.started":"2024-05-22T08:46:10.514147Z","shell.execute_reply":"2024-05-22T08:48:49.824645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, RegressorMixin\n\nimport joblib\n\nimport lightgbm as lgb\n\nimport torch\nimport torch.nn as nn\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-22T08:48:52.322911Z","iopub.execute_input":"2024-05-22T08:48:52.323277Z","iopub.status.idle":"2024-05-22T08:49:00.030761Z","shell.execute_reply.started":"2024-05-22T08:48:52.323244Z","shell.execute_reply":"2024-05-22T08:49:00.029148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightautoml.automl.presets.tabular_presets import TabularAutoML\nfrom lightautoml.tasks import Task\nfrom sklearn.metrics import mean_squared_error, roc_auc_score, log_loss","metadata":{"execution":{"iopub.status.busy":"2024-05-22T08:49:00.034129Z","iopub.execute_input":"2024-05-22T08:49:00.035350Z","iopub.status.idle":"2024-05-22T08:49:28.535434Z","shell.execute_reply.started":"2024-05-22T08:49:00.035314Z","shell.execute_reply":"2024-05-22T08:49:28.534351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pre-Fitted Voting Model","metadata":{}},{"cell_type":"code","source":"class VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n    \n    def predict_proba(self, X):\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T08:49:28.536830Z","iopub.execute_input":"2024-05-22T08:49:28.537696Z","iopub.status.idle":"2024-05-22T08:49:28.545570Z","shell.execute_reply.started":"2024-05-22T08:49:28.537664Z","shell.execute_reply":"2024-05-22T08:49:28.544613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_class(x, task, reader):\n    if task.name == 'multiclass':\n        return reader[x]\n    else:\n        return x\n\nmapped = np.vectorize(map_class)\n\ndef score(task, y_true, y_pred):\n    if task.name == 'binary':\n        return roc_auc_score(y_true, y_pred)\n    elif task.name == 'multiclass':\n        return log_loss(y_true, y_pred)\n    elif task.name == 'reg' or task.name == 'multi:reg':\n        return mean_absolute_error(y_true, y_pred)\n    else:\n        raise 'Task is not correct.'\n        \ndef take_pred_from_task(pred, task):\n    if task.name == 'binary' or task.name == 'reg':\n        return pred[:, 0]\n    elif task.name == 'multiclass' or task.name == 'multi:reg':\n        return pred\n    else:\n        raise 'Task is not correct.'\n        \ndef use_plr(USE_PLR):\n    if USE_PLR:\n        return \"plr\"\n    else:\n        return \"cont\"","metadata":{"execution":{"iopub.status.busy":"2024-05-22T08:49:28.548092Z","iopub.execute_input":"2024-05-22T08:49:28.548379Z","iopub.status.idle":"2024-05-22T08:49:28.568368Z","shell.execute_reply.started":"2024-05-22T08:49:28.548355Z","shell.execute_reply":"2024-05-22T08:49:28.567435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = joblib.load(\"/kaggle/input/home-credit-feature-engineering/df_train.pkl\")","metadata":{"execution":{"iopub.status.busy":"2024-05-22T08:49:28.569663Z","iopub.execute_input":"2024-05-22T08:49:28.570338Z","iopub.status.idle":"2024-05-22T08:49:38.595454Z","shell.execute_reply.started":"2024-05-22T08:49:28.570311Z","shell.execute_reply":"2024-05-22T08:49:38.594611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = joblib.load(\"/kaggle/input/home-credit-feature-engineering/cat_columns.pkl\")","metadata":{"execution":{"iopub.status.busy":"2024-05-22T08:49:38.596552Z","iopub.execute_input":"2024-05-22T08:49:38.596874Z","iopub.status.idle":"2024-05-22T08:49:38.604230Z","shell.execute_reply.started":"2024-05-22T08:49:38.596848Z","shell.execute_reply":"2024-05-22T08:49:38.603097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categories = joblib.load(\"/kaggle/input/home-credit-feature-engineering/categories.pkl\")","metadata":{"execution":{"iopub.status.busy":"2024-05-22T08:49:38.605815Z","iopub.execute_input":"2024-05-22T08:49:38.606180Z","iopub.status.idle":"2024-05-22T08:49:38.616322Z","shell.execute_reply.started":"2024-05-22T08:49:38.606145Z","shell.execute_reply":"2024-05-22T08:49:38.615421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_THREADS = 4\nN_FOLDS = 5\nRANDOM_STATE = 2024\nTIMEOUT = 10000\nADVANCED_ROLES = False\nUSE_QNT = False\nUSE_PLR = False\nTRAIN_BS = 128\nEPOCHS = 5\nTARGET_NAME = 'target'\n\nnp.random.seed(RANDOM_STATE)\ntorch.set_num_threads(N_THREADS)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T08:49:38.617469Z","iopub.execute_input":"2024-05-22T08:49:38.617846Z","iopub.status.idle":"2024-05-22T08:49:38.627295Z","shell.execute_reply.started":"2024-05-22T08:49:38.617812Z","shell.execute_reply":"2024-05-22T08:49:38.626373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"task = Task(\n    'binary', \n    loss = 'logloss', \n    metric = 'auc'\n)\nroles = {\n    'target': TARGET_NAME,\n    'group': \"WEEK_NUM\",\n    'drop': ['case_id', \"WEEK_NUM\", \"MONTH\",\n             \"riskassesment_302T_mean\",\n             \"riskassesment_302T_rng\",\n             \"riskassesment_940T\"],\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-22T08:55:35.562467Z","iopub.execute_input":"2024-05-22T08:55:35.563255Z","iopub.status.idle":"2024-05-22T08:55:35.579429Z","shell.execute_reply.started":"2024-05-22T08:55:35.563222Z","shell.execute_reply":"2024-05-22T08:55:35.578253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nalgo = 'denselight'\nautoml_model = TabularAutoML(\n    task = task, \n    timeout = TIMEOUT,\n    cpu_limit = N_THREADS,\n    gpu_ids='0',\n    general_params = {\"use_algos\": [[algo]]}, # ['nn', 'mlp', 'dense', 'denselight', 'resnet', 'snn', 'node', 'autoint', 'fttransformer'] or custom torch model\n    nn_params = {\n        \"n_epochs\": EPOCHS, \n        \"bs\": TRAIN_BS, \n        \"num_workers\": 0, \n        \"path_to_save\": None, \n        \"freeze_defaults\": True,\n        \"cont_embedder\": use_plr(USE_PLR),\n    },\n    nn_pipeline_params = {\n        \"use_qnt\": USE_QNT, \n        \"use_te\": False\n    },\n    reader_params = {\n        'n_jobs': N_THREADS, \n        'cv': N_FOLDS, \n        'random_state': RANDOM_STATE, \n        'advanced_roles': ADVANCED_ROLES\n    },\n)\n\noof_pred = automl_model.fit_predict(df_train, roles = roles, verbose = 3)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T08:55:36.909924Z","iopub.execute_input":"2024-05-22T08:55:36.910821Z","iopub.status.idle":"2024-05-22T09:01:25.982831Z","shell.execute_reply.started":"2024-05-22T08:55:36.910788Z","shell.execute_reply":"2024-05-22T09:01:25.981808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = score(\n    task,\n    mapped(df_train[TARGET_NAME].values, task, automl_model.reader.class_mapping),\n    take_pred_from_task(oof_pred.data, task)\n)\n\nprint(\"CV roc_auc_oof: \", oof)","metadata":{"execution":{"iopub.status.busy":"2024-05-10T15:13:16.767928Z","iopub.execute_input":"2024-05-10T15:13:16.768207Z","iopub.status.idle":"2024-05-10T15:13:17.974411Z","shell.execute_reply.started":"2024-05-10T15:13:16.768182Z","shell.execute_reply":"2024-05-10T15:13:17.973454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gini_stability(base, score_col=\"score\", w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", score_col]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", score_col]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[score_col])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std","metadata":{"execution":{"iopub.status.busy":"2024-05-10T15:13:17.975561Z","iopub.execute_input":"2024-05-10T15:13:17.975856Z","iopub.status.idle":"2024-05-10T15:13:17.983416Z","shell.execute_reply.started":"2024-05-10T15:13:17.975830Z","shell.execute_reply":"2024-05-10T15:13:17.982463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df = df_train[[\"case_id\", \"WEEK_NUM\", \"target\"]]\noof_df[\"score\"] = oof_pred.data\n\ngini_score = gini_stability(oof_df)\nprint(\"gini_score:\\t\", gini_score)","metadata":{"execution":{"iopub.status.busy":"2024-05-10T15:13:17.985352Z","iopub.execute_input":"2024-05-10T15:13:17.985595Z","iopub.status.idle":"2024-05-10T15:13:18.731773Z","shell.execute_reply.started":"2024-05-10T15:13:17.985573Z","shell.execute_reply":"2024-05-10T15:13:18.730762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(oof_pred.data, 'denselight_oof_preds.pkl')","metadata":{"execution":{"iopub.status.busy":"2024-05-10T15:13:18.732816Z","iopub.execute_input":"2024-05-10T15:13:18.733077Z","iopub.status.idle":"2024-05-10T15:13:18.746172Z","shell.execute_reply.started":"2024-05-10T15:13:18.733055Z","shell.execute_reply":"2024-05-10T15:13:18.745210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(automl_model, 'denselight_model.pkl')","metadata":{"execution":{"iopub.status.busy":"2024-05-10T15:13:18.747371Z","iopub.execute_input":"2024-05-10T15:13:18.747718Z","iopub.status.idle":"2024-05-10T15:13:19.076782Z","shell.execute_reply.started":"2024-05-10T15:13:18.747687Z","shell.execute_reply":"2024-05-10T15:13:19.075818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump((df_train.columns, cat_cols), \"train_cat_columns.pkl\")","metadata":{"execution":{"iopub.status.busy":"2024-05-10T15:13:19.078445Z","iopub.execute_input":"2024-05-10T15:13:19.078746Z","iopub.status.idle":"2024-05-10T15:13:19.085882Z","shell.execute_reply.started":"2024-05-10T15:13:19.078709Z","shell.execute_reply":"2024-05-10T15:13:19.085007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(categories, \"categories.pkl\")","metadata":{"execution":{"iopub.status.busy":"2024-05-10T15:13:19.086945Z","iopub.execute_input":"2024-05-10T15:13:19.087197Z","iopub.status.idle":"2024-05-10T15:13:19.097570Z","shell.execute_reply.started":"2024-05-10T15:13:19.087171Z","shell.execute_reply":"2024-05-10T15:13:19.096677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}