{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8430651,"sourceType":"datasetVersion","datasetId":4496896},{"sourceId":178028437,"sourceType":"kernelVersion"},{"sourceId":45779,"sourceType":"modelInstanceVersion","modelInstanceId":13676}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/kaggle-home-credit-risk-model-stability-lib/kaggle_home_credit_risk_model_stability-0.3-py3-none-any.whl --force-reinstall","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:17:21.959605Z","iopub.execute_input":"2024-05-15T07:17:21.960035Z","iopub.status.idle":"2024-05-15T07:17:55.773482Z","shell.execute_reply.started":"2024-05-15T07:17:21.960002Z","shell.execute_reply":"2024-05-15T07:17:55.772260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import packages","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport pickle\nimport gc\nfrom kaggle_home_credit_risk_model_stability.libs.lightgbm.kfold_model import *\nfrom kaggle_home_credit_risk_model_stability.libs.catboost.kfold_model import *\nfrom kaggle_home_credit_risk_model_stability.libs.model import WeightVotingModel","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:17:55.775493Z","iopub.execute_input":"2024-05-15T07:17:55.775961Z","iopub.status.idle":"2024-05-15T07:17:58.717111Z","shell.execute_reply.started":"2024-05-15T07:17:55.775919Z","shell.execute_reply":"2024-05-15T07:17:58.715718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare input","metadata":{}},{"cell_type":"code","source":"!cp /kaggle/input/homecreditinput/__notebook__.ipynb homecreditinput.ipynb\n!jupyter nbconvert --to python homecreditinput.ipynb\n!ipython homecreditinput.py\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-12T10:49:29.647732Z","iopub.status.idle":"2024-05-12T10:49:29.648622Z","shell.execute_reply.started":"2024-05-12T10:49:29.648318Z","shell.execute_reply":"2024-05-12T10:49:29.648344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Dataset","metadata":{}},{"cell_type":"code","source":"test_df = pl.read_parquet(\"/kaggle/working/test_df.parquet\")\ntest_df","metadata":{"execution":{"iopub.status.busy":"2024-05-10T21:34:54.635686Z","iopub.execute_input":"2024-05-10T21:34:54.636492Z","iopub.status.idle":"2024-05-10T21:34:54.642266Z","shell.execute_reply.started":"2024-05-10T21:34:54.636456Z","shell.execute_reply":"2024-05-10T21:34:54.640767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = pl.read_parquet(\"/kaggle/input/homecreditinput/train_df.parquet\", n_rows=5000)\n# test_df # TODO comment","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:17:58.718822Z","iopub.execute_input":"2024-05-15T07:17:58.719427Z","iopub.status.idle":"2024-05-15T07:18:04.865460Z","shell.execute_reply.started":"2024-05-15T07:17:58.719395Z","shell.execute_reply":"2024-05-15T07:18:04.863841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Model","metadata":{}},{"cell_type":"code","source":"model = pickle.load(open(\"/kaggle/input/homecreditsinglelightgbmmodel/other/300_features/73/weight_voting_model.pkl\", \"rb\"))","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:18:04.868324Z","iopub.execute_input":"2024-05-15T07:18:04.868769Z","iopub.status.idle":"2024-05-15T07:18:14.836584Z","shell.execute_reply.started":"2024-05-15T07:18:04.868732Z","shell.execute_reply":"2024-05-15T07:18:14.835260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(model.model.estimators[0].model.feature_name())","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:18:14.838223Z","iopub.execute_input":"2024-05-15T07:18:14.838703Z","iopub.status.idle":"2024-05-15T07:18:14.845840Z","shell.execute_reply.started":"2024-05-15T07:18:14.838662Z","shell.execute_reply":"2024-05-15T07:18:14.844210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(model.get_train_data())\n#print(len(model.model.estimators[0].feature_name()))","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:18:14.847590Z","iopub.execute_input":"2024-05-15T07:18:14.847967Z","iopub.status.idle":"2024-05-15T07:18:14.856760Z","shell.execute_reply.started":"2024-05-15T07:18:14.847938Z","shell.execute_reply":"2024-05-15T07:18:14.855248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"Y_predicted = model.predict_chunked(test_df, chunk_size=100000)\nY_predicted","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:18:14.858727Z","iopub.execute_input":"2024-05-15T07:18:14.859433Z","iopub.status.idle":"2024-05-15T07:18:46.013316Z","shell.execute_reply.started":"2024-05-15T07:18:14.859385Z","shell.execute_reply":"2024-05-15T07:18:46.012398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_dataframe = pd.DataFrame({\n    \"case_id\": test_df[\"case_id\"],\n    \"score\": Y_predicted\n}).set_index('case_id')\n\nscore_dataframe.to_csv(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:18:46.014553Z","iopub.execute_input":"2024-05-15T07:18:46.014845Z","iopub.status.idle":"2024-05-15T07:18:46.101488Z","shell.execute_reply.started":"2024-05-15T07:18:46.014822Z","shell.execute_reply":"2024-05-15T07:18:46.100412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%reset -f","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:18:46.103500Z","iopub.execute_input":"2024-05-15T07:18:46.103852Z","iopub.status.idle":"2024-05-15T07:18:46.474726Z","shell.execute_reply.started":"2024-05-15T07:18:46.103826Z","shell.execute_reply":"2024-05-15T07:18:46.473429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nfrom kaggle_home_credit_risk_model_stability.libs.date_decision_restorer.date_decision_restorer_by_dpdmaxdate import DateDecisionRestorerByDpDMaxDate\nfrom kaggle_home_credit_risk_model_stability.libs.date_decision_restorer.date_decision_restorer_by_overdue_amount import DateDecisionRestorerByOverdueAmount\nfrom kaggle_home_credit_risk_model_stability.libs.date_decision_restorer.date_decision_restorer_by_pmts_overdue import DateDecisionRestorerByPmtOverdue\nfrom kaggle_home_credit_risk_model_stability.libs.input.table_loader import TableLoader\nfrom kaggle_home_credit_risk_model_stability.libs.env import Env","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:22:22.428229Z","iopub.execute_input":"2024-05-15T07:22:22.428717Z","iopub.status.idle":"2024-05-15T07:22:22.437647Z","shell.execute_reply.started":"2024-05-15T07:22:22.428686Z","shell.execute_reply":"2024-05-15T07:22:22.436333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = Env(\n    \"/kaggle/input/\",\n    \"/kaggle/working/\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:22:24.214946Z","iopub.execute_input":"2024-05-15T07:22:24.215441Z","iopub.status.idle":"2024-05-15T07:22:24.221211Z","shell.execute_reply.started":"2024-05-15T07:22:24.215407Z","shell.execute_reply":"2024-05-15T07:22:24.220070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table_loader = TableLoader(env)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:22:24.768520Z","iopub.execute_input":"2024-05-15T07:22:24.769837Z","iopub.status.idle":"2024-05-15T07:22:24.775013Z","shell.execute_reply.started":"2024-05-15T07:22:24.769795Z","shell.execute_reply":"2024-05-15T07:22:24.773976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"restorer = DateDecisionRestorerByOverdueAmount(env)\nreal_date_decision_table = restorer.restore()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:23:29.935525Z","iopub.execute_input":"2024-05-15T07:23:29.936349Z","iopub.status.idle":"2024-05-15T07:26:08.779837Z","shell.execute_reply.started":"2024-05-15T07:23:29.936312Z","shell.execute_reply":"2024-05-15T07:26:08.778746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_date_decision_table = real_date_decision_table.with_columns(\n    ((pl.col(\"date_decision\") - 5) // 7).alias(\"WEEK_NUM\")\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:26:08.781673Z","iopub.execute_input":"2024-05-15T07:26:08.782212Z","iopub.status.idle":"2024-05-15T07:26:08.804689Z","shell.execute_reply.started":"2024-05-15T07:26:08.782183Z","shell.execute_reply":"2024-05-15T07:26:08.803743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"week_num_threashold = real_date_decision_table[\"WEEK_NUM\"].quantile(0.4)\nweek_num_set = real_date_decision_table.filter(pl.col(\"WEEK_NUM\") < week_num_threashold)[\"WEEK_NUM\"]","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:26:08.805972Z","iopub.execute_input":"2024-05-15T07:26:08.806678Z","iopub.status.idle":"2024-05-15T07:26:08.856133Z","shell.execute_reply.started":"2024-05-15T07:26:08.806579Z","shell.execute_reply":"2024-05-15T07:26:08.854701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"week_num_set","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:26:08.858987Z","iopub.execute_input":"2024-05-15T07:26:08.859319Z","iopub.status.idle":"2024-05-15T07:26:08.867082Z","shell.execute_reply.started":"2024-05-15T07:26:08.859294Z","shell.execute_reply":"2024-05-15T07:26:08.865707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pl.read_csv(\"./submission.csv\")\n\nmax_shift = 0.057\n\nN = week_num_set.n_unique()\nfor i in range(N):\n    l = week_num_set.quantile(i / N)\n    r = week_num_set.quantile((i + 1) / N)\n    shift = max_shift * (1 - i / N)\n    \n    case_ids = real_date_decision_table.filter(\n        (l <= pl.col(\"WEEK_NUM\")) & (pl.col(\"WEEK_NUM\") < r)\n    )[\"case_id\"]\n    print(l, r, len(case_ids), shift)\n\n    submission = submission.with_columns(\n        (pl.when(pl.col(\"case_id\").is_in(case_ids)).then(pl.col(\"score\") - shift).otherwise(pl.col(\"score\"))).alias(\"score\")\n    )\n    \nsubmission = submission.with_columns(\n    (pl.when(pl.col(\"score\") < 0).then(0).otherwise(pl.col(\"score\"))).alias(\"score\")\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:26:08.868693Z","iopub.execute_input":"2024-05-15T07:26:08.869006Z","iopub.status.idle":"2024-05-15T07:26:17.657718Z","shell.execute_reply.started":"2024-05-15T07:26:08.868983Z","shell.execute_reply":"2024-05-15T07:26:17.656394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:26:17.659798Z","iopub.execute_input":"2024-05-15T07:26:17.660159Z","iopub.status.idle":"2024-05-15T07:26:17.668126Z","shell.execute_reply.started":"2024-05-15T07:26:17.660132Z","shell.execute_reply":"2024-05-15T07:26:17.667339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_dataframe = pd.DataFrame({\n    \"case_id\": submission[\"case_id\"].to_numpy(),\n    \"score\": submission[\"score\"].to_numpy()\n}).set_index('case_id')\n\nscore_dataframe.to_csv(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-15T07:26:17.669356Z","iopub.execute_input":"2024-05-15T07:26:17.670162Z","iopub.status.idle":"2024-05-15T07:26:17.691774Z","shell.execute_reply.started":"2024-05-15T07:26:17.670129Z","shell.execute_reply":"2024-05-15T07:26:17.690393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}