{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8047956,"sourceType":"datasetVersion","datasetId":4745753},{"sourceId":8508092,"sourceType":"datasetVersion","datasetId":4805046},{"sourceId":162470947,"sourceType":"kernelVersion"},{"sourceId":169863671,"sourceType":"kernelVersion"}],"dockerImageVersionId":30674,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":196.226411,"end_time":"2024-02-19T08:01:03.0917","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-02-19T07:57:46.865289","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Dependencies","metadata":{"papermill":{"duration":0.01447,"end_time":"2024-02-19T07:57:49.64935","exception":false,"start_time":"2024-02-19T07:57:49.63488","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install --no-index -U --find-links=/kaggle/input/lightautoml-038-dependencies lightautoml==0.3.8","metadata":{"_kg_hide-output":true,"papermill":{"duration":141.486992,"end_time":"2024-02-19T08:00:11.149939","exception":false,"start_time":"2024-02-19T07:57:49.662947","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:20:35.608262Z","iopub.execute_input":"2024-05-24T16:20:35.608676Z","iopub.status.idle":"2024-05-24T16:23:44.601145Z","shell.execute_reply.started":"2024-05-24T16:20:35.608641Z","shell.execute_reply":"2024-05-24T16:23:44.598965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport joblib\nimport lightgbm as lgb\nimport torch\nimport torch.nn as nn\nimport shutil\nimport glob\nimport random\nimport sys\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom category_encoders import CatBoostEncoder\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":7.927981,"end_time":"2024-02-19T08:00:19.098809","exception":false,"start_time":"2024-02-19T08:00:11.170828","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:23:44.605830Z","iopub.execute_input":"2024-05-24T16:23:44.606454Z","iopub.status.idle":"2024-05-24T16:23:52.499482Z","shell.execute_reply.started":"2024-05-24T16:23:44.606398Z","shell.execute_reply":"2024-05-24T16:23:52.498294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightautoml.automl.presets.tabular_presets import TabularAutoML\nfrom lightautoml.tasks import Task\nfrom sklearn.metrics import mean_squared_error","metadata":{"papermill":{"duration":25.755103,"end_time":"2024-02-19T08:00:44.874692","exception":false,"start_time":"2024-02-19T08:00:19.119589","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:23:52.501627Z","iopub.execute_input":"2024-05-24T16:23:52.502410Z","iopub.status.idle":"2024-05-24T16:24:39.943006Z","shell.execute_reply.started":"2024-05-24T16:23:52.502348Z","shell.execute_reply":"2024-05-24T16:24:39.941096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.insert(0, '/kaggle/input/fork-3-of-home-credit-baseline-data-ds/')","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:39.946108Z","iopub.execute_input":"2024-05-24T16:24:39.946944Z","iopub.status.idle":"2024-05-24T16:24:39.953058Z","shell.execute_reply.started":"2024-05-24T16:24:39.946902Z","shell.execute_reply":"2024-05-24T16:24:39.951676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import fork_of_home_credit_baseline_data as data_nb\n# import fork_5_of_home_credit_baseline_data as data_nb\nimport fork_3_of_home_credit_baseline_data as data_nb","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:39.954793Z","iopub.execute_input":"2024-05-24T16:24:39.955214Z","iopub.status.idle":"2024-05-24T16:24:39.992932Z","shell.execute_reply.started":"2024-05-24T16:24:39.955170Z","shell.execute_reply":"2024-05-24T16:24:39.991669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    load_model=False\n#     credit_b_a_1_cols_num=100\n    credit_b_a_1_cols_num=20\n    credit_b_a_1_cols_shift=50\n    debug=False\n    debug_subsample=0.15\n    model_path=Path('/kaggle/input/lama-cv-metric-home-credit-training-ds')\n    data_path=Path(\"/kaggle/input/fork-3-of-home-credit-baseline-data-ds/train_base.parquet\")\n    data_path_2=Path(\"/kaggle/input/fork-3-of-home-credit-baseline-data-ds/credit_bureau_a_1_train_df.parquet\")\n    drop_cols_endwith=[]\n    drop_cols_exclude = [\n        \"last_last_conts_type_509L\", \n        \"first_last_conts_type_509L\", \n        \"first_first_credacc_cards_status_52L\",\n        \"last_first_credacc_cards_status_52L\",\n        \"cnt_requesttype_4525192L\",\n    ]\n    drop_cols_contains = [\n        \"conts_type_509L\", \n        \"_credacc_cards_status_52L\",\n        \"empls_employer_name_740M\"\n    ]\n    isnull_threshold=1","metadata":{"papermill":{"duration":0.030805,"end_time":"2024-02-19T08:00:44.92767","exception":false,"start_time":"2024-02-19T08:00:44.896865","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:39.995125Z","iopub.execute_input":"2024-05-24T16:24:39.995899Z","iopub.status.idle":"2024-05-24T16:24:40.005546Z","shell.execute_reply.started":"2024-05-24T16:24:39.995840Z","shell.execute_reply":"2024-05-24T16:24:40.003986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_THREADS = 4\n# N_FOLDS = 5\nN_FOLDS = None\nRANDOM_STATE = 2024\nTIMEOUT = 30000\nADVANCED_ROLES = False\n# USE_QNT = True\nUSE_QNT = True\nUSE_PLR = False\nTRAIN_BS = 128\nEPOCHS = 2 if CFG.debug else 10\nTARGET_NAME = 'target'\n# MEMORY = 14\n\nnp.random.seed(RANDOM_STATE)\ntorch.set_num_threads(N_THREADS)","metadata":{"papermill":{"duration":0.029468,"end_time":"2024-02-19T08:00:46.077274","exception":false,"start_time":"2024-02-19T08:00:46.047806","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.006987Z","iopub.execute_input":"2024-05-24T16:24:40.007409Z","iopub.status.idle":"2024-05-24T16:24:40.026829Z","shell.execute_reply.started":"2024-05-24T16:24:40.007369Z","shell.execute_reply":"2024-05-24T16:24:40.025202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data collection","metadata":{"papermill":{"duration":0.019582,"end_time":"2024-02-19T08:00:44.96817","exception":false,"start_time":"2024-02-19T08:00:44.948588","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_drop_cols(df, drop_cols_endwith=[], drop_cols_contains=[], drop_cols_exclude=[], isnull_threshold=1):\n    drop_cols = []\n    for name_prefix in drop_cols_endwith:\n        cols_names = df.columns[df.columns.str.endswith(name_prefix)]\n        drop_cols += cols_names.to_list()\n    for name_prefix in drop_cols_contains:\n        cols_names = df.columns[df.columns.str.contains(name_prefix)]\n        drop_cols += cols_names.to_list()\n    for col_name in drop_cols_exclude:\n        if col_name in drop_cols:\n            drop_cols.remove(col_name)\n        \n    if isnull_threshold < 1:\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].isnull().mean()\n                if isnull > isnull_threshold:\n                    drop_cols.append(col)\n    return drop_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.029325Z","iopub.execute_input":"2024-05-24T16:24:40.030501Z","iopub.status.idle":"2024-05-24T16:24:40.040472Z","shell.execute_reply.started":"2024-05-24T16:24:40.030456Z","shell.execute_reply":"2024-05-24T16:24:40.039324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Base Data","metadata":{"papermill":{"duration":0.019786,"end_time":"2024-02-19T08:00:45.114478","exception":false,"start_time":"2024-02-19T08:00:45.094692","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not CFG.load_model:\n    train_base_df = pd.read_parquet(CFG.data_path)\n    base_cat_cols = list(train_base_df.select_dtypes(\"object\").columns)\n    train_base_df[base_cat_cols] = train_base_df[base_cat_cols].astype(\"category\")\n    base_cat_cols = list(train_base_df.select_dtypes(\"category\").columns)\n    display(train_base_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.028083,"end_time":"2024-02-19T08:00:45.16288","exception":false,"start_time":"2024-02-19T08:00:45.134797","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.041962Z","iopub.execute_input":"2024-05-24T16:24:40.043103Z","iopub.status.idle":"2024-05-24T16:24:40.054180Z","shell.execute_reply.started":"2024-05-24T16:24:40.043060Z","shell.execute_reply":"2024-05-24T16:24:40.052638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    display(base_cat_cols)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.026757,"end_time":"2024-02-19T08:00:45.209531","exception":false,"start_time":"2024-02-19T08:00:45.182774","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.059645Z","iopub.execute_input":"2024-05-24T16:24:40.060447Z","iopub.status.idle":"2024-05-24T16:24:40.072181Z","shell.execute_reply.started":"2024-05-24T16:24:40.060405Z","shell.execute_reply":"2024-05-24T16:24:40.070311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    train_drop_cols = get_drop_cols(train_base_df, CFG.drop_cols_endwith, CFG.drop_cols_contains, CFG.drop_cols_exclude)\n    print(\"train_drop_cols len:\", len(train_drop_cols))\n    print(\"train_drop_cols:\", train_drop_cols)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.074049Z","iopub.execute_input":"2024-05-24T16:24:40.074896Z","iopub.status.idle":"2024-05-24T16:24:40.081692Z","shell.execute_reply.started":"2024-05-24T16:24:40.074854Z","shell.execute_reply":"2024-05-24T16:24:40.080238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    train_base_df = train_base_df.drop(columns=train_drop_cols)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.083674Z","iopub.execute_input":"2024-05-24T16:24:40.084591Z","iopub.status.idle":"2024-05-24T16:24:40.091786Z","shell.execute_reply.started":"2024-05-24T16:24:40.084552Z","shell.execute_reply":"2024-05-24T16:24:40.090668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data-f3-v16\ntop_base_feats = [\n    \"min_cancelreason_3545846M\",\n    \"price_1097A\",\n    \"annuity_780A\",\n    \"pmtnum_254L\",\n    \"lastapprcommoditycat_1041M\",\n    \"lastcancelreason_561M\",\n    \"credamount_770A\",\n    \"dateofbirth_337D\",\n    \"eir_270L\",\n    \"disbursedcredamount_1113A\",\n    \"first_cancelreason_3545846M\",\n    \"interestrate_311L\",\n    \"lastrejectcommoditycat_161M\",\n    \"max_employedfrom_700D\",\n    \"lastdelinqdate_224D\",\n    \"lastrejectdate_50D\",\n    \"days360_512L\",\n    \"numberofqueries_373L\",\n    \"pctinstlsallpaidlate1d_3546856L\",\n    \"max_dtlastpmt_581D\",\n    \"maxdpdinstldate_3546855D\",\n    \"last_annuity_853A\",\n    \"thirdquarter_1082L\",\n    \"pmtssum_45A\",\n    \"mobilephncnt_593L\",\n    \"numincomingpmts_3546848L\",\n    \"mean_amount_4527230A\",\n    \"max_birth_259D\",\n    \"mean_employedfrom_700D\",\n    \"firstclxcampaign_1125D\",\n    \"max_amount_4527230A\",\n    \"mean_birth_259D\",\n    \"last_credamount_590A\",\n    \"maxinstallast24m_3658928A\",\n    \"first_birth_259D\",\n    \"lastrejectcredamount_222A\",\n    \"cntpmts24_3658933L\",\n    \"maxannuity_159A\",\n    \"last_amount_4527230A\",\n    \"days180_256L\",\n    \"last_employedfrom_700D\",\n    \"min_employedfrom_700D\",\n    \"fourthquarter_440L\",\n    \"first_firstnonzeroinstldate_307D\",\n    \"mean_annuity_853A\",\n    \"days90_310L\",\n    \"mean_mainoccupationinc_437A\",\n    \"datelastunpaid_3546854D\",\n    \"pctinstlsallpaidearl3d_427L\",\n    \"first_mainoccupationinc_437A\",\n    \"min_birth_259D\",\n    \"min_relationshiptoclient_415T\",\n    \"secondquarter_766L\",\n    \"maxdbddpdlast1m_3658939P\",\n    \"max_dtlastpmtallstes_3545839D\",\n    \"mean_maxdpdtolerance_577P\",\n    \"days120_123L\",\n    \"applicationscnt_867L\",\n    \"riskassesment_302T\",\n    \"max_annuity_853A\",\n    \"firstquarter_103L\",\n    \"cntincpaycont9m_3716944L\",\n    \"maxdbddpdtollast12m_3658940P\",\n    \"month_decision\",\n    \"last_birth_259D\",\n    \"mindbddpdlast24m_3658935P\",\n    \"numinstlswithdpd10_728L\",\n    \"maininc_215A\",\n    \"avgdbddpdlast24m_3658932P\",\n    \"min_relationshiptoclient_642T\",\n    \"amtinstpaidbefduel24m_4187115A\",\n    \"first_annuity_853A\",\n    \"validfrom_1069D\",\n    \"datelastinstal40dpd_247D\",\n    \"first_credamount_590A\",\n    \"mean_last_numgroup2_applprev_2_A\",\n    \"lastapprcredamount_781A\",\n    \"min_credamount_590A\",\n    \"min_annuity_853A\",\n    \"min_amount_4527230A\",\n    \"avgdbddpdlast3m_4187120P\",\n    \"mean_dtlastpmt_581D\",\n    \"avginstallast24m_3658937A\",\n    \"max_relationshiptoclient_642T\",\n    \"min_mainoccupationinc_437A\",\n    \"max_tenor_203L\",\n    \"inittransactionamount_650A\",\n    \"mean_max_numgroup2_applprev_2_A\",\n    \"max_relationshiptoclient_415T\",\n    \"datefirstoffer_1144D\",\n    \"min_rejectreason_755M\",\n    \"max_numgroup1_tax_registry_a_1\",\n    \"max_firstnonzeroinstldate_307D\",\n    \"mean_credamount_590A\",\n    \"daysoverduetolerancedd_3976961L\",\n    \"avgoutstandbalancel6m_4187114A\",\n    \"last_mainoccupationinc_437A\",\n    \"mean_dtlastpmtallstes_3545839D\",\n    \"first_amount_4527230A\",\n    \"max_mainoccupationinc_437A\",\n    \"maxdpdinstlnum_3546846P\",\n    \"last_numgroup1_tax_registry_a_1\",\n    \"maxdebt4_972A\",\n    \"last_firstnonzeroinstldate_307D\",\n    \"totalsettled_863A\",\n    \"max_credamount_590A\",\n    \"pctinstlsallpaidlat10d_839L\",\n    \"weekday_decision\",\n    \"pmtaverage_4527227A\",\n    \"numinstlswithoutdpd_562L\",\n    \"days30_165L\",\n    \"min_dtlastpmtallstes_3545839D\",\n    \"last_mainoccupationinc_384A\",\n    \"maxdpdtolerance_374P\",\n    \"monthsannuity_845L\",\n    \"min_dtlastpmt_581D\",\n    \"dtlastpmtallstes_4499206D\",\n    \"max_pmtnum_8L\",\n    \"numinstlsallpaid_934L\",\n    \"pctinstlsallpaidlate6d_3546844L\",\n    \"avgdpdtolclosure24_3658938P\",\n    \"max_maxdpdtolerance_577P\",\n    \"first_mainoccupationinc_384A\",\n    \"min_mainoccupationinc_384A\",\n    \"mean_firstnonzeroinstldate_307D\",\n    \"mean_outstandingdebt_522A\",\n    \"maxoutstandbalancel12m_4187113A\",\n    \"first_pmtnum_8L\",\n    \"min_amount_4917619A\",\n    \"max_empl_employedfrom_271D\",\n    \"mean_empl_employedfrom_271D\",\n    \"first_tenor_203L\",\n    \"min_pmtamount_36A\",\n    \"max_mainoccupationinc_384A\",\n    \"numinstregularpaid_973L\",\n    \"pctinstlsallpaidlate4d_3546849L\",\n    \"numinsttopaygr_769L\",\n    \"pmtaverage_3A\",\n    \"birthdate_574D\",\n    \"maxdbddpdtollast6m_4187119P\",\n    \"last_relationshiptoclient_642T\",\n    \"first_empl_employedfrom_271D\",\n    \"min_empl_employedfrom_271D\",\n    \"downpmt_116A\",\n    \"numinstpaidlate1d_3546852L\",\n    \"education_1103M\",\n    \"last_approvaldate_319D\",\n    \"mean_mainoccupationinc_384A\",\n    \"max_incometype_1044T\",\n    \"numinstlallpaidearly3d_817L\",\n    \"numinstpaidearly3d_3546850L\",\n    \"avgpmtlast12m_4525200A\",\n    \"last_pmtnum_8L\",\n    \"mindbdtollast24m_4525191P\",\n    \"numinstpaidlastcontr_4325080L\",\n    \"numinstunpaidmax_3546851L\",\n    \"last_dateactivated_425D\",\n    \"mean_pmtamount_36A\",\n    \"first_pmtamount_36A\",\n    \"contractssum_5085716L\",\n    \"totinstallast1m_4525188A\",\n    \"max_approvaldate_319D\",\n    \"maxdpdfrom6mto36m_3546853P\",\n    \"maxdpdlast24m_143P\",\n    \"min_incometype_1044T\",\n    \"last_pmtamount_36A\",\n    \"avgdbdtollast24m_4525197P\",\n    \"lastactivateddate_801D\",\n    \"first_dateactivated_425D\",\n    \"mean_dateactivated_425D\",\n    \"mean_currdebt_94A\",\n    \"first_dtlastpmtallstes_3545839D\",\n    \"lastapplicationdate_877D\",\n    \"last_numgroup1_applprev_2\",\n    \"numinstlswithdpd5_4187116L\",\n    \"first_approvaldate_319D\",\n    \"mean_approvaldate_319D\",\n    \"max_pmtamount_36A\",\n    \"max_creationdate_885D\",\n    \"last_tenor_203L\",\n    \"lastrejectreason_759M\",\n    \"first_incometype_1044T\",\n    \"max_numgroup1_applprev_2\",\n    \"maxlnamtstart6m_4525199A\",\n    \"min_dateactivated_425D\",\n    \"numinstmatpaidtearly2d_4499204L\",\n    \"min_pmtnum_8L\",\n    \"first_empl_industry_691L\",\n    \"min_firstnonzeroinstldate_307D\",\n    \"maxpmtlast3m_4525190A\",\n    \"sellerplacescnt_216L\",\n    \"numinstpaidearly_338L\",\n    \"max_dateactivated_425D\",\n    \"mean_downpmt_134A\",\n    \"first_maxdpdtolerance_577P\",\n    \"first_creationdate_885D\",\n    \"lastapprdate_640D\",\n    \"min_tenor_203L\",\n    \"firstdatedue_489D\",\n    \"mean_creationdate_885D\",\n    \"max_downpmt_134A\",\n    \"min_familystate_726L\",\n    \"numinstregularpaidest_4493210L\",\n    \"numinstpaidearly3dest_4493216L\",\n    \"max_byoccupationinc_3656910L\",\n    \"min_processingdate_168D\",\n    \"numinstpaidearly5dobd_4499205L\",\n    \"pmtscount_423L\",\n    \"max_empl_industry_691L\",\n    \"min_credor_3940957M\",\n    \"min_approvaldate_319D\",\n    \"min_empl_industry_691L\",\n    \"numinstpaidearlyest_4493214L\",\n    \"mean_processingdate_168D\",\n    \"min_creationdate_885D\",\n    \"avgmaxdpdlast9m_3716943P\",\n    \"pmtaverage_4955615A\",\n    \"currdebt_22A\",\n    \"last_dtlastpmtallstes_3545839D\",\n    \"maxdpdlast12m_727P\",\n    \"totaldebt_9A\",\n    \"maritalst_385M\",\n    \"mean_amount_4917619A\",\n    \"numinstpaid_4499208L\",\n    \"max_childnum_21L\",\n    \"numinstunpaidmaxest_4493212L\",\n    \"last_maxdpdtolerance_577P\",\n    \"first_employedfrom_700D\",\n    \"first_processingdate_168D\",\n    \"last_dtlastpmt_581D\",\n    \"min_education_1138M\",\n    \"sumoutstandtotal_3546847A\",\n    \"last_creationdate_885D\",\n    \"max_currdebt_94A\",\n    \"max_amount_4917619A\",\n    \"maxdpdlast3m_392P\",\n    \"last_processingdate_168D\",\n    \"last_credor_3940957M\",\n    \"max_sex_738L\",\n    \"isbidproduct_1095L\",\n    \"max_processingdate_168D\",\n    \"avglnamtstart24m_4525187A\",\n    \"max_outstandingdebt_522A\",\n    \"first_currdebt_94A\",\n    \"max_familystate_726L\",\n    \"numinsttopaygrest_4493213L\",\n    \"first_outstandingdebt_522A\",\n    \"maxdpdlast9m_1059P\",\n    \"currdebtcredtyperange_828A\",\n    \"first_language1_981M\",\n    \"mean_credacc_actualbalance_314A\",\n    \"min_language1_981M\",\n    \"annuitynextmonth_57A\",\n    \"min_byoccupationinc_3656910L\",\n    \"max_postype_4733339M\",\n    \"last_education_1138M\",\n    \"last_postype_4733339M\",\n    \"numinstpaidearly5d_1087L\",\n    \"sumoutstandtotalest_4493215A\",\n    \"last_numgroup1_tax_registry_c_1\",\n    \"last_byoccupationinc_3656910L\",\n    \"maxdpdlast6m_474P\",\n    \"max_credacc_actualbalance_314A\",\n    \"last_last_conts_type_509L\",\n    \"last_amount_4917619A\",\n    \"max_numgroup1_tax_registry_c_1\",\n    \"responsedate_4527233D\",\n    \"first_postype_4733339M\",\n    \"lastrejectcommodtypec_5251769M\",\n    \"min_credtype_587L\",\n    \"homephncnt_628L\",\n    \"numrejects9m_859L\",\n    \"last_incometype_1044T\",\n    \"numinstpaidearly5dest_4493211L\",\n    \"min_inittransactioncode_279L\",\n    \"first_amount_4917619A\",\n    \"last_downpmt_134A\",\n    \"min_sex_738L\",\n    \"credtype_322L\",\n    \"clientscnt_887L\",\n    \"first_dtlastpmt_581D\",\n    \"min_maxdpdtolerance_577P\",\n    \"last_relationshiptoclient_415T\",\n    \"mean_deductiondate_4917603D\",\n    \"first_education_927M\",\n    \"last_childnum_21L\",\n    \"first_familystate_447L\",\n    \"min_familystate_447L\",\n    \"min_credacc_minhisbal_90A\",\n    \"first_rejectreason_755M\",\n    \"first_sex_738L\",\n    \"min_education_927M\",\n    \"min_deductiondate_4917603D\",\n    \"last_cancelreason_3545846M\",\n    \"mean_credacc_credlmt_575A\",\n    \"max_familystate_447L\",\n    \"first_deductiondate_4917603D\",\n    \"requesttype_4525192L\",\n    \"disbursementtype_67L\",\n    \"min_revolvingaccount_394A\",\n    \"mean_credacc_minhisbal_90A\",\n    \"inittransactioncode_186L\",\n    \"numinstls_657L\",\n    \"min_credacc_actualbalance_314A\",\n    \"max_credacc_credlmt_575A\",\n    \"last_numgroup1_tax_registry_b_1\",\n    \"riskassesment_940T\",\n    \"last_familystate_726L\",\n    \"first_downpmt_134A\",\n    \"max_language1_981M\",\n    \"min_childnum_21L\",\n    \"maxannuity_4075009A\",\n    \"first_credor_3940957M\",\n    \"max_numgroup1_tax_registry_b_1\",\n    \"max_revolvingaccount_394A\",\n    \"max_credacc_minhisbal_90A\",\n    \"max_credacc_maxhisbal_375A\",\n    \"max_status_219L\",\n    \"min_postype_4733339M\",\n    \"posfstqpd30lastmonth_3976962P\",\n    \"last_language1_981M\",\n    \"mean_credacc_maxhisbal_375A\",\n    \"lastst_736L\",\n    \"assignmentdate_4955616D\",\n    \"max_amount_416A\",\n    \"clientscnt_1022L\",\n    \"first_status_219L\",\n    \"posfpd30lastmonth_3976960P\",\n    \"typesuite_864L\",\n    \"mean_revolvingaccount_394A\",\n    \"max_deductiondate_4917603D\",\n    \"min_rejectreasonclient_4145042M\",\n    \"first_last_conts_type_509L\",\n    \"bankacctype_710L\",\n    \"responsedate_4917613D\",\n    \"last_rejectreason_755M\",\n    \"last_deductiondate_4917603D\",\n    \"last_currdebt_94A\",\n    \"mean_amount_416A\",\n    \"clientscnt12m_3712952L\",\n    \"first_credacc_credlmt_575A\",\n    \"applicationscnt_464L\",\n    \"pmtcount_693L\",\n    \"first_openingdate_313D\",\n    \"min_credacc_maxhisbal_375A\",\n    \"applicationscnt_1086L\",\n    \"first_openingdate_857D\",\n    \"twobodfilling_608L\",\n    \"max_credacc_transactions_402L\",\n    \"cardtype_51L\",\n    \"assignmentdate_238D\",\n    \"last_amount_416A\",\n    \"mean_totalamount_881A\",\n    \"clientscnt_533L\",\n    \"max_openingdate_313D\",\n    \"min_interestrateyearly_538L\",\n    \"first_childnum_21L\",\n    \"last_sex_738L\",\n    \"last_credacc_credlmt_575A\",\n    \"first_inittransactioncode_279L\",\n    \"min_amount_1115A\",\n    \"first_familystate_726L\",\n    \"last_openingdate_313D\",\n    \"pmtcount_4527229L\",\n    \"numcontrs3months_479L\",\n    \"clientscnt_946L\",\n    \"mean_openingdate_857D\",\n    \"last_outstandingdebt_522A\",\n    \"last_numgroup1_person_1\",\n    \"first_revolvingaccount_394A\",\n    \"first_credtype_587L\",\n    \"first_last_numgroup2_applprev_2_A\",\n    \"lastrejectreasonclient_4145040M\",\n    \"max_last_numgroup2_applprev_2_A\",\n    \"min_empl_employedtotal_800L\",\n    \"pmtcount_4955617L\",\n    \"last_max_numgroup2_applprev_2_A\",\n    \"min_currdebt_94A\",\n    \"min_outstandingdebt_522A\",\n    \"first_education_1138M\",\n    \"mean_openingdate_313D\",\n    \"mean_contractdate_551D\",\n    \"min_type_25L\",\n    \"max_credor_3940957M\",\n    \"min_openingdate_313D\",\n    \"responsedate_1012D\",\n    \"last_credacc_actualbalance_314A\",\n    \"max_type_25L\",\n    \"first_type_25L\",\n    \"max_numgroup1_person_1\",\n    \"last_status_219L\",\n    \"actualdpdtolerance_344P\",\n    \"description_5085714M\",\n    \"min_openingdate_857D\",\n    \"max_actualdpd_943P\",\n    \"for3years_504L\",\n    \"first_max_numgroup2_applprev_2_A\",\n    \"first_empl_employedtotal_800L\",\n    \"first_contractst_516M\",\n    \"last_revolvingaccount_394A\",\n    \"last_credtype_587L\",\n    \"mean_mean_pmts_date_1107D\",\n    \"min_min_pmts_date_1107D\",\n    \"last_openingdate_857D\",\n    \"min_first_pmts_date_1107D\",\n    \"min_last_numgroup2_applprev_2_A\",\n    \"min_overdueamountmaxdateyear_432T\",\n    \"last_personindex_1023L\",\n    \"clientscnt6m_3712949L\",\n    \"first_amount_416A\"   \n]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-24T16:24:40.093900Z","iopub.execute_input":"2024-05-24T16:24:40.094853Z","iopub.status.idle":"2024-05-24T16:24:40.132145Z","shell.execute_reply.started":"2024-05-24T16:24:40.094811Z","shell.execute_reply":"2024-05-24T16:24:40.130655Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(top_base_feats)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.133738Z","iopub.execute_input":"2024-05-24T16:24:40.134092Z","iopub.status.idle":"2024-05-24T16:24:40.154523Z","shell.execute_reply.started":"2024-05-24T16:24:40.134062Z","shell.execute_reply":"2024-05-24T16:24:40.152381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    train_base_df = train_base_df[[\"case_id\", \"WEEK_NUM\", \"target\"] + top_base_feats]","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.156219Z","iopub.execute_input":"2024-05-24T16:24:40.156764Z","iopub.status.idle":"2024-05-24T16:24:40.169013Z","shell.execute_reply.started":"2024-05-24T16:24:40.156730Z","shell.execute_reply":"2024-05-24T16:24:40.167736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    train_base_cols = train_base_df.columns\n    base_cat_cols = list(train_base_df.select_dtypes(\"category\").columns)\n    print(\"train_base_cols len:\", len(train_base_cols))","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.026622,"end_time":"2024-02-19T08:00:45.255993","exception":false,"start_time":"2024-02-19T08:00:45.229371","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.170742Z","iopub.execute_input":"2024-05-24T16:24:40.171409Z","iopub.status.idle":"2024-05-24T16:24:40.181640Z","shell.execute_reply.started":"2024-05-24T16:24:40.171356Z","shell.execute_reply":"2024-05-24T16:24:40.180370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif not CFG.load_model:\n    train_base_df = data_nb.reduce_mem_usage(train_base_df, float16_as32=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.183742Z","iopub.execute_input":"2024-05-24T16:24:40.184145Z","iopub.status.idle":"2024-05-24T16:24:40.197500Z","shell.execute_reply.started":"2024-05-24T16:24:40.184113Z","shell.execute_reply":"2024-05-24T16:24:40.196158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Credit bureau Data","metadata":{"papermill":{"duration":0.019427,"end_time":"2024-02-19T08:00:45.2956","exception":false,"start_time":"2024-02-19T08:00:45.276173","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not CFG.load_model:\n    train_credit_bureau_a_1_df = pd.read_parquet(CFG.data_path_2).drop(columns=['WEEK_NUM', 'target'])\n    cat_credit_bureau_a_cols = list(train_credit_bureau_a_1_df.select_dtypes(\"object\").columns)\n    train_credit_bureau_a_1_df[cat_credit_bureau_a_cols] = train_credit_bureau_a_1_df[cat_credit_bureau_a_cols].astype(\"category\")\n    cat_credit_bureau_a_cols = list(train_credit_bureau_a_1_df.select_dtypes(\"category\").columns)\n    all_credit_bureau_a_cols = train_credit_bureau_a_1_df.columns\n    display(train_credit_bureau_a_1_df)","metadata":{"papermill":{"duration":0.028709,"end_time":"2024-02-19T08:00:45.344558","exception":false,"start_time":"2024-02-19T08:00:45.315849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.199121Z","iopub.execute_input":"2024-05-24T16:24:40.199577Z","iopub.status.idle":"2024-05-24T16:24:40.211094Z","shell.execute_reply.started":"2024-05-24T16:24:40.199544Z","shell.execute_reply":"2024-05-24T16:24:40.209465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif not CFG.load_model:\n    train_credit_bureau_a_1_df = data_nb.reduce_mem_usage(train_credit_bureau_a_1_df, float16_as32=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.212698Z","iopub.execute_input":"2024-05-24T16:24:40.213100Z","iopub.status.idle":"2024-05-24T16:24:40.225657Z","shell.execute_reply.started":"2024-05-24T16:24:40.213066Z","shell.execute_reply":"2024-05-24T16:24:40.224303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    display(cat_credit_bureau_a_cols)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.028173,"end_time":"2024-02-19T08:00:45.393261","exception":false,"start_time":"2024-02-19T08:00:45.365088","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.227381Z","iopub.execute_input":"2024-05-24T16:24:40.227941Z","iopub.status.idle":"2024-05-24T16:24:40.238593Z","shell.execute_reply.started":"2024-05-24T16:24:40.227901Z","shell.execute_reply":"2024-05-24T16:24:40.237280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    credit_bureau_a_1_feature_imp_df = joblib.load('/kaggle/input/cred-b-fork-of-home-credit-baseline-training/feature_imp_gain_df.pkl')\n    display(credit_bureau_a_1_feature_imp_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.035303,"end_time":"2024-02-19T08:00:45.448704","exception":false,"start_time":"2024-02-19T08:00:45.413401","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.240310Z","iopub.execute_input":"2024-05-24T16:24:40.240763Z","iopub.status.idle":"2024-05-24T16:24:40.249916Z","shell.execute_reply.started":"2024-05-24T16:24:40.240731Z","shell.execute_reply":"2024-05-24T16:24:40.248669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    all_credit_bureau_a_cols = credit_bureau_a_1_feature_imp_df.index.to_list()[\n        CFG.credit_b_a_1_cols_shift : CFG.credit_b_a_1_cols_num + CFG.credit_b_a_1_cols_shift\n    ]\n    display(all_credit_bureau_a_cols)","metadata":{"papermill":{"duration":0.035647,"end_time":"2024-02-19T08:00:45.557704","exception":false,"start_time":"2024-02-19T08:00:45.522057","status":"completed"},"tags":[],"_kg_hide-output":true,"scrolled":true,"execution":{"iopub.status.busy":"2024-05-24T16:24:40.251485Z","iopub.execute_input":"2024-05-24T16:24:40.251990Z","iopub.status.idle":"2024-05-24T16:24:40.262560Z","shell.execute_reply.started":"2024-05-24T16:24:40.251957Z","shell.execute_reply":"2024-05-24T16:24:40.261309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_credit_bureau_a_cols = [\n    'min_totalamount_6A',\n    'first_dateofcredstart_739D',\n    'max_totalamount_996A',\n    'max_overdueamountmax2_14A',\n    'min_prolongationcount_599L',\n    'mean_overdueamountmax2_14A',\n    'first_contractst_964M',\n    'min_overdueamountmax_35A',\n    'max_financialinstitution_382M',\n    'last_refreshdate_3813885D',\n    'min_annualeffectiverate_199L',\n    'first_credlmt_230A',\n    'first_totalamount_6A',\n    'first_numberofcontrsvalue_358L',\n    'max_prolongationcount_1120L',\n    'max_numberofoverdueinstlmax_1151L',\n    'first_residualamount_856A',\n    'max_dateofrealrepmt_138D',\n    'mean_monthlyinstlamount_674A',\n    'max_prolongationcount_599L'\n]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-24T16:24:40.264303Z","iopub.execute_input":"2024-05-24T16:24:40.264723Z","iopub.status.idle":"2024-05-24T16:24:40.279242Z","shell.execute_reply.started":"2024-05-24T16:24:40.264691Z","shell.execute_reply":"2024-05-24T16:24:40.277761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# top_credit_bureau_a_cols = [\n#     \"min_financialinstitution_591M\",\n#     \"max_financialinstitution_382M\",\n#     \"max_contractst_964M\",\n#     \"mean_dpdmax_757P\",\n#     \"max_totalamount_6A\",\n#     \"max_numberofoverdueinstlmax_1039L\",\n#     \"first_contractst_545M\",\n#     \"max_classificationofcontr_400M\",\n#     \"first_description_351M\",\n#     \"min_classificationofcontr_13M\",\n#     \"min_purposeofcred_874M\",\n#     \"first_purposeofcred_426M\",\n#     \"mean_totalamount_6A\",\n#     \"min_totalamount_6A\",\n#     \"first_dateofcredstart_739D\",\n#     \"max_overdueamountmax2_14A\",\n    \n#     \"min_prolongationcount_599L\",\n#     \"mean_overdueamountmax2_14A\",\n#     \"first_contractst_964M\",\n#     \"min_overdueamountmax_35A\"\n# ]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-24T16:24:40.281055Z","iopub.execute_input":"2024-05-24T16:24:40.281505Z","iopub.status.idle":"2024-05-24T16:24:40.297302Z","shell.execute_reply.started":"2024-05-24T16:24:40.281460Z","shell.execute_reply":"2024-05-24T16:24:40.295534Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not CFG.load_model:\n#     joblib.dump(\n#         (cat_credit_bureau_a_cols, credit_bureau_a_1_top_cols), \n#         \"credit_bureau_a_1_top_columns.pkl\"\n#     )\n# else:\n#     cat_credit_bureau_a_cols, credit_bureau_a_1_top_cols = joblib.load(\n#         \"credit_bureau_a_1_top_columns.pkl\"\n#     )","metadata":{"papermill":{"duration":0.027623,"end_time":"2024-02-19T08:00:45.608548","exception":false,"start_time":"2024-02-19T08:00:45.580925","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.299520Z","iopub.execute_input":"2024-05-24T16:24:40.299991Z","iopub.status.idle":"2024-05-24T16:24:40.309593Z","shell.execute_reply.started":"2024-05-24T16:24:40.299953Z","shell.execute_reply":"2024-05-24T16:24:40.307953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n#     train_df = train_base_df[[\"case_id\", \"WEEK_NUM\", \"target\"] + top_base_feats].merge(\n    train_df = train_base_df.merge(\n        train_credit_bureau_a_1_df[[\"case_id\"] + all_credit_bureau_a_cols], on=\"case_id\", how=\"left\"\n    )\n#     train_df = train_base_df.merge(train_credit_bureau_a_1_df, on=\"case_id\", how=\"left\")\n    cat_cols = list(train_df.select_dtypes(include=[\"category\", \"object\", \"string\", \"boolean\"]).columns)\n    display(train_df)","metadata":{"papermill":{"duration":0.028795,"end_time":"2024-02-19T08:00:45.65745","exception":false,"start_time":"2024-02-19T08:00:45.628655","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.311384Z","iopub.execute_input":"2024-05-24T16:24:40.311807Z","iopub.status.idle":"2024-05-24T16:24:40.325801Z","shell.execute_reply.started":"2024-05-24T16:24:40.311773Z","shell.execute_reply":"2024-05-24T16:24:40.324400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing","metadata":{"papermill":{"duration":0.019644,"end_time":"2024-02-19T08:00:45.69716","exception":false,"start_time":"2024-02-19T08:00:45.677516","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not CFG.load_model:\n    drop_cols = get_drop_cols(train_df, CFG.drop_cols_endwith, CFG.drop_cols_contains, CFG.drop_cols_exclude)\n    print(\"drop_cols len:\", len(drop_cols))\nelse:\n    train_base_cols, base_cat_cols, drop_cols = joblib.load(CFG.model_path / \"train_cat_columns.pkl\")","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.327907Z","iopub.execute_input":"2024-05-24T16:24:40.328424Z","iopub.status.idle":"2024-05-24T16:24:40.357716Z","shell.execute_reply.started":"2024-05-24T16:24:40.328363Z","shell.execute_reply":"2024-05-24T16:24:40.355595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(top_feats)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.369819Z","iopub.execute_input":"2024-05-24T16:24:40.370302Z","iopub.status.idle":"2024-05-24T16:24:40.375459Z","shell.execute_reply.started":"2024-05-24T16:24:40.370267Z","shell.execute_reply":"2024-05-24T16:24:40.374156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    train_df.drop(columns=drop_cols, inplace=True)\n    display(train_df.shape)\n    train_cols = train_df.columns\n# test_df.drop(columns=drop_cols, inplace=True)","metadata":{"papermill":{"duration":0.026663,"end_time":"2024-02-19T08:00:45.799312","exception":false,"start_time":"2024-02-19T08:00:45.772649","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.379307Z","iopub.execute_input":"2024-05-24T16:24:40.379720Z","iopub.status.idle":"2024-05-24T16:24:40.388162Z","shell.execute_reply.started":"2024-05-24T16:24:40.379685Z","shell.execute_reply":"2024-05-24T16:24:40.386951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not CFG.load_model:\n#     train_df = data_nb.reduce_mem_usage(train_df, float16_as32=False)","metadata":{"papermill":{"duration":0.027015,"end_time":"2024-02-19T08:00:45.8462","exception":false,"start_time":"2024-02-19T08:00:45.819185","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.389903Z","iopub.execute_input":"2024-05-24T16:24:40.390383Z","iopub.status.idle":"2024-05-24T16:24:40.403621Z","shell.execute_reply.started":"2024-05-24T16:24:40.390339Z","shell.execute_reply":"2024-05-24T16:24:40.402024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    del train_credit_bureau_a_1_df, train_base_df\n    gc.collect()","metadata":{"papermill":{"duration":0.026284,"end_time":"2024-02-19T08:00:45.892472","exception":false,"start_time":"2024-02-19T08:00:45.866188","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.405127Z","iopub.execute_input":"2024-05-24T16:24:40.405978Z","iopub.status.idle":"2024-05-24T16:24:40.419465Z","shell.execute_reply.started":"2024-05-24T16:24:40.405916Z","shell.execute_reply":"2024-05-24T16:24:40.417752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Additional preprocessing","metadata":{}},{"cell_type":"code","source":"# def normalize_data(data, path_to_scaler='scaler.pkl', force_fit=False, scaler_name='zsco'):   \n#     from sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler, RobustScaler, Normalizer, QuantileTransformer, PowerTransformer\n#     if os.path.isfile(path_to_scaler) and not force_fit:\n#         print(f'  load scaler {path_to_scaler}...')\n#         scaler = joblib.load(path_to_scaler)\n#     else:\n#         print(f\"  fit {scaler_name}...\")\n#         scaler_dict = {\n#             'zsco': StandardScaler(copy=False),\n#             'mima': MinMaxScaler(feature_range=(0, 1.71828), copy=False),\n#             'maxb': MaxAbsScaler(copy=False), \n#             'robu': RobustScaler(copy=False),\n#             'norm': Normalizer(copy=False), \n#             'quan': QuantileTransformer(n_quantiles=100, random_state=0, output_distribution=\"normal\", copy=False),\n#             'powe': PowerTransformer(copy=False)\n#         }\n#         scaler = scaler_dict[scaler_name]\n#         scaler.fit(data)\n#         joblib.dump(scaler, path_to_scaler)\n#     print(\"  scale data...\")\n#     df_np = scaler.transform(data)\n#     return df_np","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.421207Z","iopub.execute_input":"2024-05-24T16:24:40.421770Z","iopub.status.idle":"2024-05-24T16:24:40.434670Z","shell.execute_reply.started":"2024-05-24T16:24:40.421727Z","shell.execute_reply":"2024-05-24T16:24:40.433520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def fill_missing_num(df, num_cols, fillna_value=0):\n#     for i, col in enumerate(num_cols):\n#         if col not in df.columns:\n#             print(f\"{col} not in df columns\")\n#             continue\n#         if i % 20 == 0:\n#             print('.', end='')\n#         df[col] = df[col].fillna(fillna_value)\n#     print()\n#     return df","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.436218Z","iopub.execute_input":"2024-05-24T16:24:40.436648Z","iopub.status.idle":"2024-05-24T16:24:40.452423Z","shell.execute_reply.started":"2024-05-24T16:24:40.436617Z","shell.execute_reply":"2024-05-24T16:24:40.451015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def preprocess_data(df, num_cols, path_to_scaler='scaler.pkl', scaler_name='mima'):\n#     print(\"num_cols len:\", len(num_cols))\n    \n#     print(\"replace inf...\")\n#     df.replace([np.inf, -np.inf], np.nan, inplace=True)\n#     inf_count = np.isinf(df).values.sum() \n#     print(\"inf count:\", inf_count)\n\n#     print(\"fillna num_cols...\")\n#     df = fill_missing_num(df, num_cols)\n\n#     print(\"scale num_cols...\")\n#     df[num_cols] = normalize_data(df[num_cols].to_numpy(), path_to_scaler=path_to_scaler, scaler_name=scaler_name)\n#     gc.collect()\n    \n# #     print(\"process_outliers...\")\n# #     df = process_outliers(df, path_to_outliers)\n#     print(\"log1p transform...\")\n#     for i, col in enumerate(num_cols):\n#         if i % 20 == 0:\n#             print('.', end='')\n#         df[col] = np.log1p(df[col])    \n    \n#     print(\"inf count:\", np.isinf(df).values.sum())    \n#     print(\"isna count:\", df[num_cols].isna().sum().sum())\n#     return df","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.454100Z","iopub.execute_input":"2024-05-24T16:24:40.454733Z","iopub.status.idle":"2024-05-24T16:24:40.465858Z","shell.execute_reply.started":"2024-05-24T16:24:40.454699Z","shell.execute_reply":"2024-05-24T16:24:40.464478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not CFG.load_model:\n#     num_cols = list(train_df.select_dtypes(exclude=\"category\").columns)\n#     joblib.dump(num_cols, \"num_cols.pkl\")\n# #     df = fill_missing_num(train_df, num_cols, 0)\n# else:\n#     num_cols = joblib.load(CFG.model_path / \"num_cols.pkl\")","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.467386Z","iopub.execute_input":"2024-05-24T16:24:40.468504Z","iopub.status.idle":"2024-05-24T16:24:40.486827Z","shell.execute_reply.started":"2024-05-24T16:24:40.468467Z","shell.execute_reply":"2024-05-24T16:24:40.485368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not CFG.load_model:\n#     X = preprocess_data(X, num_cols)\n#     print('Memory usage of dataframe is {:.2f} MB'.format(X.memory_usage().sum() / 1024**2))","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.488625Z","iopub.execute_input":"2024-05-24T16:24:40.489140Z","iopub.status.idle":"2024-05-24T16:24:40.499819Z","shell.execute_reply.started":"2024-05-24T16:24:40.489096Z","shell.execute_reply":"2024-05-24T16:24:40.498269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CatBoostEncoding","metadata":{}},{"cell_type":"code","source":"# def catboost_encoding(df, target_col='', cat_cols=[], path_to_encoder='catboost_encoder.pkl', polars_mode=False, force_fit=False):\n#     if polars_mode:\n#         df = df.to_pandas()\n#     if len(cat_cols) == 0:\n#         cat_cols = list(df.select_dtypes(include=[\"category\", \"object\", \"string\"]).columns)\n#     df[cat_cols] = df[cat_cols].astype(\"string\")\n#     print(\"cat_cols len:\", len(cat_cols))\n    \n#     if os.path.isfile(path_to_encoder) and not force_fit:\n#         print('  load catboost encoder...')\n#         catboost_enc = joblib.load(path_to_encoder)\n#     else:\n#         print('  fit catboost encoder...')\n#         df.sort_values(['WEEK_NUM', 'case_id'], inplace=True)\n#         catboost_enc = CatBoostEncoder(cols=cat_cols, random_state=42) #, sigma=1.0\n#         catboost_enc.fit(df[cat_cols], df[target_col])\n#         joblib.dump(catboost_enc, path_to_encoder)\n        \n#     print(\"  feature_names_in_ len:\", len(catboost_enc.feature_names_in_))\n#     print('  catboost encoder transform...')\n#     df[catboost_enc.feature_names_in_] = catboost_enc.transform(df[catboost_enc.feature_names_in_])\n#     df.rename(columns={col: col+'_catb' for col in cat_cols}, inplace=True)\n#     if polars_mode:\n#         df = pl.from_pandas(df)\n#     return df","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.501538Z","iopub.execute_input":"2024-05-24T16:24:40.501925Z","iopub.status.idle":"2024-05-24T16:24:40.515673Z","shell.execute_reply.started":"2024-05-24T16:24:40.501896Z","shell.execute_reply":"2024-05-24T16:24:40.514014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not CFG.load_model:\n#     train_df = catboost_encoding(\n#         train_df, 'target', cat_cols, path_to_encoder='/kaggle/working/catboost_encoder.pkl'\n#     )\n#     catb_cols = train_df.columns[train_df.columns.str.endswith(\"_catb\")]\n#     display(train_df[catb_cols])","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.517569Z","iopub.execute_input":"2024-05-24T16:24:40.518010Z","iopub.status.idle":"2024-05-24T16:24:40.533676Z","shell.execute_reply.started":"2024-05-24T16:24:40.517972Z","shell.execute_reply":"2024-05-24T16:24:40.532344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## End CatBoostEncoding","metadata":{}},{"cell_type":"code","source":"### from https://www.kaggle.com/code/batprem/home-credit-risk-mode-utility-scripts\n\ndef gini_stability(base, score_col=\"score\", w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", score_col]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", score_col]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[score_col])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std","metadata":{"papermill":{"duration":0.029789,"end_time":"2024-02-19T08:00:46.027995","exception":false,"start_time":"2024-02-19T08:00:45.998206","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.535284Z","iopub.execute_input":"2024-05-24T16:24:40.535675Z","iopub.status.idle":"2024-05-24T16:24:40.548152Z","shell.execute_reply.started":"2024-05-24T16:24:40.535644Z","shell.execute_reply":"2024-05-24T16:24:40.546649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_class(x, task, reader):\n    if task.name == 'multiclass':\n        return reader[x]\n    else:\n        return x\n\nmapped = np.vectorize(map_class)\n\ndef score(task, y_true, y_pred):\n    if task.name == 'binary':\n        return roc_auc_score(y_true, y_pred)\n    elif task.name == 'multiclass':\n        return log_loss(y_true, y_pred)\n    elif task.name == 'reg' or task.name == 'multi:reg':\n        return mean_absolute_error(y_true, y_pred)\n    else:\n        raise 'Task is not correct.'\n        \ndef take_pred_from_task(pred, task):\n    if task.name == 'binary' or task.name == 'reg':\n        return pred[:, 0]\n    elif task.name == 'multiclass' or task.name == 'multi:reg':\n        return pred\n    else:\n        raise 'Task is not correct.'\n        \ndef use_plr(USE_PLR):\n    if USE_PLR:\n        return \"plr\"\n    else:\n        return \"cont\"","metadata":{"papermill":{"duration":0.032568,"end_time":"2024-02-19T08:00:46.134497","exception":false,"start_time":"2024-02-19T08:00:46.101929","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.549997Z","iopub.execute_input":"2024-05-24T16:24:40.550512Z","iopub.status.idle":"2024-05-24T16:24:40.567847Z","shell.execute_reply.started":"2024-05-24T16:24:40.550468Z","shell.execute_reply":"2024-05-24T16:24:40.566025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"task = Task(\n    'binary', \n    loss = 'logloss', \n    metric = 'auc'\n)\nroles = {\n    'target': TARGET_NAME,\n    'group': \"WEEK_NUM\",\n    'drop': ['case_id', \"WEEK_NUM\"],\n}","metadata":{"papermill":{"duration":0.037239,"end_time":"2024-02-19T08:00:46.191812","exception":false,"start_time":"2024-02-19T08:00:46.154573","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.569676Z","iopub.execute_input":"2024-05-24T16:24:40.570722Z","iopub.status.idle":"2024-05-24T16:24:40.593180Z","shell.execute_reply.started":"2024-05-24T16:24:40.570687Z","shell.execute_reply":"2024-05-24T16:24:40.591996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.debug:\n#     train_df = train_df[:int(train_df.shape[0] * CFG.debug_subsample)]\n    train_df = train_df.sample(frac=CFG.debug_subsample, random_state=42)","metadata":{"papermill":{"duration":0.027201,"end_time":"2024-02-19T08:00:46.240081","exception":false,"start_time":"2024-02-19T08:00:46.21288","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:40.595615Z","iopub.execute_input":"2024-05-24T16:24:40.596203Z","iopub.status.idle":"2024-05-24T16:24:40.604009Z","shell.execute_reply.started":"2024-05-24T16:24:40.596056Z","shell.execute_reply":"2024-05-24T16:24:40.602150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not CFG.load_model:\n#     train_df = train_df[~train_df.WEEK_NUM.eq(0)]","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.605938Z","iopub.execute_input":"2024-05-24T16:24:40.606448Z","iopub.status.idle":"2024-05-24T16:24:40.616112Z","shell.execute_reply.started":"2024-05-24T16:24:40.606407Z","shell.execute_reply":"2024-05-24T16:24:40.614901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LMVotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators=[]):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X).data.squeeze() for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n    \n    def save(self, folder):\n        if not os.path.exists(folder):\n            os.makedirs(folder)\n        for i, estimator in enumerate(self.estimators):\n            joblib.dump(estimator, f\"{folder}/estimator_{i}.pkl\")\n            \n    def load_model(self, folder):\n        for file in glob.glob(f\"{folder}/*\"):\n            estimator = joblib.load(file)\n            self.estimators.append(estimator)\n        return self","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.617513Z","iopub.execute_input":"2024-05-24T16:24:40.617974Z","iopub.status.idle":"2024-05-24T16:24:40.632948Z","shell.execute_reply.started":"2024-05-24T16:24:40.617932Z","shell.execute_reply":"2024-05-24T16:24:40.631368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def cycle_list_getter(list_a, start, stop):\n#     if (start <= stop):\n#         return list_a[start:stop]\n#     else:\n#         return list_a[start:] + list_a[:stop]","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.634590Z","iopub.execute_input":"2024-05-24T16:24:40.634981Z","iopub.status.idle":"2024-05-24T16:24:40.650680Z","shell.execute_reply.started":"2024-05-24T16:24:40.634944Z","shell.execute_reply":"2024-05-24T16:24:40.649353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"algo = 'mlp'\nparams = {\n#     \"task\": task, \n    \"timeout\": TIMEOUT,\n    \"cpu_limit\": N_THREADS,\n    \"gpu_ids\": 'all',\n    \"general_params\": {\"use_algos\": [[algo]]}, # ['nn', 'mlp', 'dense', 'denselight', 'resnet', 'snn', 'node', 'autoint', 'fttransformer'] or custom torch model\n    \"nn_params\": {\n        \"n_epochs\": EPOCHS, \n        \"bs\": TRAIN_BS, \n        \"num_workers\": 0, \n#         \"path_to_save\": \"lightautoml\",\n        \"path_to_save\": None,\n        \"freeze_defaults\": True,\n        \"cont_embedder\": use_plr(USE_PLR),\n        #             \"emb_dropout\": 0.1\n        #             'drop_rate': 0.1,\n        \"opt_params\": {'lr': 0.01},\n        'scheduler_params': {'patience': 1, 'factor': 0.1, 'min_lr': 1e-05},\n#         'stop_by_metric': True,\n#         'snap_params': {'k': 3, 'early_stopping': True, 'patience': 5, 'swa': True},\n#         'hidden_size': [512, 512],\n#         'dataset': 'CSRSparseDataset',\n#         'pin_memory': True, # gpu\n    },\n    \"nn_pipeline_params\": {\n        \"use_qnt\": USE_QNT, \n        \"use_te\": False,\n#         \"use_te\": True,\n    },\n    \"reader_params\": {\n        'n_jobs': N_THREADS, \n#         'cv': N_FOLDS, \n        'random_state': RANDOM_STATE, \n        'advanced_roles': ADVANCED_ROLES\n    }\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.652504Z","iopub.execute_input":"2024-05-24T16:24:40.652994Z","iopub.status.idle":"2024-05-24T16:24:40.667389Z","shell.execute_reply.started":"2024-05-24T16:24:40.652954Z","shell.execute_reply":"2024-05-24T16:24:40.665461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model and not CFG.debug:\n    train_df = train_df.sample(frac=1, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.669521Z","iopub.execute_input":"2024-05-24T16:24:40.670134Z","iopub.status.idle":"2024-05-24T16:24:40.687352Z","shell.execute_reply.started":"2024-05-24T16:24:40.670088Z","shell.execute_reply":"2024-05-24T16:24:40.685505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MetricsCallback():\n    def __init__(self, weeks):\n        self.weeks = weeks\n        \n    def __call__(self, y_true, y_pred, sample_weight = None):\n        unique_weeks = np.unique(self.weeks)\n        unique_weeks = np.sort(unique_weeks)\n        ginis = []\n\n        for week in unique_weeks:\n            mask = self.weeks == week\n            y_true_week = y_true[mask]\n            y_pred_week = y_pred[mask]\n            auc_score = roc_auc_score(y_true_week, y_pred_week)\n            ginis.append((auc_score - 0.5) * 2)\n\n        x = np.arange(len(ginis))\n        y = ginis\n        a, b = np.polyfit(x, y, 1)\n        y_hat = a*x + b\n        residuals = y - y_hat\n        res_std = np.std(residuals)\n        avg_gini = np.mean(ginis)\n\n        stab_value = avg_gini + 88.0 * min(0, a) + 0.5 * res_std\n        return stab_value","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.689270Z","iopub.execute_input":"2024-05-24T16:24:40.689732Z","iopub.status.idle":"2024-05-24T16:24:40.705161Z","shell.execute_reply.started":"2024-05-24T16:24:40.689700Z","shell.execute_reply":"2024-05-24T16:24:40.703612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.019535,"end_time":"2024-02-19T08:00:45.978512","exception":false,"start_time":"2024-02-19T08:00:45.958977","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# from lightautoml.report.report_deco import ReportDeco","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.706821Z","iopub.execute_input":"2024-05-24T16:24:40.707234Z","iopub.status.idle":"2024-05-24T16:24:40.723576Z","shell.execute_reply.started":"2024-05-24T16:24:40.707201Z","shell.execute_reply":"2024-05-24T16:24:40.721850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    fitted_models = []\n    oof_pred = np.zeros(train_df.shape[0])\n    cv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n    weeks = train_df[\"WEEK_NUM\"]\n    oof_df = train_df[[\"WEEK_NUM\", \"target\"]].copy()\n    \n    for i, (idx_train, idx_valid) in enumerate(cv.split(train_df, train_df['target'], groups=weeks)):\n        print(\"Fold: \", i)\n        X_train = train_df.iloc[idx_train] #, y.iloc[idx_train]\n        X_valid = train_df.iloc[idx_valid] #, y.iloc[idx_valid]\n        X_valid_week = weeks.iloc[idx_valid]\n#         RD = ReportDeco(output_path = f'tabularAutoML_model_report_{i}')\n        \n        metrics_callback = MetricsCallback(weeks=X_valid_week)\n        task = Task(\n            'binary', \n            loss = 'logloss', \n            metric = metrics_callback,\n            greater_is_better = True\n        )\n        params[\"task\"] = task\n        params[\"reader_params\"]['random_state'] += i\n\n        automl_model = TabularAutoML(**params)\n        val_pred = automl_model.fit_predict(\n            train_data=X_train, \n            valid_data=X_valid, \n            roles=roles,\n            verbose=3           \n        ).data\n        fitted_models.append(automl_model)\n        oof_pred[idx_valid] = val_pred.squeeze()\n        del X_train, X_valid\n        gc.collect()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-05-24T16:24:40.725185Z","iopub.execute_input":"2024-05-24T16:24:40.725655Z","iopub.status.idle":"2024-05-24T16:24:40.739363Z","shell.execute_reply.started":"2024-05-24T16:24:40.725620Z","shell.execute_reply":"2024-05-24T16:24:40.737818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    model = LMVotingModel(fitted_models)\n    oof_df[\"pred_oof\"] = oof_pred","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.741430Z","iopub.execute_input":"2024-05-24T16:24:40.741974Z","iopub.status.idle":"2024-05-24T16:24:40.758215Z","shell.execute_reply.started":"2024-05-24T16:24:40.741924Z","shell.execute_reply":"2024-05-24T16:24:40.756787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    model.save(\"denselight_voting_model\")","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.760524Z","iopub.execute_input":"2024-05-24T16:24:40.760991Z","iopub.status.idle":"2024-05-24T16:24:40.773303Z","shell.execute_reply.started":"2024-05-24T16:24:40.760956Z","shell.execute_reply":"2024-05-24T16:24:40.771840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.load_model:\n    from_directory = CFG.model_path / \"denselight_voting_model\"\n    to_directory = \"/kaggle/working/denselight_voting_model\"\n    shutil.copytree(from_directory, to_directory)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:40.775059Z","iopub.execute_input":"2024-05-24T16:24:40.775581Z","iopub.status.idle":"2024-05-24T16:24:41.786174Z","shell.execute_reply.started":"2024-05-24T16:24:40.775536Z","shell.execute_reply":"2024-05-24T16:24:41.784876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.load_model:\n    train_df = pd.read_parquet(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_base.parquet\")\n#     train_df = train_df[~train_df.WEEK_NUM.eq(0)]\n    train_df = train_df.sample(frac=1, random_state=42)\n    y = train_df[\"target\"]\n    oof_df = train_df[[\"WEEK_NUM\", \"target\"]].copy()\n    model = LMVotingModel().load_model(\"denselight_voting_model\")\n    oof_pred = joblib.load(CFG.model_path / \"denselight_oof_preds.pkl\")\n    base_cat_cols, train_base_cols, cat_credit_bureau_a_cols, all_credit_bureau_a_cols = joblib.load(\n        CFG.model_path / \"train_base_credit_bureau_a_1_top_columns.pkl\"\n    )","metadata":{"papermill":{"duration":1.223745,"end_time":"2024-02-19T08:00:47.534677","exception":false,"start_time":"2024-02-19T08:00:46.310932","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:41.787788Z","iopub.execute_input":"2024-05-24T16:24:41.788156Z","iopub.status.idle":"2024-05-24T16:24:43.976555Z","shell.execute_reply.started":"2024-05-24T16:24:41.788125Z","shell.execute_reply":"2024-05-24T16:24:43.975227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# oof = score(\n#     task,\n#     mapped(train_df[TARGET_NAME].values, task, automl_model.reader.class_mapping),\n#     take_pred_from_task(oof_pred, task)\n# )\n# print(\"CV roc_auc_oof: \", oof)","metadata":{"papermill":{"duration":1.403721,"end_time":"2024-02-19T08:00:48.959349","exception":false,"start_time":"2024-02-19T08:00:47.555628","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:43.978056Z","iopub.execute_input":"2024-05-24T16:24:43.978460Z","iopub.status.idle":"2024-05-24T16:24:43.984855Z","shell.execute_reply.started":"2024-05-24T16:24:43.978426Z","shell.execute_reply":"2024-05-24T16:24:43.983434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.debug:\n    oof_df = train_df[[\"WEEK_NUM\", \"target\"]].copy()\n    oof_df[\"pred_oof\"] = oof_pred #[:oof_df.shape[0]]\n    gini_score = gini_stability(oof_df, score_col=\"pred_oof\")\n    print(\"gini_score:\\t\", gini_score)","metadata":{"papermill":{"duration":0.86813,"end_time":"2024-02-19T08:00:49.848117","exception":false,"start_time":"2024-02-19T08:00:48.979987","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:43.987133Z","iopub.execute_input":"2024-05-24T16:24:43.988758Z","iopub.status.idle":"2024-05-24T16:24:45.079606Z","shell.execute_reply.started":"2024-05-24T16:24:43.988707Z","shell.execute_reply":"2024-05-24T16:24:45.077829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc_oof = roc_auc_score(oof_df[\"target\"], oof_df[\"pred_oof\"])\nprint(\"CV roc_auc_oof: \", roc_auc_oof)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:45.081650Z","iopub.execute_input":"2024-05-24T16:24:45.082130Z","iopub.status.idle":"2024-05-24T16:24:45.786446Z","shell.execute_reply.started":"2024-05-24T16:24:45.082089Z","shell.execute_reply":"2024-05-24T16:24:45.784704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_feature_importance(models):\n#     feature_imp_all = []\n#     for model in models:\n#         fi = model.model.get_feature_scores('fast')\n#         feature_imp_df = pd.DataFrame(\n#             {'Importance': fi['Importance']}, \n#             index=fi['Feature']\n#         )\n#         feature_imp_all.append(feature_imp_df)\n#     feature_imp_all_df = pd.concat(feature_imp_all)\n#     feature_imp_all_df = feature_imp_all_df.groupby(feature_imp_all_df.index).mean()\n#     return feature_imp_all_df","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:45.787953Z","iopub.execute_input":"2024-05-24T16:24:45.788436Z","iopub.status.idle":"2024-05-24T16:24:45.794572Z","shell.execute_reply.started":"2024-05-24T16:24:45.788401Z","shell.execute_reply":"2024-05-24T16:24:45.792713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not CFG.load_model:\n#     feature_imp_df = get_feature_importance(fitted_models)\n#     with pd.option_context('display.max_rows', None, 'display.max_columns', None): \n#         display(feature_imp_df.sort_values(\"Value\", ascending=False))","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:45.796129Z","iopub.execute_input":"2024-05-24T16:24:45.796915Z","iopub.status.idle":"2024-05-24T16:24:45.808960Z","shell.execute_reply.started":"2024-05-24T16:24:45.796882Z","shell.execute_reply":"2024-05-24T16:24:45.806303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    joblib.dump(oof_pred, 'denselight_oof_preds.pkl')","metadata":{"papermill":{"duration":0.027161,"end_time":"2024-02-19T08:00:49.896071","exception":false,"start_time":"2024-02-19T08:00:49.86891","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:45.812038Z","iopub.execute_input":"2024-05-24T16:24:45.813516Z","iopub.status.idle":"2024-05-24T16:24:45.821961Z","shell.execute_reply.started":"2024-05-24T16:24:45.813429Z","shell.execute_reply":"2024-05-24T16:24:45.820363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not CFG.load_model:\n#     joblib.dump(automl_model, 'denselight_model.pkl')","metadata":{"papermill":{"duration":0.028531,"end_time":"2024-02-19T08:00:49.947029","exception":false,"start_time":"2024-02-19T08:00:49.918498","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:45.823724Z","iopub.execute_input":"2024-05-24T16:24:45.824168Z","iopub.status.idle":"2024-05-24T16:24:45.835306Z","shell.execute_reply.started":"2024-05-24T16:24:45.824135Z","shell.execute_reply":"2024-05-24T16:24:45.833731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not CFG.load_model:\n    joblib.dump((train_df.columns, cat_cols, drop_cols), \"train_cat_columns.pkl\")","metadata":{"papermill":{"duration":0.027046,"end_time":"2024-02-19T08:00:49.994328","exception":false,"start_time":"2024-02-19T08:00:49.967282","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:45.837196Z","iopub.execute_input":"2024-05-24T16:24:45.837961Z","iopub.status.idle":"2024-05-24T16:24:45.853821Z","shell.execute_reply.started":"2024-05-24T16:24:45.837899Z","shell.execute_reply":"2024-05-24T16:24:45.852581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del oof_pred\ngc.collect()","metadata":{"papermill":{"duration":0.286356,"end_time":"2024-02-19T08:00:50.301085","exception":false,"start_time":"2024-02-19T08:00:50.014729","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:45.855677Z","iopub.execute_input":"2024-05-24T16:24:45.856084Z","iopub.status.idle":"2024-05-24T16:24:46.230528Z","shell.execute_reply.started":"2024-05-24T16:24:45.856054Z","shell.execute_reply":"2024-05-24T16:24:46.229155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test Data Collection","metadata":{"papermill":{"duration":0.020227,"end_time":"2024-02-19T08:00:50.342163","exception":false,"start_time":"2024-02-19T08:00:50.321936","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not CFG.load_model:\n    joblib.dump(\n        (base_cat_cols, train_base_cols, cat_credit_bureau_a_cols, all_credit_bureau_a_cols), \n        \"train_base_credit_bureau_a_1_top_columns.pkl\"\n    )","metadata":{"papermill":{"duration":0.05213,"end_time":"2024-02-19T08:00:55.809722","exception":false,"start_time":"2024-02-19T08:00:55.757592","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:46.232317Z","iopub.execute_input":"2024-05-24T16:24:46.232759Z","iopub.status.idle":"2024-05-24T16:24:46.243786Z","shell.execute_reply.started":"2024-05-24T16:24:46.232722Z","shell.execute_reply":"2024-05-24T16:24:46.242557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not CFG.load_model:\ncat_credit_bureau_a_cols = np.array(\n    cat_credit_bureau_a_cols\n)[np.in1d(cat_credit_bureau_a_cols, all_credit_bureau_a_cols)]\ndisplay(cat_credit_bureau_a_cols)","metadata":{"papermill":{"duration":0.028036,"end_time":"2024-02-19T08:00:50.390415","exception":false,"start_time":"2024-02-19T08:00:50.362379","status":"completed"},"tags":[],"_kg_hide-output":true,"scrolled":true,"execution":{"iopub.status.busy":"2024-05-24T16:24:46.245526Z","iopub.execute_input":"2024-05-24T16:24:46.245928Z","iopub.status.idle":"2024-05-24T16:24:46.262754Z","shell.execute_reply.started":"2024-05-24T16:24:46.245896Z","shell.execute_reply":"2024-05-24T16:24:46.261168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if \"case_id\" not in list(all_credit_bureau_a_cols):\n    all_credit_bureau_a_cols = list(all_credit_bureau_a_cols) + [\"case_id\" ]","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:46.264521Z","iopub.execute_input":"2024-05-24T16:24:46.265013Z","iopub.status.idle":"2024-05-24T16:24:46.273374Z","shell.execute_reply.started":"2024-05-24T16:24:46.264969Z","shell.execute_reply":"2024-05-24T16:24:46.271709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_a_1_df = data_nb.prepare_df(\n    data_nb.credit_bureau_a_1_files,\n    data_nb.CFG.test_dir,\n    data_nb.credit_b_a_1_agg,\n    mode=\"test\",\n    cat_cols=list(cat_credit_bureau_a_cols),\n    train_cols=all_credit_bureau_a_cols\n)\ndisplay(test_credit_bureau_a_1_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:46.275066Z","iopub.execute_input":"2024-05-24T16:24:46.276513Z","iopub.status.idle":"2024-05-24T16:24:46.985928Z","shell.execute_reply.started":"2024-05-24T16:24:46.276473Z","shell.execute_reply":"2024-05-24T16:24:46.984370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# credit_bureau_a_1_test_df = data_nb.prepare_df(\n#     [],\n#     data_nb.CFG.test_dir,\n#     data_nb.base_agg,\n#     mode=\"test\",\n#     cat_cols=cat_credit_bureau_a_cols,\n#     train_cols=[\"case_id\"] + all_credit_bureau_a_cols,\n#     join_by_groups=data_nb.join_by_groups_cred_b_a,\n# )\n# display(credit_bureau_a_1_test_df)","metadata":{"papermill":{"duration":0.542869,"end_time":"2024-02-19T08:00:50.95357","exception":false,"start_time":"2024-02-19T08:00:50.410701","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:46.987606Z","iopub.execute_input":"2024-05-24T16:24:46.988095Z","iopub.status.idle":"2024-05-24T16:24:46.994619Z","shell.execute_reply.started":"2024-05-24T16:24:46.988051Z","shell.execute_reply":"2024-05-24T16:24:46.992801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_a_1_df = data_nb.reduce_mem_usage(test_credit_bureau_a_1_df, float16_as32=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:46.996590Z","iopub.execute_input":"2024-05-24T16:24:46.997219Z","iopub.status.idle":"2024-05-24T16:24:47.028468Z","shell.execute_reply.started":"2024-05-24T16:24:46.997124Z","shell.execute_reply":"2024-05-24T16:24:47.027313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = data_nb.prepare_df(\n    data_nb.base_files,\n    data_nb.CFG.test_dir,\n    data_nb.base_agg,\n    mode=\"test\", \n    cat_cols=base_cat_cols, \n    train_cols=train_base_cols,\n    join_by_groups=data_nb.join_by_groups,\n)\ndisplay(test_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:47.030642Z","iopub.execute_input":"2024-05-24T16:24:47.031032Z","iopub.status.idle":"2024-05-24T16:24:51.879229Z","shell.execute_reply.started":"2024-05-24T16:24:47.031001Z","shell.execute_reply":"2024-05-24T16:24:51.877741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = data_nb.prepare_df(\n#     data_nb.base_files, \n#     data_nb.CFG.test_dir, \n#     data_nb.base_agg, \n#     mode=\"test\", \n#     cat_cols=base_cat_cols, \n#     train_cols=train_base_cols,\n#     join_by_groups=data_nb.join_by_groups_applprev_cred_b_b,\n# )\n# display(test_df)","metadata":{"papermill":{"duration":3.938698,"end_time":"2024-02-19T08:00:54.915455","exception":false,"start_time":"2024-02-19T08:00:50.976757","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:51.881095Z","iopub.execute_input":"2024-05-24T16:24:51.881578Z","iopub.status.idle":"2024-05-24T16:24:51.888787Z","shell.execute_reply.started":"2024-05-24T16:24:51.881538Z","shell.execute_reply":"2024-05-24T16:24:51.887198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = data_nb.reduce_mem_usage(test_df, float16_as32=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:51.890518Z","iopub.execute_input":"2024-05-24T16:24:51.890965Z","iopub.status.idle":"2024-05-24T16:24:52.153498Z","shell.execute_reply.started":"2024-05-24T16:24:51.890926Z","shell.execute_reply":"2024-05-24T16:24:52.152056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df.merge(\n    test_credit_bureau_a_1_df, \n#     test_credit_bureau_a_1_df[all_credit_bureau_a_cols], \n    on=\"case_id\", how=\"left\")\ndisplay(test_df)","metadata":{"papermill":{"duration":0.678042,"end_time":"2024-02-19T08:00:55.625661","exception":false,"start_time":"2024-02-19T08:00:54.947619","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-05-24T16:24:52.155650Z","iopub.execute_input":"2024-05-24T16:24:52.156149Z","iopub.status.idle":"2024-05-24T16:24:52.843280Z","shell.execute_reply.started":"2024-05-24T16:24:52.156105Z","shell.execute_reply":"2024-05-24T16:24:52.841610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_credit_bureau_a_1_df\ngc.collect()","metadata":{"papermill":{"duration":0.300981,"end_time":"2024-02-19T08:00:56.151507","exception":false,"start_time":"2024-02-19T08:00:55.850526","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:52.845041Z","iopub.execute_input":"2024-05-24T16:24:52.845519Z","iopub.status.idle":"2024-05-24T16:24:53.146730Z","shell.execute_reply.started":"2024-05-24T16:24:52.845480Z","shell.execute_reply":"2024-05-24T16:24:53.145420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Additional preprocessing","metadata":{}},{"cell_type":"code","source":"# test_df = preprocess_data(\n#     test_df, \n#     num_cols, \n#     path_to_scaler='/kaggle/working/scaler.pkl'\n# )","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:53.148618Z","iopub.execute_input":"2024-05-24T16:24:53.149086Z","iopub.status.idle":"2024-05-24T16:24:53.157051Z","shell.execute_reply.started":"2024-05-24T16:24:53.149040Z","shell.execute_reply":"2024-05-24T16:24:53.155832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = fill_missing_num(test_df, num_cols, 0)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:53.158681Z","iopub.execute_input":"2024-05-24T16:24:53.159629Z","iopub.status.idle":"2024-05-24T16:24:53.171157Z","shell.execute_reply.started":"2024-05-24T16:24:53.159589Z","shell.execute_reply":"2024-05-24T16:24:53.169453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = test_df[train_cols]","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:53.172847Z","iopub.execute_input":"2024-05-24T16:24:53.173357Z","iopub.status.idle":"2024-05-24T16:24:53.183189Z","shell.execute_reply.started":"2024-05-24T16:24:53.173319Z","shell.execute_reply":"2024-05-24T16:24:53.181669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if CFG.load_model:\n#     path_to_encoder=CFG.load_folder / 'catboost_encoder.pkl'\n# else:\n#     path_to_encoder='/kaggle/working/catboost_encoder.pkl'\n# test_df = catboost_encoding(test_df, '', cat_cols, path_to_encoder=path_to_encoder)\n# display(test_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:53.184827Z","iopub.execute_input":"2024-05-24T16:24:53.185383Z","iopub.status.idle":"2024-05-24T16:24:53.195593Z","shell.execute_reply.started":"2024-05-24T16:24:53.185225Z","shell.execute_reply":"2024-05-24T16:24:53.193858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{"papermill":{"duration":0.039821,"end_time":"2024-02-19T08:00:56.232329","exception":false,"start_time":"2024-02-19T08:00:56.192508","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def predict_proba_in_batches(model, data, batch_size=30000):\n    num_samples = len(data)\n    num_batches = int(np.ceil(num_samples / batch_size))\n    probabilities = np.zeros((num_samples,))\n\n    for batch_idx in range(num_batches):\n        print(f\"Processing batch: {batch_idx+1}/{num_batches}\")\n        start_idx = batch_idx * batch_size\n        end_idx = min((batch_idx + 1) * batch_size, num_samples)\n        X_batch = data.iloc[start_idx:end_idx]\n        batch_probs = model.predict(X_batch)\n        probabilities[start_idx:end_idx] = batch_probs\n        gc.collect()\n\n    return probabilities","metadata":{"papermill":{"duration":0.049519,"end_time":"2024-02-19T08:00:56.321784","exception":false,"start_time":"2024-02-19T08:00:56.272265","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:53.197174Z","iopub.execute_input":"2024-05-24T16:24:53.197608Z","iopub.status.idle":"2024-05-24T16:24:53.207030Z","shell.execute_reply.started":"2024-05-24T16:24:53.197579Z","shell.execute_reply":"2024-05-24T16:24:53.205740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.load_model:\n    if os.path.exists(CFG.model_path / \"lightautoml/\"):\n        shutil.copytree(\n            CFG.model_path / \"lightautoml/\", \n            \"/kaggle/working/lightautoml\"\n        )","metadata":{"execution":{"iopub.status.busy":"2024-05-24T16:24:53.208753Z","iopub.execute_input":"2024-05-24T16:24:53.209125Z","iopub.status.idle":"2024-05-24T16:24:53.223011Z","shell.execute_reply.started":"2024-05-24T16:24:53.209081Z","shell.execute_reply":"2024-05-24T16:24:53.221476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_df.drop(columns=[\"WEEK_NUM\"])\nX_test = X_test.set_index(\"case_id\")\nprint(\"X_test shape: \", X_test.shape)\n\ny_pred = pd.Series(predict_proba_in_batches(model, X_test), index=X_test.index)\ny_pred[:10]","metadata":{"papermill":{"duration":3.411858,"end_time":"2024-02-19T08:00:59.773672","exception":false,"start_time":"2024-02-19T08:00:56.361814","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:53.224889Z","iopub.execute_input":"2024-05-24T16:24:53.225368Z","iopub.status.idle":"2024-05-24T16:24:59.149013Z","shell.execute_reply.started":"2024-05-24T16:24:53.225334Z","shell.execute_reply":"2024-05-24T16:24:59.147729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.041709,"end_time":"2024-02-19T08:00:59.858183","exception":false,"start_time":"2024-02-19T08:00:59.816474","status":"completed"},"tags":[]}},{"cell_type":"code","source":"subm_df = pd.read_csv(data_nb.CFG.root_dir / \"sample_submission.csv\")\nsubm_df = subm_df.set_index(\"case_id\")\nsubm_df[\"score\"] = y_pred","metadata":{"papermill":{"duration":0.058209,"end_time":"2024-02-19T08:00:59.95948","exception":false,"start_time":"2024-02-19T08:00:59.901271","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:59.161241Z","iopub.execute_input":"2024-05-24T16:24:59.161700Z","iopub.status.idle":"2024-05-24T16:24:59.175893Z","shell.execute_reply.started":"2024-05-24T16:24:59.161669Z","shell.execute_reply":"2024-05-24T16:24:59.174657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Check null: \", subm_df[\"score\"].isnull().any())\nsubm_df.head()","metadata":{"papermill":{"duration":0.054178,"end_time":"2024-02-19T08:01:00.057108","exception":false,"start_time":"2024-02-19T08:01:00.00293","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:59.177487Z","iopub.execute_input":"2024-05-24T16:24:59.177851Z","iopub.status.idle":"2024-05-24T16:24:59.190777Z","shell.execute_reply.started":"2024-05-24T16:24:59.177821Z","shell.execute_reply":"2024-05-24T16:24:59.189437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.load_model:\n    subm_df.to_csv(\"submission.csv\")","metadata":{"papermill":{"duration":0.050529,"end_time":"2024-02-19T08:01:00.149424","exception":false,"start_time":"2024-02-19T08:01:00.098895","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-24T16:24:59.192577Z","iopub.execute_input":"2024-05-24T16:24:59.193701Z","iopub.status.idle":"2024-05-24T16:24:59.205497Z","shell.execute_reply.started":"2024-05-24T16:24:59.193655Z","shell.execute_reply":"2024-05-24T16:24:59.204161Z"},"trusted":true},"execution_count":null,"outputs":[]}]}