{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Overview\n- Train-test split: `WEEK_NUM < 40` and `WEEK_NUM >= 40`\n- Some of `A`, `P`, and `M` features\n- Model: LightGBM with the default parameters","metadata":{}},{"cell_type":"markdown","source":"Tips\n- Polars `scan_csv` and `scan_parquet` accept a globbing pattern, e.g. `/path/to/train_static_0_*.parquet`. The result is the concatenation of all the tables whose filenames match the pattern.\n- `scan_csv` and `scan_parquet` construct a LazyFrame. Data will not be read until needed. I find it efficient particularly when not all columns are used.\n- `pl.col` accepts a regular expression starting with `^` and ending with `$`. For example, `pl.col(r'^.*A$')` selects all columns whose names end with `A`.","metadata":{}},{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport json","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:29.765698Z","iopub.execute_input":"2024-02-15T04:08:29.766651Z","iopub.status.idle":"2024-02-15T04:08:30.084890Z","shell.execute_reply.started":"2024-02-15T04:08:29.766562Z","shell.execute_reply":"2024-02-15T04:08:30.083549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load dataset","metadata":{}},{"cell_type":"code","source":"df_feature_definitions = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:30.087321Z","iopub.execute_input":"2024-02-15T04:08:30.087944Z","iopub.status.idle":"2024-02-15T04:08:30.290636Z","shell.execute_reply.started":"2024-02-15T04:08:30.087906Z","shell.execute_reply":"2024-02-15T04:08:30.289645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_base = pl.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_base.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:30.292326Z","iopub.execute_input":"2024-02-15T04:08:30.293714Z","iopub.status.idle":"2024-02-15T04:08:30.524068Z","shell.execute_reply.started":"2024-02-15T04:08:30.293656Z","shell.execute_reply":"2024-02-15T04:08:30.522212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_all = df_train_base.drop('target', 'WEEK_NUM')\ny_train_all = df_train_base['target']\nw_train_all = df_train_base['WEEK_NUM']","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:30.527009Z","iopub.execute_input":"2024-02-15T04:08:30.527394Z","iopub.status.idle":"2024-02-15T04:08:30.537775Z","shell.execute_reply.started":"2024-02-15T04:08:30.527365Z","shell.execute_reply":"2024-02-15T04:08:30.536875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For other tables,\nwe fix dtypes at read time.\nFrom the data description, we know that `A` and `P` group are numerical,\n`M` group is categorical,\nand `D` group is of dtype `date`.","metadata":{}},{"cell_type":"code","source":"def scan_data_and_fix_dtype(parquet_file):\n    return pl.scan_parquet(parquet_file).with_columns(\n        pl.col(r'^case_id$').cast(pl.Int64),\n        pl.col(r'^num_group\\d$').cast(pl.Int64),\n        pl.col(r'^.*A$').cast(pl.Float64),\n        pl.col(r'^.*M$').cast(pl.Categorical),\n        pl.col(r'^.*D$').str.to_date(),\n    )","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:30.539627Z","iopub.execute_input":"2024-02-15T04:08:30.540341Z","iopub.status.idle":"2024-02-15T04:08:30.551520Z","shell.execute_reply.started":"2024-02-15T04:08:30.540304Z","shell.execute_reply":"2024-02-15T04:08:30.550175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train-test split","metadata":{}},{"cell_type":"code","source":"def custom_split(X, y, w, split_at):\n    b = w < split_at\n    train_idx, = np.nonzero(b)\n    test_idx, = np.nonzero(~b)\n    return (\n        X[:, train_idx], X[:, test_idx],\n        y[train_idx], y[test_idx],\n        w[train_idx], w[test_idx],\n    )","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:30.553359Z","iopub.execute_input":"2024-02-15T04:08:30.554030Z","iopub.status.idle":"2024-02-15T04:08:30.566123Z","shell.execute_reply.started":"2024-02-15T04:08:30.553987Z","shell.execute_reply":"2024-02-15T04:08:30.564900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test, w_train, w_test = custom_split(\n    X_train_all, y_train_all, w_train_all, 40,\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:30.567994Z","iopub.execute_input":"2024-02-15T04:08:30.568505Z","iopub.status.idle":"2024-02-15T04:08:31.235424Z","shell.execute_reply.started":"2024-02-15T04:08:30.568469Z","shell.execute_reply":"2024-02-15T04:08:31.233861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `static_0` features","metadata":{}},{"cell_type":"code","source":"df_train_static_0 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_*.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.237603Z","iopub.execute_input":"2024-02-15T04:08:31.238232Z","iopub.status.idle":"2024-02-15T04:08:31.268993Z","shell.execute_reply.started":"2024-02-15T04:08:31.238195Z","shell.execute_reply":"2024-02-15T04:08:31.267760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_static_0_features(X, df_static_0):\n    X = X.lazy()\n    a = df_static_0.select('case_id', pl.col(r'^.*(?:A|P)$'))\n    return X.join(a, on = 'case_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.271117Z","iopub.execute_input":"2024-02-15T04:08:31.271576Z","iopub.status.idle":"2024-02-15T04:08:31.278847Z","shell.execute_reply.started":"2024-02-15T04:08:31.271534Z","shell.execute_reply":"2024-02-15T04:08:31.277310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_static_0_features(X_train, df_train_static_0).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.284678Z","iopub.execute_input":"2024-02-15T04:08:31.285349Z","iopub.status.idle":"2024-02-15T04:08:31.302506Z","shell.execute_reply.started":"2024-02-15T04:08:31.285313Z","shell.execute_reply":"2024-02-15T04:08:31.301055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `static_cb_0` features","metadata":{}},{"cell_type":"code","source":"df_train_static_cb_0 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_cb_0.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.304248Z","iopub.execute_input":"2024-02-15T04:08:31.304687Z","iopub.status.idle":"2024-02-15T04:08:31.328594Z","shell.execute_reply.started":"2024-02-15T04:08:31.304653Z","shell.execute_reply":"2024-02-15T04:08:31.327181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_static_cb_0_features(X, df_static_cb_0):\n    X = X.lazy()\n    a = df_static_cb_0.select('case_id', pl.col(r'^.*(?:A|M)$'))\n    return X.join(a, on = 'case_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.330319Z","iopub.execute_input":"2024-02-15T04:08:31.330766Z","iopub.status.idle":"2024-02-15T04:08:31.338561Z","shell.execute_reply.started":"2024-02-15T04:08:31.330714Z","shell.execute_reply":"2024-02-15T04:08:31.337661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_static_cb_0_features(X_train, df_train_static_cb_0).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.339488Z","iopub.execute_input":"2024-02-15T04:08:31.339783Z","iopub.status.idle":"2024-02-15T04:08:31.353504Z","shell.execute_reply.started":"2024-02-15T04:08:31.339758Z","shell.execute_reply":"2024-02-15T04:08:31.352139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `applprev_1` features","metadata":{}},{"cell_type":"code","source":"df_train_applprev_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_applprev_1_*.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.355085Z","iopub.execute_input":"2024-02-15T04:08:31.355592Z","iopub.status.idle":"2024-02-15T04:08:31.378120Z","shell.execute_reply.started":"2024-02-15T04:08:31.355555Z","shell.execute_reply":"2024-02-15T04:08:31.376705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_applprev_1_features(X, df_applprev_1):\n    X = X.lazy()\n    Y = df_applprev_1.group_by('case_id').agg(\n        pl.col(r'^.*A$').sort_by('num_group1').last().name.suffix('_last'),\n        pl.len().alias('applprev_1_count'),\n    )\n    return X.join(Y, on = 'case_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.379838Z","iopub.execute_input":"2024-02-15T04:08:31.380278Z","iopub.status.idle":"2024-02-15T04:08:31.388159Z","shell.execute_reply.started":"2024-02-15T04:08:31.380242Z","shell.execute_reply":"2024-02-15T04:08:31.386854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_applprev_1_features(X_train, df_train_applprev_1).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.389730Z","iopub.execute_input":"2024-02-15T04:08:31.390437Z","iopub.status.idle":"2024-02-15T04:08:31.406423Z","shell.execute_reply.started":"2024-02-15T04:08:31.390396Z","shell.execute_reply":"2024-02-15T04:08:31.405445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `other_1` features","metadata":{}},{"cell_type":"code","source":"df_train_other_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_other_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.408062Z","iopub.execute_input":"2024-02-15T04:08:31.408380Z","iopub.status.idle":"2024-02-15T04:08:31.431559Z","shell.execute_reply.started":"2024-02-15T04:08:31.408342Z","shell.execute_reply":"2024-02-15T04:08:31.430141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_other_1_features(X, df_other_1):\n    X = X.lazy()\n    Y = df_other_1.group_by('case_id').agg(\n        pl.col(r'^.*A$').mean().name.suffix('_mean'),\n    )\n    return X.join(Y, on = 'case_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.432828Z","iopub.execute_input":"2024-02-15T04:08:31.433223Z","iopub.status.idle":"2024-02-15T04:08:31.440740Z","shell.execute_reply.started":"2024-02-15T04:08:31.433191Z","shell.execute_reply":"2024-02-15T04:08:31.439065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_other_1_features(X_train, df_train_other_1).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.442885Z","iopub.execute_input":"2024-02-15T04:08:31.443381Z","iopub.status.idle":"2024-02-15T04:08:31.457637Z","shell.execute_reply.started":"2024-02-15T04:08:31.443335Z","shell.execute_reply":"2024-02-15T04:08:31.456024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `tax_registry_*_1` features","metadata":{}},{"cell_type":"markdown","source":"`tax_registry_*_1` tables are of similar schema.\nWe process them in the same way.","metadata":{}},{"cell_type":"code","source":"df_train_tax_registry_a_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_tax_registry_a_1.parquet')\ndf_train_tax_registry_b_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_tax_registry_b_1.parquet')\ndf_train_tax_registry_c_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_tax_registry_c_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.459464Z","iopub.execute_input":"2024-02-15T04:08:31.460009Z","iopub.status.idle":"2024-02-15T04:08:31.501315Z","shell.execute_reply.started":"2024-02-15T04:08:31.459971Z","shell.execute_reply":"2024-02-15T04:08:31.500024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_tax_registry_1_features(X, df_tax_registry_1):\n    X = X.lazy()\n    Y = df_tax_registry_1.group_by('case_id').agg(\n        pl.col(r'^.*A$').sum().name.suffix('_sum'),\n    )\n    return X.join(Y, on = 'case_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.502822Z","iopub.execute_input":"2024-02-15T04:08:31.503617Z","iopub.status.idle":"2024-02-15T04:08:31.510778Z","shell.execute_reply.started":"2024-02-15T04:08:31.503577Z","shell.execute_reply":"2024-02-15T04:08:31.509226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_tax_registry_1_features(X_train, df_train_tax_registry_a_1).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.512763Z","iopub.execute_input":"2024-02-15T04:08:31.513268Z","iopub.status.idle":"2024-02-15T04:08:31.525660Z","shell.execute_reply.started":"2024-02-15T04:08:31.513224Z","shell.execute_reply":"2024-02-15T04:08:31.524083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `credit_bureau_*_1` features","metadata":{}},{"cell_type":"code","source":"df_train_credit_bureau_a_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_a_1_*.parquet')\ndf_train_credit_bureau_b_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_b_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.527973Z","iopub.execute_input":"2024-02-15T04:08:31.528912Z","iopub.status.idle":"2024-02-15T04:08:31.567064Z","shell.execute_reply.started":"2024-02-15T04:08:31.528853Z","shell.execute_reply":"2024-02-15T04:08:31.565608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_credit_bureau_1_features(X, df_credit_bureau_1):\n    X = X.lazy()\n    Y = df_credit_bureau_1.group_by('case_id').agg(\n        pl.col(r'^.*A$').sum().name.suffix('_sum'),\n    )\n    return X.join(Y, on = 'case_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.568820Z","iopub.execute_input":"2024-02-15T04:08:31.569239Z","iopub.status.idle":"2024-02-15T04:08:31.577442Z","shell.execute_reply.started":"2024-02-15T04:08:31.569198Z","shell.execute_reply":"2024-02-15T04:08:31.575710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_credit_bureau_1_features(X_train, df_train_credit_bureau_a_1).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.579615Z","iopub.execute_input":"2024-02-15T04:08:31.580172Z","iopub.status.idle":"2024-02-15T04:08:31.595244Z","shell.execute_reply.started":"2024-02-15T04:08:31.580128Z","shell.execute_reply":"2024-02-15T04:08:31.593483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `deposit_1` features","metadata":{}},{"cell_type":"code","source":"df_train_deposit_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_deposit_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.596572Z","iopub.execute_input":"2024-02-15T04:08:31.597674Z","iopub.status.idle":"2024-02-15T04:08:31.621904Z","shell.execute_reply.started":"2024-02-15T04:08:31.597621Z","shell.execute_reply":"2024-02-15T04:08:31.620297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_deposit_1_features(X, df_deposit_1):\n    X = X.lazy()\n    Y = df_deposit_1.group_by('case_id').agg(\n        pl.col(r'^.*A$').sum().name.suffix('_sum'),\n    )\n    return X.join(Y, on = 'case_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.623887Z","iopub.execute_input":"2024-02-15T04:08:31.624361Z","iopub.status.idle":"2024-02-15T04:08:31.631414Z","shell.execute_reply.started":"2024-02-15T04:08:31.624320Z","shell.execute_reply":"2024-02-15T04:08:31.630117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_deposit_1_features(X_train, df_train_deposit_1).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.642222Z","iopub.execute_input":"2024-02-15T04:08:31.643432Z","iopub.status.idle":"2024-02-15T04:08:31.652168Z","shell.execute_reply.started":"2024-02-15T04:08:31.643382Z","shell.execute_reply":"2024-02-15T04:08:31.651028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `person_1` features","metadata":{}},{"cell_type":"markdown","source":"Remember that `num_group1 == 0` is the applicant.","metadata":{}},{"cell_type":"code","source":"df_train_person_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_person_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.653566Z","iopub.execute_input":"2024-02-15T04:08:31.654033Z","iopub.status.idle":"2024-02-15T04:08:31.672993Z","shell.execute_reply.started":"2024-02-15T04:08:31.653934Z","shell.execute_reply":"2024-02-15T04:08:31.671466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_person_1_features(X, df_person_1):\n    X = X.lazy()\n    Y = df_person_1.group_by('case_id').agg(\n        pl.col(r'^.*A$').sort_by('num_group1').first().name.suffix('_first'),\n        pl.len().alias('person_1_count'),\n    )\n    return X.join(Y, on = 'case_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.674747Z","iopub.execute_input":"2024-02-15T04:08:31.675276Z","iopub.status.idle":"2024-02-15T04:08:31.682343Z","shell.execute_reply.started":"2024-02-15T04:08:31.675232Z","shell.execute_reply":"2024-02-15T04:08:31.680369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_person_1_features(X_train, df_train_person_1).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.684180Z","iopub.execute_input":"2024-02-15T04:08:31.684671Z","iopub.status.idle":"2024-02-15T04:08:31.697257Z","shell.execute_reply.started":"2024-02-15T04:08:31.684635Z","shell.execute_reply":"2024-02-15T04:08:31.695667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `debitcard_1` features","metadata":{}},{"cell_type":"code","source":"df_train_debitcard_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_debitcard_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.700290Z","iopub.execute_input":"2024-02-15T04:08:31.701297Z","iopub.status.idle":"2024-02-15T04:08:31.716686Z","shell.execute_reply.started":"2024-02-15T04:08:31.701256Z","shell.execute_reply":"2024-02-15T04:08:31.715778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_debitcard_1_features(X, df_debitcard_1):\n    X = X.lazy()\n    Y = df_debitcard_1.group_by('case_id').agg(\n        pl.col(r'^.*A$').sum().name.suffix('_sum'),\n        pl.len().alias('debitcard_1_count'),\n    )\n    return X.join(Y, on = 'case_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.717772Z","iopub.execute_input":"2024-02-15T04:08:31.718470Z","iopub.status.idle":"2024-02-15T04:08:31.726325Z","shell.execute_reply.started":"2024-02-15T04:08:31.718429Z","shell.execute_reply":"2024-02-15T04:08:31.724803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_debitcard_1_features(X_train, df_train_debitcard_1).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.728194Z","iopub.execute_input":"2024-02-15T04:08:31.728623Z","iopub.status.idle":"2024-02-15T04:08:31.745941Z","shell.execute_reply.started":"2024-02-15T04:08:31.728583Z","shell.execute_reply":"2024-02-15T04:08:31.744319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `applprev_2` features","metadata":{}},{"cell_type":"code","source":"df_train_applprev_2 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_applprev_2.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.747754Z","iopub.execute_input":"2024-02-15T04:08:31.748259Z","iopub.status.idle":"2024-02-15T04:08:31.769750Z","shell.execute_reply.started":"2024-02-15T04:08:31.748222Z","shell.execute_reply":"2024-02-15T04:08:31.768215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We don't use `applprev_2` for now.","metadata":{}},{"cell_type":"code","source":"def join_applprev_2_features(X, df_applprev_2):\n    return X","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.771422Z","iopub.execute_input":"2024-02-15T04:08:31.771823Z","iopub.status.idle":"2024-02-15T04:08:31.783976Z","shell.execute_reply.started":"2024-02-15T04:08:31.771783Z","shell.execute_reply":"2024-02-15T04:08:31.782302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_applprev_2_features(X_train, df_train_applprev_2).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.785993Z","iopub.execute_input":"2024-02-15T04:08:31.786501Z","iopub.status.idle":"2024-02-15T04:08:31.802425Z","shell.execute_reply.started":"2024-02-15T04:08:31.786450Z","shell.execute_reply":"2024-02-15T04:08:31.800598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `person_2` features","metadata":{}},{"cell_type":"code","source":"df_train_person_2 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_person_2.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.803922Z","iopub.execute_input":"2024-02-15T04:08:31.805013Z","iopub.status.idle":"2024-02-15T04:08:31.821936Z","shell.execute_reply.started":"2024-02-15T04:08:31.804935Z","shell.execute_reply":"2024-02-15T04:08:31.820884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We don't use `person_2` for now.","metadata":{}},{"cell_type":"code","source":"def join_person_2_features(X, df_person_2):\n    return X","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.823144Z","iopub.execute_input":"2024-02-15T04:08:31.823528Z","iopub.status.idle":"2024-02-15T04:08:31.834161Z","shell.execute_reply.started":"2024-02-15T04:08:31.823494Z","shell.execute_reply":"2024-02-15T04:08:31.832980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_person_2_features(X_train, df_train_person_2).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.835328Z","iopub.execute_input":"2024-02-15T04:08:31.836399Z","iopub.status.idle":"2024-02-15T04:08:31.848497Z","shell.execute_reply.started":"2024-02-15T04:08:31.836347Z","shell.execute_reply":"2024-02-15T04:08:31.847062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `credit_bureau_*_2` features","metadata":{}},{"cell_type":"code","source":"df_train_credit_bureau_a_2 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_a_2_*.parquet')\ndf_train_credit_bureau_b_2 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_b_2.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.849521Z","iopub.execute_input":"2024-02-15T04:08:31.849851Z","iopub.status.idle":"2024-02-15T04:08:31.882691Z","shell.execute_reply.started":"2024-02-15T04:08:31.849823Z","shell.execute_reply":"2024-02-15T04:08:31.881287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_credit_bureau_2_features(X, df_credit_bureau_2):\n    X = X.lazy()\n    Y = df_credit_bureau_2.group_by('case_id').agg(\n        pl.col(r'^.*A$').sum().name.suffix('_sum'),\n    )\n    return X.join(Y, on = 'case_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.884276Z","iopub.execute_input":"2024-02-15T04:08:31.884635Z","iopub.status.idle":"2024-02-15T04:08:31.890594Z","shell.execute_reply.started":"2024-02-15T04:08:31.884605Z","shell.execute_reply":"2024-02-15T04:08:31.889254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(join_credit_bureau_2_features(X_train, df_train_credit_bureau_a_2).schema)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.892430Z","iopub.execute_input":"2024-02-15T04:08:31.892801Z","iopub.status.idle":"2024-02-15T04:08:31.904710Z","shell.execute_reply.started":"2024-02-15T04:08:31.892764Z","shell.execute_reply":"2024-02-15T04:08:31.903473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# All features","metadata":{}},{"cell_type":"code","source":"def drop_unwanted_features(X):\n    return X.drop('case_id', 'date_decision', 'MONTH')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.906068Z","iopub.execute_input":"2024-02-15T04:08:31.906531Z","iopub.status.idle":"2024-02-15T04:08:31.916800Z","shell.execute_reply.started":"2024-02-15T04:08:31.906491Z","shell.execute_reply":"2024-02-15T04:08:31.915370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_all_features(X,\n                          df_static_0,\n                          df_static_cb_0,\n                          df_applprev_1,\n                          df_other_1,\n                          df_tax_registry_a_1,\n                          df_tax_registry_b_1,\n                          df_tax_registry_c_1,\n                          df_credit_bureau_a_1,\n                          df_credit_bureau_b_1,\n                          df_deposit_1,\n                          df_person_1,\n                          df_debitcard_1,\n                          df_applprev_2,\n                          df_person_2,\n                          df_credit_bureau_a_2,\n                          df_credit_bureau_b_2):\n    X = X.lazy()\n    X = join_static_0_features(X, df_static_0)\n    X = join_static_cb_0_features(X, df_static_cb_0)\n    X = join_applprev_1_features(X, df_applprev_1)\n    X = join_other_1_features(X, df_other_1)\n    for Y in [df_tax_registry_a_1, df_tax_registry_b_1, df_tax_registry_c_1]:\n        X = join_tax_registry_1_features(X, Y)\n    for Y in [df_credit_bureau_a_1, df_credit_bureau_b_1]:\n        X = join_credit_bureau_1_features(X, Y)\n    X = join_deposit_1_features(X, df_deposit_1)\n    X = join_person_1_features(X, df_person_1)\n    X = join_debitcard_1_features(X, df_debitcard_1)\n    X = join_applprev_2_features(X, df_applprev_2)\n    X = join_person_2_features(X, df_person_2)\n    for Y in [df_credit_bureau_a_2, df_credit_bureau_b_2]:\n        X = join_credit_bureau_2_features(X, Y)\n    X = drop_unwanted_features(X)\n    return X","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.918364Z","iopub.execute_input":"2024-02-15T04:08:31.918725Z","iopub.status.idle":"2024-02-15T04:08:31.929315Z","shell.execute_reply.started":"2024-02-15T04:08:31.918687Z","shell.execute_reply":"2024-02-15T04:08:31.927799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_feats = generate_all_features(\n    X_train,\n    df_train_static_0,\n    df_train_static_cb_0,\n    df_train_applprev_1,\n    df_train_other_1,\n    df_train_tax_registry_a_1,\n    df_train_tax_registry_b_1,\n    df_train_tax_registry_c_1,\n    df_train_credit_bureau_a_1,\n    df_train_credit_bureau_b_1,\n    df_train_deposit_1,\n    df_train_person_1,\n    df_train_debitcard_1,\n    df_train_applprev_2,\n    df_train_person_2,\n    df_train_credit_bureau_a_2,\n    df_train_credit_bureau_b_2,\n).collect(streaming = True).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:08:31.931089Z","iopub.execute_input":"2024-02-15T04:08:31.931545Z","iopub.status.idle":"2024-02-15T04:09:11.313593Z","shell.execute_reply.started":"2024-02-15T04:08:31.931503Z","shell.execute_reply":"2024-02-15T04:09:11.312348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(X_train_feats.info())","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:09:11.315425Z","iopub.execute_input":"2024-02-15T04:09:11.315773Z","iopub.status.idle":"2024-02-15T04:09:11.350223Z","shell.execute_reply.started":"2024-02-15T04:09:11.315745Z","shell.execute_reply":"2024-02-15T04:09:11.348841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_feats = generate_all_features(\n    X_test,\n    df_train_static_0,\n    df_train_static_cb_0,\n    df_train_applprev_1,\n    df_train_other_1,\n    df_train_tax_registry_a_1,\n    df_train_tax_registry_b_1,\n    df_train_tax_registry_c_1,\n    df_train_credit_bureau_a_1,\n    df_train_credit_bureau_b_1,\n    df_train_deposit_1,\n    df_train_person_1,\n    df_train_debitcard_1,\n    df_train_applprev_2,\n    df_train_person_2,\n    df_train_credit_bureau_a_2,\n    df_train_credit_bureau_b_2,\n).collect(streaming = True).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:09:11.351759Z","iopub.execute_input":"2024-02-15T04:09:11.352143Z","iopub.status.idle":"2024-02-15T04:09:46.393060Z","shell.execute_reply.started":"2024-02-15T04:09:11.352112Z","shell.execute_reply":"2024-02-15T04:09:46.391910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train LightGBM","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgbm","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:09:46.394506Z","iopub.execute_input":"2024-02-15T04:09:46.395341Z","iopub.status.idle":"2024-02-15T04:09:48.470581Z","shell.execute_reply.started":"2024-02-15T04:09:46.395308Z","shell.execute_reply":"2024-02-15T04:09:48.469212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_train = lgbm.Dataset(X_train_feats, label = y_train.to_pandas())","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:09:48.472582Z","iopub.execute_input":"2024-02-15T04:09:48.473127Z","iopub.status.idle":"2024-02-15T04:09:48.480307Z","shell.execute_reply.started":"2024-02-15T04:09:48.473071Z","shell.execute_reply":"2024-02-15T04:09:48.478863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'seed': 0,\n    'objective': 'binary',\n    'verbosity': -1,\n}","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:09:48.481851Z","iopub.execute_input":"2024-02-15T04:09:48.482242Z","iopub.status.idle":"2024-02-15T04:09:48.498819Z","shell.execute_reply.started":"2024-02-15T04:09:48.482210Z","shell.execute_reply":"2024-02-15T04:09:48.496890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"booster = lgbm.train(\n    params,\n    ds_train,\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:09:48.500702Z","iopub.execute_input":"2024-02-15T04:09:48.501244Z","iopub.status.idle":"2024-02-15T04:10:11.875033Z","shell.execute_reply.started":"2024-02-15T04:09:48.501206Z","shell.execute_reply":"2024-02-15T04:10:11.873726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"booster.save_model('lgbm_eval.txt');","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:11.876604Z","iopub.execute_input":"2024-02-15T04:10:11.876922Z","iopub.status.idle":"2024-02-15T04:10:11.890665Z","shell.execute_reply.started":"2024-02-15T04:10:11.876895Z","shell.execute_reply":"2024-02-15T04:10:11.889645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:11.892379Z","iopub.execute_input":"2024-02-15T04:10:11.892943Z","iopub.status.idle":"2024-02-15T04:10:11.897108Z","shell.execute_reply.started":"2024-02-15T04:10:11.892911Z","shell.execute_reply":"2024-02-15T04:10:11.896282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gini_stability(y_true, y_pred, w, w_fallingrate = 88.0, w_resstd = -0.5):\n    X = pl.DataFrame({\n        'true_target': y_true,\n        'predicted_target': y_pred,\n        'WEEK_NUM': w,\n    })\n    z = np.array([\n        [x['WEEK_NUM'][0], roc_auc_score(x['true_target'], x['predicted_target'])]\n        for _, x in X.group_by(['WEEK_NUM'])\n    ], dtype = np.float64)\n    weekly_auc = z[:, 1]\n    weekly_gini = 2 * weekly_auc - 1\n    x = z[:, 0]\n    y = weekly_gini\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a * x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(weekly_gini)\n    return {\n        'score': avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std,\n        'avg_gini': avg_gini,\n        'a': a,\n        'std(residuals)': res_std,\n        'week': list(x),\n        'gini': list(y),\n    }","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:11.898522Z","iopub.execute_input":"2024-02-15T04:10:11.899072Z","iopub.status.idle":"2024-02-15T04:10:11.912515Z","shell.execute_reply.started":"2024-02-15T04:10:11.899043Z","shell.execute_reply":"2024-02-15T04:10:11.911071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_score = gini_stability(y_test, booster.predict(X_test_feats), w_test)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:11.914196Z","iopub.execute_input":"2024-02-15T04:10:11.914524Z","iopub.status.idle":"2024-02-15T04:10:16.022203Z","shell.execute_reply.started":"2024-02-15T04:10:11.914498Z","shell.execute_reply":"2024-02-15T04:10:16.020721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_score)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:16.024206Z","iopub.execute_input":"2024-02-15T04:10:16.024643Z","iopub.status.idle":"2024-02-15T04:10:16.032313Z","shell.execute_reply.started":"2024-02-15T04:10:16.024608Z","shell.execute_reply":"2024-02-15T04:10:16.031142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\nsns.regplot(x = test_score['week'], y = test_score['gini']);","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:16.033876Z","iopub.execute_input":"2024-02-15T04:10:16.034531Z","iopub.status.idle":"2024-02-15T04:10:16.684867Z","shell.execute_reply.started":"2024-02-15T04:10:16.034492Z","shell.execute_reply":"2024-02-15T04:10:16.683181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('score.json', 'w') as f:\n    json.dump(test_score, f)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:16.686863Z","iopub.execute_input":"2024-02-15T04:10:16.687307Z","iopub.status.idle":"2024-02-15T04:10:16.693512Z","shell.execute_reply.started":"2024-02-15T04:10:16.687271Z","shell.execute_reply":"2024-02-15T04:10:16.692237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_feature_importance = pl.DataFrame({\n    'feature': booster.feature_name(),\n    'gain': booster.feature_importance('gain'),\n    'split': booster.feature_importance('split'),\n}).with_columns(\n    pl.col('feature').str.extract(r'^(.*[A-Z])').alias('Variable'),\n).join(df_feature_definitions, on = 'Variable', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:16.695056Z","iopub.execute_input":"2024-02-15T04:10:16.695465Z","iopub.status.idle":"2024-02-15T04:10:16.720086Z","shell.execute_reply.started":"2024-02-15T04:10:16.695432Z","shell.execute_reply":"2024-02-15T04:10:16.718791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with pl.Config() as cfg:\n    cfg.set_tbl_rows(-1)\n    cfg.set_fmt_str_lengths(300)\n    display(df_feature_importance.sort('gain', descending = True))","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:16.721564Z","iopub.execute_input":"2024-02-15T04:10:16.721927Z","iopub.status.idle":"2024-02-15T04:10:16.765034Z","shell.execute_reply.started":"2024-02-15T04:10:16.721896Z","shell.execute_reply":"2024-02-15T04:10:16.763588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm.plot_importance(booster, importance_type = 'gain',\n                     figsize = (8, X_train_feats.shape[1] // 4));","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:16.766737Z","iopub.execute_input":"2024-02-15T04:10:16.767196Z","iopub.status.idle":"2024-02-15T04:10:19.074518Z","shell.execute_reply.started":"2024-02-15T04:10:16.767160Z","shell.execute_reply":"2024-02-15T04:10:19.073244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"X_train_all_feats = generate_all_features(\n    X_train_all,\n    df_train_static_0,\n    df_train_static_cb_0,\n    df_train_applprev_1,\n    df_train_other_1,\n    df_train_tax_registry_a_1,\n    df_train_tax_registry_b_1,\n    df_train_tax_registry_c_1,\n    df_train_credit_bureau_a_1,\n    df_train_credit_bureau_b_1,\n    df_train_deposit_1,\n    df_train_person_1,\n    df_train_debitcard_1,\n    df_train_applprev_2,\n    df_train_person_2,\n    df_train_credit_bureau_a_2,\n    df_train_credit_bureau_b_2,\n).collect(streaming = True).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:19.076582Z","iopub.execute_input":"2024-02-15T04:10:19.076962Z","iopub.status.idle":"2024-02-15T04:10:53.970341Z","shell.execute_reply.started":"2024-02-15T04:10:19.076917Z","shell.execute_reply":"2024-02-15T04:10:53.969017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_train_all = lgbm.Dataset(X_train_all_feats, label = y_train_all.to_pandas())","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:53.972201Z","iopub.execute_input":"2024-02-15T04:10:53.972555Z","iopub.status.idle":"2024-02-15T04:10:53.978427Z","shell.execute_reply.started":"2024-02-15T04:10:53.972526Z","shell.execute_reply":"2024-02-15T04:10:53.977104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_booster = lgbm.train(\n    params, ds_train_all,\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:10:53.980620Z","iopub.execute_input":"2024-02-15T04:10:53.981105Z","iopub.status.idle":"2024-02-15T04:11:44.468459Z","shell.execute_reply.started":"2024-02-15T04:10:53.981066Z","shell.execute_reply":"2024-02-15T04:11:44.467009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_booster.save_model('lgbm_submission.txt');","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:11:44.469901Z","iopub.execute_input":"2024-02-15T04:11:44.470245Z","iopub.status.idle":"2024-02-15T04:11:44.482997Z","shell.execute_reply.started":"2024-02-15T04:11:44.470216Z","shell.execute_reply":"2024-02-15T04:11:44.481579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pl.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_base.parquet')\ndf_test = df_test.drop('WEEK_NUM')\ndf_test_static_0 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_static_0_*.parquet')\ndf_test_static_cb_0 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_static_cb_0.parquet')\ndf_test_applprev_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_applprev_1_*.parquet')\ndf_test_other_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_other_1.parquet')\ndf_test_tax_registry_a_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_tax_registry_a_1.parquet')\ndf_test_tax_registry_b_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_tax_registry_b_1.parquet')\ndf_test_tax_registry_c_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_tax_registry_c_1.parquet')\ndf_test_credit_bureau_a_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_credit_bureau_a_1_*.parquet')\ndf_test_credit_bureau_b_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_credit_bureau_b_1.parquet')\ndf_test_deposit_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_deposit_1.parquet')\ndf_test_person_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_person_1.parquet')\ndf_test_debitcard_1 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_debitcard_1.parquet')\ndf_test_applprev_2 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_applprev_2.parquet')\ndf_test_person_2 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_person_2.parquet')\ndf_test_credit_bureau_a_2 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_credit_bureau_a_2_*.parquet')\ndf_test_credit_bureau_b_2 = scan_data_and_fix_dtype('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_credit_bureau_b_2.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:11:44.484779Z","iopub.execute_input":"2024-02-15T04:11:44.485141Z","iopub.status.idle":"2024-02-15T04:11:44.632488Z","shell.execute_reply.started":"2024-02-15T04:11:44.485110Z","shell.execute_reply":"2024-02-15T04:11:44.631595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_feats = generate_all_features(\n    df_test,\n    df_test_static_0,\n    df_test_static_cb_0,\n    df_test_applprev_1,\n    df_test_other_1,\n    df_test_tax_registry_a_1,\n    df_test_tax_registry_b_1,\n    df_test_tax_registry_c_1,\n    df_test_credit_bureau_a_1,\n    df_test_credit_bureau_b_1,\n    df_test_deposit_1,\n    df_test_person_1,\n    df_test_debitcard_1,\n    df_test_applprev_2,\n    df_test_person_2,\n    df_test_credit_bureau_a_2,\n    df_test_credit_bureau_b_2,\n).collect(streaming = True).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:11:44.633898Z","iopub.execute_input":"2024-02-15T04:11:44.634671Z","iopub.status.idle":"2024-02-15T04:11:44.779217Z","shell.execute_reply.started":"2024-02-15T04:11:44.634639Z","shell.execute_reply":"2024-02-15T04:11:44.777792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(df_test_feats.info())","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:11:44.780609Z","iopub.execute_input":"2024-02-15T04:11:44.780916Z","iopub.status.idle":"2024-02-15T04:11:44.801608Z","shell.execute_reply.started":"2024-02-15T04:11:44.780890Z","shell.execute_reply":"2024-02-15T04:11:44.800445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = pl.DataFrame({\n    'case_id': df_test['case_id'],\n    'score': final_booster.predict(df_test_feats),\n})","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:11:44.803040Z","iopub.execute_input":"2024-02-15T04:11:44.804271Z","iopub.status.idle":"2024-02-15T04:11:44.820112Z","shell.execute_reply.started":"2024-02-15T04:11:44.804227Z","shell.execute_reply":"2024-02-15T04:11:44.818850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(df_submission)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:11:44.821726Z","iopub.execute_input":"2024-02-15T04:11:44.822180Z","iopub.status.idle":"2024-02-15T04:11:44.832799Z","shell.execute_reply.started":"2024-02-15T04:11:44.822142Z","shell.execute_reply":"2024-02-15T04:11:44.831438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.write_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T04:11:44.834478Z","iopub.execute_input":"2024-02-15T04:11:44.834898Z","iopub.status.idle":"2024-02-15T04:11:44.847314Z","shell.execute_reply.started":"2024-02-15T04:11:44.834860Z","shell.execute_reply":"2024-02-15T04:11:44.845978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}