{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nfrom pandas.api.types import is_numeric_dtype\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \nfrom sklearn.cluster import KMeans\nfrom sklearn.linear_model import LinearRegression","metadata":{"execution":{"iopub.status.busy":"2024-04-20T01:27:06.858738Z","iopub.execute_input":"2024-04-20T01:27:06.859499Z","iopub.status.idle":"2024-04-20T01:27:11.943807Z","shell.execute_reply.started":"2024-04-20T01:27:06.859462Z","shell.execute_reply":"2024-04-20T01:27:11.942470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:\n        if not is_numeric_dtype(df[col]):\n#         if df[col].dtype.name in ['object', 'string', \"DateTime64DType\"]:\n          \n#             df.drop(col, axis=1, inplace=True)\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n#         else:\n#             df[col] = df[col].fill_null(0)\n    return df","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-20T01:27:11.946525Z","iopub.execute_input":"2024-04-20T01:27:11.947340Z","iopub.status.idle":"2024-04-20T01:27:11.956219Z","shell.execute_reply.started":"2024-04-20T01:27:11.947288Z","shell.execute_reply":"2024-04-20T01:27:11.955010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_processed_data(batch_type=\"train\"):\n    dataPath = f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{batch_type}/\"\n    basetable = pl.read_csv(dataPath + f\"{batch_type}_base.csv\")\n    \n    static_cb = pl.read_csv(dataPath + f\"{batch_type}_static_cb_0.csv\")\n    person_1 = pl.read_csv(dataPath + f\"{batch_type}_person_1.csv\")\n    \n    credit_bureau_b_2 = pl.read_csv(dataPath + f\"{batch_type}_credit_bureau_b_2.csv\")\n    credit_bureau_a_2 =  pl.read_csv(dataPath + f\"{batch_type}_credit_bureau_a_2_0.csv\")\n    credit_bureau_a_1 = pl.read_csv(dataPath + f\"{batch_type}_credit_bureau_a_1_0.csv\")\n    credit_bureau_b_1 = pl.read_csv(dataPath + f\"{batch_type}_credit_bureau_b_1.csv\")\n    debitcard_1 = pl.read_csv(dataPath + f\"{batch_type}_debitcard_1.csv\")\n    deposit_1 =  pl.read_csv(dataPath + f\"{batch_type}_deposit_1.csv\")\n    \n    applprev_1 = pl.read_csv(dataPath + f\"{batch_type}_applprev_1_0.csv\")\n    applprev_2 = pl.read_csv(dataPath + f\"{batch_type}_applprev_2.csv\")\n    \n    other_1 = pl.read_csv(dataPath + f\"{batch_type}_other_1.csv\")\n    \n    \n    \n#     person_2 = pl.read_csv(dataPath + f\"{batch_type}_person_2.csv\")\n    \n    static_0 = pl.read_csv(dataPath + f\"{batch_type}_static_0_0.csv\")\n    \n    \n    applprev_1_feats = applprev_1.group_by(\"case_id\").agg(\n        pl.col(\"annuity_853A\").mean().alias(\"annuity_853A_mean\"),\n        pl.col(\"actualdpd_943P\").mean().alias(\"actualdpd_943P_mean\"),\n        (pl.col(\"actualdpd_943P\")/pl.col(\"annuity_853A\")).mean().alias(\"annuity_853A_mean_ratio\"),\n        (pl.col(\"annuity_853A\")/pl.col(\"byoccupationinc_3656910L\")).mean().alias(\"ann_to_incom_ratio\"),\n        pl.col(\"cancelreason_3545846M\").mode().first().alias(\"cancelreason_3545846M_mode\"),\n        (pl.col(\"case_id\").count()/applprev_1.height).alias(\"applprev_freq\")\n    \n    )\n    \n    applprev_2_feats = applprev_2.group_by(\"case_id\").agg(\n        pl.col(\"conts_type_509L\").mode().first().alias(\"phone_number\")\n    ).select(\"phone_number\", \"case_id\")\n    \n    credit_bureau_a_1_feats = credit_bureau_a_1.filter(pl.col(\"num_group1\") == 0).drop(\"num_group1\").select([\"credlmt_935A\", \"dpdmax_139P\", \"financialinstitution_382M\",\n                                                                                          \"residualamount_856A\", \"totaloutstanddebtvalue_668A\", \"classificationofcontr_400M\",\n                                                                                          \"contractst_964M\", \"case_id\"])\n    \n    credit_bureau_b_1_feats = credit_bureau_b_1.group_by(\"case_id\").agg(\n        pl.col(\"amount_1115A\").mean().alias(\"amount_mean\"),\n        pl.col(\"credlmt_1052A\").mean().alias(\"cred_limit_mean\"),\n        pl.col(\"debtvalue_227A\").mean().alias(\"debt_value_mean\"),\n        pl.col(\"numberofinstls_810L\").mean().alias(\"instls_mean\"),\n        pl.col(\"overdueamountmax_950A\").mean().alias(\"overdue_mean\"),\n        pl.col(\"pmtnumpending_403L\").mean().alias(\"pending_mean\"),\n        ((pl.col(\"amount_1115A\")+pl.col(\"debtvalue_227A\"))/pl.col(\"credlmt_1052A\")).mean().alias(\"debt_ratio\"),\n        (pl.col(\"overdueamountmax_950A\")/pl.col(\"numberofinstls_810L\")).mean().alias(\"inst_ratio\")\n        \n    )\n    \n    credit_bureau_a_2_feats = credit_bureau_a_2.group_by(\"case_id\").agg(\n        pl.col(\"collaterals_typeofguarante_669M\").mode().first().alias(\"mode_guarentee\"),\n        pl.col(\"collater_typofvalofguarant_298M\").mode().first().alias(\"mode_guarent\"),\n        pl.col(\"subjectroles_name_838M\").mode().first().alias(\"mode_subjectrole\"),\n        pl.col(\"pmts_year_1139T\").mean().alias(\"pmts_year_1138T_mean\"),\n        pl.col(\"pmts_dpd_1073P\").mean().alias(\"pmts_dpd_1073P_mean\"),\n        pl.col(\"pmts_overdue_1140A\").mean().alias(\"pmts_overdue_1140A_mean\"),\n        \n    )\n    \n    \n    person_1_feats = person_1.filter(pl.col(\"num_group1\") == 0).drop(\"num_group1\")\n    \n    \n#     df = df.with_columns(pl.col(\"foo\").cast(pl.Int64))\n    person_1_feats = person_1_feats.with_columns(\n    pl.col(\"birth_259D\").str.to_datetime(\"%Y-%m-%d\"),\n    pl.col(\"empl_employedfrom_271D\").str.to_datetime(\"%Y-%m-%d\")\n)\n    \n    person_1_feats = person_1_feats.with_columns(\n        pl.col(\"birth_259D\").dt.round(every=\"10y\").alias(\"rounded_bday\"),\n        pl.col(\"empl_employedfrom_271D\").dt.round(every=\"10y\").alias(\"rounded_work_day\")\n)\n    \n    debitcard_1_feats = debitcard_1.filter(pl.col(\"num_group1\") == 0).drop(\"num_group1\")\n    debitcard_1_feats = debitcard_1_feats.with_columns(\n    pl.col(\"openingdate_857D\").str.to_datetime(\"%Y-%m-%d\")\n)\n    debitcard_1_feats = debitcard_1_feats.with_columns(pl.col(\"openingdate_857D\").dt.round(every=\"1y\").alias(\"rounded_cc_day\")).select([\"case_id\", \"rounded_cc_day\"])\n    \n    deposit_1_feats = deposit_1.group_by(\"case_id\").agg(\n        pl.col(\"amount_416A\").mean().alias(\"mean_deposit\"),\n    )\n    \n    other_1_feats = other_1.group_by(\"case_id\").agg(\n        pl.col(\"amtdepositbalance_4809441A\").mean().alias(\"balance_mean\"),\n        (pl.col(\"amtdebitincoming_4809443A\")-pl.col(\"amtdebitoutgoing_4809440A\")).mean().alias(\"debt_net\"),\n        (pl.col(\"amtdepositincoming_4809444A\")-pl.col(\"amtdepositoutgoing_4809442A\")).mean().alias(\"deposit_net\")\n    )\n    \n    \n    \n                                    \n    person_1_morefeats_1 = person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n    person_1_feats = person_1_feats.select([\"case_id\",\"rounded_bday\", \"contaddr_district_15M\", \"contaddr_matchlist_1032L\",\n                                           \"education_927M\", \"rounded_work_day\", \"mainoccupationinc_384A\", \"language1_981M\",\n                                           \"remitter_829L\", \"sex_738L\", \"type_25L\", \"relationshiptoclient_415T\", \"incometype_1044T\",\n                                           \"familystate_447L\", \"empl_employedtotal_800L\", \"empl_industry_691L\", \"contaddr_smempladdr_334L\"])\n    \n    credit_bureau_b_2_feats = credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n    \n    \n   \n\n\n    \n#     print(person_1_feats.select(pl.col(\"rounded_bday\")))\n    \n    data = basetable.join(\n    person_1_feats, how=\"left\", on=\"case_id\"\n    ).join(\n        applprev_1_feats, how=\"left\", on=\"case_id\"\n    ).join(\n        applprev_2_feats, how=\"left\", on=\"case_id\"\n    ).join(\n        static_0, how=\"left\", on=\"case_id\"\n    ).join(\n        static_cb, how=\"left\", on=\"case_id\"\n    ).join(\n        credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n    ).join(\n        credit_bureau_a_2_feats, how=\"left\", on=\"case_id\"\n    ).join(\n         person_1_morefeats_1, how=\"left\", on=\"case_id\"\n    ).join(\n        person_1_morefeats_1, how=\"left\", on=\"case_id\"\n    ).join(credit_bureau_a_1_feats, how=\"left\", on=\"case_id\"\n    ).join(credit_bureau_b_1_feats, how=\"left\", on=\"case_id\"\n    ).join(other_1_feats, how=\"left\", on=\"case_id\"\n    ).join(debitcard_1_feats, how=\"left\", on=\"case_id\"\n    ).join(deposit_1, how=\"left\", on=\"case_id\")\n    \n#     max_nas_perc = 0.6\n#     data = data.select(col.name for col in data.null_count() / data.height if col.item() <= max_nas_perc)\n    \n    \n\n    \n    print(data)\n    return data\n    ","metadata":{"execution":{"iopub.status.busy":"2024-04-20T01:27:11.958110Z","iopub.execute_input":"2024-04-20T01:27:11.958862Z","iopub.status.idle":"2024-04-20T01:27:12.108006Z","shell.execute_reply.started":"2024-04-20T01:27:11.958820Z","shell.execute_reply":"2024-04-20T01:27:12.106440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch = get_processed_data(\"train\")\nlabels = batch.to_pandas()[\"target\"]\n\nfeatures = batch.drop(\"case_id\").drop(\"target\").to_pandas()\nc = features.select_dtypes(np.number).columns\nfeatures[c] = features[c].fillna(0)\n\nreg_data = features[[\"pmts_pmtsoverdue_635A_max\", \"mainoccupationinc_384A_max\", \"debt_net\", \n                                \"deposit_net\", \"pmts_dpd_1073P_mean\"]]\n    \nreg_data = reg_data.fillna(0)\nreg = LinearRegression().fit(reg_data, labels)\nreg_values = reg.predict(reg_data)\n\n# kmeans = KMeans(n_clusters=8, random_state=0, n_init=\"auto\").fit(k_means_data)\n# k_values = kmeans.predict(k_means_data)\n\nfeatures[\"k_means\"] = reg_values\n\n\n\nfeatures = convert_strings(features)\n\nx_train, x_valid, y_train, y_valid = train_test_split(features, labels, train_size=0.6, random_state=1)\n\nlgb_train = lgb.Dataset(x_train, label=y_train)\nlgb_valid = lgb.Dataset(x_valid, label=y_valid)\n\n\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 3,\n    \"num_leaves\": 31,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.9,\n    \"bagging_fraction\": 0.8,\n    \"bagging_freq\": 5,\n    \"n_estimators\": 2000,\n    \"verbose\": -1,\n}\n\ngbm = lgb.train(\n    params,\n    lgb_train,\n    valid_sets=lgb_valid,\n    callbacks=[lgb.log_evaluation(50), lgb.early_stopping(10)]\n)\n# lgb_valid = lgb.Dataset(X_valid, label=y_valid, reference=lgb_train)","metadata":{"execution":{"iopub.status.busy":"2024-04-20T01:27:12.109942Z","iopub.execute_input":"2024-04-20T01:27:12.110389Z","iopub.status.idle":"2024-04-20T01:36:08.058310Z","shell.execute_reply.started":"2024-04-20T01:27:12.110341Z","shell.execute_reply":"2024-04-20T01:36:08.057167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}