{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":175709972,"sourceType":"kernelVersion"}],"dockerImageVersionId":30775,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nfrom pathlib import Path\nimport subprocess\nimport os\nimport gc\nfrom glob import glob\n\nimport joblib\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom datetime import datetime\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nROOT = '/kaggle/input/home-credit-credit-risk-model-stability'\n\n\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\nclass Pipeline:\n    @staticmethod\n    def set_table_dtypes(df): #Standardize the dtype.\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))            \n\n        return df\n    \n    @staticmethod\n    def handle_dates(df): #Change the feature for D to the difference in days from date_decision.\n        for col in df.columns:\n            if (col[-1] in (\"D\",)) and ('count' not in col):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n                df = df.with_columns(pl.col(col).dt.total_days())\n                \n        df = df.drop(\"date_decision\", \"MONTH\")\n\n        return df\n    \n    @staticmethod\n    def filter_cols(df): #Remove those with an average is_null exceeding 0.95 and those that do not fall within the range 1 < nunique < 200.\n        for col in df.columns:\n            # if col in [\"decision_month\", \"decision_weekday\"]:\n            #     df = df.drop(col)\n            #     continue\n            # if ('amtde' in col) or ('bureau_b2' in col): # for ohter option\n            #     continue\n\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n                if isnull > 0.95:\n                    df = df.drop(col)\n\n        for col in df.columns:\n            # if '_depth2_' in col:\n            #     continue\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\", ]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n\n                if (freq == 1) | (freq > 50):#50 #len(df) * 0.20): # 95 # fe4 down at fq20\n                    df = df.drop(col)\n            \n            # eliminate yaer, month feature\n            # 644\n            if (col[-1] not in [\"P\", \"A\", \"L\", \"M\"]) and (('month_' in col) or ('year_' in col)):# or ('num_group' in col):\n            # if (('month_' in col) or ('year_' in col)):# or ('num_group' in col):\n                df = df.drop(col)\n\n        return df\n\nclass Aggregator:\n    @staticmethod\n    def num_expr(df):\n        cols = [col for col in df.columns if (col[-1] in (\"T\",\"L\",\"M\",\"D\",\"P\",\"A\")) or (\"num_group\" in col)]\n\n        expr_1 = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_2 = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        # expr_3 = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n        # expr_3 = [pl.var(col).alias(f\"var_{col}\") for col in cols]+ [pl.sum(col).alias(f\"sum_{col}\") for col in cols]\n        # expr_3 = [pl.last(col).alias(f\"last_{col}\") for col in cols] #+ \\\n        #     [pl.first(col).alias(f\"first_{col}\") for col in cols] + \\\n        #     [pl.mean(col).alias(f\"mean_{col}\") for col in cols] + \\\n        #     [pl.std(col).alias(f\"std_{col}\") for col in cols]\n        # expr_3 = [pl.count(col).alias(f\"count_{col}\") for col in cols]\n\n        cols2 = [col for col in df.columns if col[-1] in (\"L\", \"A\")]\n        expr_3 = [pl.mean(col).alias(f\"mean_{col}\") for col in cols2] + [pl.std(col).alias(f\"std_{col}\") for col in cols2] + \\\n            [pl.sum(col).alias(f\"sum_{col}\") for col in cols2] + [pl.median(col).alias(f\"median_{col}\") for col in cols2] # + \\\n            # [pl.first(col).alias(f\"first_{col}\") for col in cols2] + [pl.last(col).alias(f\"last_{col}\") for col in cols2]\n        \n        # BAD\n        # cols3 = [col for col in df.columns if col[-1] in (\"A\")]\n        # expr_4 = [pl.col(col).fill_null(strategy=\"zero\").apply(lambda x: x.max() - x.min()).alias(f\"max-min_gap_{col}\") \n        #           for col in cols3]\n        return expr_1 + expr_2 + expr_3 # + [pl.col(col).diff().last().alias(f\"diff-last_{col}\") for col in cols3] # + expr_4\n    \n    @staticmethod\n    def applprev2_exprs(df):\n        cols = [col for col in df.columns if \"num_group\" not in col]\n        # expr_1 = [pl.max(col).alias(f\"max_{col}\") for col in cols] + [pl.min(col).alias(f\"min_{col}\") for col in cols] \n        expr_2 = [pl.first(col).alias(f\"first_{col}\") for col in cols]#  + [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        return []#expr_2\n\n    @staticmethod\n    def bureau_a1(df):\n        cols = [col for col in df.columns if (col[-1] in (\"T\",\"L\",\"M\",\"D\",\"P\",\"A\")) or (\"num_group\" in col)]\n        expr_1 = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_2 = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n\n        cols2 = [\n            # bad\n        'annualeffectiverate_199L', 'annualeffectiverate_63L',\n        'contractsum_5085717L', \n        'credlmt_230A', 'credlmt_935A',\n        # 'debtoutstand_525A', 'debtoverdue_47A', 'dpdmax_139P', 'dpdmax_757P',\n    #    'instlamount_768A', 'instlamount_852A',\n    #    'interestrate_508L', 'monthlyinstlamount_332A',\n    #    'monthlyinstlamount_674A', \n            # good?\n       'nominalrate_281L', 'nominalrate_498L',\n       'numberofcontrsvalue_258L', 'numberofcontrsvalue_358L',\n       'numberofinstls_229L', 'numberofinstls_320L',\n       'numberofoutstandinstls_520L', 'numberofoutstandinstls_59L',\n       'numberofoverdueinstlmax_1039L', 'numberofoverdueinstlmax_1151L',\n       'numberofoverdueinstls_725L', 'numberofoverdueinstls_834L',\n            # bad?\n    #    'outstandingamount_354A', 'outstandingamount_362A', 'overdueamount_31A',\n    #    'overdueamount_659A', 'overdueamountmax2_14A', 'overdueamountmax2_398A',\n    #    'overdueamountmax_155A', 'overdueamountmax_35A',\n        # bad ?\n    #    'periodicityofpmts_1102L', 'periodicityofpmts_837L',\n    #    'prolongationcount_1120L', 'prolongationcount_599L',\n        # 520?\n    #    'residualamount_488A', 'residualamount_856A', 'totalamount_6A',\n    #    'totalamount_996A', 'totaldebtoverduevalue_178A',\n    #    'totaldebtoverduevalue_718A', 'totaloutstanddebtvalue_39A',\n    #    'totaloutstanddebtvalue_668A',\n       ]\n\n        # .697\n        # expr_3 = [pl.mean(col).alias(f\"mean_{col}\") for col in cols2] + [pl.std(col).alias(f\"std_{col}\") for col in cols2]\n        \n        # .696\n        # expr_3 = [pl.mean(col).alias(f\"mean_{col}\") for col in cols2]\n\n        # .697\n        # expr_3 = [pl.std(col).alias(f\"std_{col}\") for col in cols2]\n        \n        # .6985\n        # expr_3 = [pl.sum(col).alias(f\"sum_{col}\") for col in cols2] + [pl.median(col).alias(f\"median_{col}\") for col in cols2]\n\n        # .696\n        # expr_3 = [pl.sum(col).alias(f\"sum_{col}\") for col in cols2] \n\n        # .6981\n        # expr_3 = [pl.median(col).alias(f\"median_{col}\") for col in cols2]\n\n        # .696\n        # expr_3 = [pl.first(col).alias(f\"first_{col}\") for col in cols2] + [pl.last(col).alias(f\"last_{col}\") for col in cols2] # + \\\n        \n        # .696\n        # expr_3 = [pl.std(col).alias(f\"std_{col}\") for col in cols2] + [pl.median(col).alias(f\"median_{col}\") for col in cols2]\n\n        # .699\n        # expr_3 = [pl.mean(col).alias(f\"mean_{col}\") for col in cols2] + [pl.std(col).alias(f\"std_{col}\") for col in cols2] + \\\n        #     [pl.sum(col).alias(f\"sum_{col}\") for col in cols2] + [pl.median(col).alias(f\"median_{col}\") for col in cols2]\n\n        expr_3 = [pl.mean(col).alias(f\"mean_{col}\") for col in cols2] + [pl.std(col).alias(f\"std_{col}\") for col in cols2] + \\\n            [pl.sum(col).alias(f\"sum_{col}\") for col in cols2] + [pl.median(col).alias(f\"median_{col}\") for col in cols2] + \\\n            [pl.first(col).alias(f\"first_{col}\") for col in cols2] # + [pl.last(col).alias(f\"last_{col}\") for col in cols2] # not applied\n        \n        \n\n        # expr_3 = [pl.col(col).fill_null(strategy=\"zero\").apply(lambda x: x.max() - x.min()).alias(f\"max-min_gap_depth2_{col}\") for col in cols2]\n        return expr_1 + expr_2 + expr_3    \n\n\n    @staticmethod\n    def bureau_b1(df):  # 0.95에서 미적용 중 # 36500\n        # cols = [col for col in df.columns if (col[-1] in (\"T\",\"L\",\"M\",\"D\",\"P\",\"A\")) or (\"num_group\" in col)]\n\n        # expr_1 = [pl.max(col).alias(f\"bureau_b1_max_{col}\") for col in cols]\n        # expr_2 = [pl.min(col).alias(f\"bureau_b1_min_{col}\") for col in cols]\n\n        # return expr_1 + expr_2 #  + expr_3\n        return []\n    \n    \n    @staticmethod\n    def bureau_b2(df):  # 0.95에서 미적용 중 # 36500\n        # cols = [col for col in df.columns if (col[-1] in (\"T\",\"L\",\"M\",\"D\",\"P\",\"A\")) or (\"num_group\" in col)]\n\n        # expr_1 = [pl.max(col).alias(f\"bureau_b2_max_{col}\") for col in cols]\n        # expr_2 = [pl.min(col).alias(f\"bureau_b2_min_{col}\") for col in cols]\n\n        # return expr_1 + expr_2 #  + expr_3\n        return []\n\n\n    @staticmethod\n    def deposit_exprs(df):\n        cols = [col for col in df.columns if (col[-1] in (\"T\",\"L\",\"M\",\"D\",\"P\",\"A\")) or (\"num_group\" in col)]\n        expr_1 = [pl.max(col).alias(f\"max_{col}\") for col in cols] + [pl.min(col).alias(f\"min_{col}\") for col in cols] # + \\\n            # [pl.last(col).alias(f\"last_{col}\") for col in cols]\n            # [pl.mean(col).alias(f\"mean_{col}\") for col in cols] # + \\\n            # [pl.std(col).alias(f\"std_{col}\") for col in cols]  + \\\n             \n            # [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        # expr_2 = [pl.first('openingdate_857D').alias(f'first_openingdate_857D')] + [pl.last('openingdate_857D').alias(f'last_openingdate_857D')]\n        \n        return expr_1 # + expr_2 #+ expr_ngmax\n\n    @staticmethod\n    def debitcard_exprs(df):\n        # cols = [col for col in df.columns if (col[-1] in [\"A\"])]\n        cols = [col for col in df.columns if (col[-1] in (\"T\",\"L\",\"M\",\"D\",\"P\",\"A\")) or (\"num_group\" in col)]\n        expr_1 = [pl.max(col).alias(f\"max_{col}\") for col in cols] + [pl.min(col).alias(f\"min_{col}\") for col in cols] \n            # [pl.mean(col).alias(f\"mean_{col}\") for col in cols] + \\\n            # [pl.std(col).alias(f\"std_{col}\") for col in cols]\n        # expr_2 = [pl.first('openingdate_857D').alias(f'first_openingdate_857D')] + [pl.last('openingdate_857D').alias(f'last_openingdate_857D')]\n        \n        return expr_1 # + expr_2 #+ expr_ngmax\n        # return expr_1\n\n\n    @staticmethod\n    def person_expr(df):\n        cols1 = ['empl_employedtotal_800L', 'empl_employedfrom_271D', 'empl_industry_691L', \n                 'familystate_447L', 'incometype_1044T', 'sex_738L', 'housetype_905L', 'housingtype_772L',\n                 'isreference_387L', 'birth_259D', ]\n        # cols1 = [col for col in df.columns]\n        expr_1 = [pl.first(col).alias(f\"first_{col}\") for col in cols1]\n        \n        expr_2 = [pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"), \n                  pl.col(\"mainoccupationinc_384A\").filter(pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")]\n        \n        # No Effect ...\n        # cols = ['personindex_1023L', 'persontype_1072L', 'persontype_792L']\n        # expr_3 = [pl.col(col).last().alias(f\"last_{col}\") for col in cols] + [pl.col(col).drop_nulls().mean().alias(f\"mean_{col}\") for col in cols]\n\n        # cols2 = [col for col in df.columns if col not in cols1]\n        # expr_4 = [pl.max(col).alias(f\"max_{col}\") for col in cols2] + [pl.min(col).alias(f\"min_{col}\") for col in cols2] #  good at cv, bad at lb ?\n            # [pl.col(col).drop_nulls().last().alias(f\"last_{col}\") for col in cols2] + [pl.col(col).drop_nulls().first().alias(f\"first_{col}\") for col in cols2] # no effect\n\n        return expr_1 + expr_2 # + expr_4 # + expr_3\n    \n    @staticmethod\n    def person_2_expr(df):\n        # cols = [col for col in df.columns]\n        cols = ['empls_economicalst_849M', 'empls_employedfrom_796D', 'empls_employer_name_740M'] # + \\\n            # ['relatedpersons_role_762T', 'conts_role_79M']\n            # ['addres_district_368M', 'addres_role_871L', 'addres_zip_823M']\n\n        expr_1 = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_2 = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n\n        # BAD\n        # expr_ngc = [pl.count(\"num_group2\").alias(f\"count_num_group2\")]\n        # cols2 = [col for col in df.columns if (col in (\"num_group1\", \"num_group2\"))]\n        # expr_ngmax = [pl.min(col).alias(f\"min_{col}\") for col in cols2] + [pl.max(col).alias(f\"max_{col}\") for col in cols2]\n\n        # cols2 = [col for col in df.columns if col not in cols]\n        # # expr_3 = [pl.max(col).alias(f\"max_{col}\") for col in cols2] + [pl.min(col).alias(f\"min_{col}\") for col in cols2] # no effect\n        # expr_3 = [pl.col(col).drop_nulls().last().alias(f\"last_{col}\") for col in cols2] # no effect\n\n        return expr_1 + expr_2 # + expr_3# + expr_ngc \n\n    @staticmethod\n    def other_expr(df):\n        expr_1 = [pl.first(col).alias(f\"__other_{col}\") for col in df.columns if ('num_group' not in col) and (col != 'case_id')]\n        # cols1 = ['amtdepositbalance_4809441A', 'amtdepositincoming_4809444A', 'amtdepositoutgoing_4809442A']\n        # expr_1 = [pl.last(col).alias(f\"last_{col}\") for col in cols1]\n        # cols2 = ['amtdebitincoming_4809443A', 'amtdebitoutgoing_4809440A']\n        # expr_3 = [(pl.col('amtdebitincoming_4809443A') - pl.col('amtdebitoutgoing_4809440A')).alias('amtdebit_incoming-outgoing')]\n        return expr_1 # + expr_2 + expr_3\n    \n    \n    @staticmethod\n    def tax_a_exprs(df):\n        cols = [col for col in df.columns if (col[-1] in (\"T\",\"L\",\"M\",\"D\",\"P\",\"A\")) or (\"num_group\" in col)]\n        expr_1 = [pl.max(col).alias(f\"max_{col}\") for col in cols] + [pl.min(col).alias(f\"min_{col}\") for col in cols] + \\\n            [pl.last(col).alias(f\"last_{col}\") for col in cols] + \\\n            [pl.first(col).alias(f\"first_{col}\") for col in cols] + \\\n            [pl.mean(col).alias(f\"mean_{col}\") for col in cols] + \\\n            [pl.std(col).alias(f\"std_{col}\") for col in cols]\n        # expr_1 = [pl.max(col).alias(f\"max_{col}\") for col in ['amount_4527230A', 'recorddate_4527225D', 'num_group1']] + \\\n        #     [pl.min(col).alias(f\"min_{col}\") for col in ['amount_4527230A', 'recorddate_4527225D', ]] + \\\n        #     [pl.mean(col).alias(f\"mean_{col}\") for col in ['amount_4527230A']] + \\\n        #     [pl.std(col).alias(f\"std_{col}\") for col in ['amount_4527230A']] + \\\n        #     [pl.last(col).alias(f\"last_{col}\") for col in ['amount_4527230A', 'recorddate_4527225D', 'name_4527232M']] + \\\n        #     [pl.first(col).alias(f\"first_{col}\") for col in ['amount_4527230A', 'recorddate_4527225D', 'name_4527232M']] # BAD?\n\n#         expr_4 = [pl.col(col).fill_null(strategy=\"zero\").apply(lambda x: x.max() - x.min()).alias(f\"max-min_gap_depth2_{col}\") for col in ['amount_4527230A']]\n#         expr_4 = [\n#     (pl.max(pl.col(col)) - pl.min(pl.col(col))).alias(f\"max-min_gap_depth2_{col}\") \n#     for col in ['amount_4527230A']\n# ]\n        expr_4 = [\n            (pl.max(col) - pl.min(col)).alias(f\"max-min_gap_depth2_{col}\") \n            for col in ['amount_4527230A']  # 你可以替换为其他列\n        ]\n        return expr_1 + expr_4\n\n\n    @staticmethod\n    def bureau_a2(df): # 122만\n        # cols = ['collater_valueofguarantee_1124L', 'pmts_dpd_1073P', 'pmts_overdue_1140A',]\n        cols = [col for col in df.columns if (col[-1] in (\"T\",\"L\",\"M\",\"D\",\"P\",\"A\")) or (\"num_group\" in col)]\n\n        expr_1 = [pl.max(col).alias(f\"max_depth2_{col}\") for col in cols]\n        expr_2 = [pl.min(col).alias(f\"min_depth2_{col}\") for col in cols]\n        expr_3 = [pl.mean(col).alias(f\"mean_depth2_{col}\") for col in cols] + \\\n            [pl.std(col).alias(f\"std_{col}\") for col in cols]\n        # expr_ngs = [pl.max(col).alias(f\"max_{col}\") for col in ['num_group1', 'num_group2', ]]\n\n#         expr_4 = [pl.col(col).fill_null(strategy=\"zero\").apply(lambda x: x.max() - x.min()).alias(f\"max-min_gap_depth2_{col}\") for col in ['collater_valueofguarantee_1124L', 'pmts_dpd_1073P', 'pmts_overdue_1140A',]]\n#         expr_4 = [\n#     (pl.max(pl.col(col)) - pl.min(pl.col(col))).alias(f\"max-min_gap_depth2_{col}\") \n#     for col in ['collater_valueofguarantee_1124L', 'pmts_dpd_1073P', 'pmts_overdue_1140A',]\n# ]\n        expr_4 = [\n            (pl.max(col) - pl.min(col)).alias(f\"max-min_gap_depth2_{col}\") \n            for col in ['collater_valueofguarantee_1124L', 'pmts_dpd_1073P', 'pmts_overdue_1140A',]  # 你可以替换为其他列\n        ]\n        expr_ngc = [pl.count(\"num_group2\").alias(f\"count_depth2_a2_num_group2\")]\n\n        # expr_5 = [pl.last(col).alias(f\"last_{col}\") for col in cols] + \\\n        #     [pl.first(col).alias(f\"first_{col}\") for col in cols] + \\\n        #     [pl.std(col).alias(f\"std_{col}\") for col in cols]\n\n        return expr_1 + expr_2 + expr_3 + expr_4 + expr_ngc # + expr_5\n    \n    @staticmethod\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:28:56.254321Z","iopub.execute_input":"2024-10-27T06:28:56.254733Z","iopub.status.idle":"2024-10-27T06:29:01.102322Z","shell.execute_reply.started":"2024-10-27T06:28:56.254692Z","shell.execute_reply":"2024-10-27T06:29:01.101016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    \n# %% [code] {\"jupyter\":{\"outputs_hidden\":false},\"execution\":{\"iopub.status.busy\":\"2024-05-02T02:35:12.546131Z\",\"iopub.execute_input\":\"2024-05-02T02:35:12.547239Z\",\"iopub.status.idle\":\"2024-05-02T02:35:12.573617Z\",\"shell.execute_reply.started\":\"2024-05-02T02:35:12.547191Z\",\"shell.execute_reply\":\"2024-05-02T02:35:12.572323Z\"}}\n\ndef agg_by_case(path, df):\n    path = str(path)\n    if '_applprev_1' in path:\n        df = df.sort(\"num_group1\").group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n\n#     elif '_applprev_2' in path:\n#         df = df.group_by(\"case_id\").agg(Aggregator.applprev2_exprs(df))\n\n    elif '_credit_bureau_a_1' in path:\n        df = df.sort(\"num_group1\").group_by(\"case_id\").agg(Aggregator.bureau_a1(df))\n\n    elif '_credit_bureau_b_1' in path:\n        df = df.sort(\"num_group1\").group_by(\"case_id\").agg(Aggregator.bureau_b1(df))\n\n    elif '_deposit_1' in path:\n        df = df.sort(\"num_group1\").group_by(\"case_id\").agg(Aggregator.deposit_exprs(df))\n    elif '_debitcard_1' in path:\n        df = df.sort(\"num_group1\").group_by(\"case_id\").agg(Aggregator.debitcard_exprs(df))\n        \n    elif '_tax_registry_a' in path:\n        df = df.sort(\"num_group1\").group_by(\"case_id\").agg(Aggregator.tax_a_exprs(df))\n    elif '_tax_registry_b' in path:\n        df = df.sort(\"num_group1\").group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n    elif '_tax_registry_c' in path:\n        df = df.sort(\"num_group1\").group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        \n    elif '_other_1' in path:\n        df = df.sort(\"num_group1\").group_by(\"case_id\").agg(Aggregator.other_expr(df))\n    elif '_person_1' in path:\n        df = df.sort(\"num_group1\").group_by(\"case_id\").agg(Aggregator.person_expr(df))\n    elif '_person_2' in path:\n        df = df.group_by(\"case_id\").agg(Aggregator.person_2_expr(df))\n\n    elif '_credit_bureau_a_2' in path:\n        df = df.group_by(\"case_id\").agg(Aggregator.bureau_a2(df))\n    elif '_credit_bureau_b_2' in path:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n    \n    return df\n\ndef read_file(path, depth=None): \n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    \n    if depth in [1, 2]:\n        df = agg_by_case(path, df)\n    \n    return df\n\ndef read_files(regex_path, depth=None):\n    print(regex_path)\n    chunks = []\n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        if depth in [1, 2]:\n            df = agg_by_case(path, df)\n        chunks.append(df)\n        #     del df\n        #     gc.collect()\n        #     print('delete chunk')\n        \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    \n    return df\n\ndef feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base.with_columns(\n            decision_month = pl.col(\"date_decision\").dt.month(),\n            decision_weekday = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n        \n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n        \n    df_base = df_base.pipe(Pipeline.handle_dates)\n    return df_base\n\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    print(df_data.info())\n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n        # cat_cols = [c for c in cat_cols if 'diff_' not in c]\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data, cat_cols\n\n# %% [code] {\"jupyter\":{\"outputs_hidden\":false},\"execution\":{\"iopub.status.busy\":\"2024-05-02T02:35:12.575798Z\",\"iopub.execute_input\":\"2024-05-02T02:35:12.576686Z\",\"iopub.status.idle\":\"2024-05-02T02:35:12.593361Z\",\"shell.execute_reply.started\":\"2024-05-02T02:35:12.576632Z\",\"shell.execute_reply\":\"2024-05-02T02:35:12.591807Z\"}}\nROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\n\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR        = ROOT / \"parquet_files\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:29:01.105135Z","iopub.execute_input":"2024-10-27T06:29:01.105911Z","iopub.status.idle":"2024-10-27T06:29:01.132016Z","shell.execute_reply.started":"2024-10-27T06:29:01.105850Z","shell.execute_reply":"2024-10-27T06:29:01.130717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_store_train = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n        \n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),   \n    ],\n    \"depth_2\": [\n        read_files(TRAIN_DIR / \"train_applprev_2.parquet\", 2),#新加\n        read_file(TRAIN_DIR / \"train_person_2.parquet\", 2),#新加\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n        \n    ]\n}\n\ndf_train = feature_eng(**data_store_train)\nprint(\"train data shape:\\t\", df_train.shape)\ndf_train.write_parquet(\"train.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:29:01.133805Z","iopub.execute_input":"2024-10-27T06:29:01.134316Z","iopub.status.idle":"2024-10-27T06:33:51.833320Z","shell.execute_reply.started":"2024-10-27T06:29:01.134257Z","shell.execute_reply":"2024-10-27T06:33:51.832297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_store_test = {\n#     \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n#     \"depth_0\": [\n#         read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n#         read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n        \n#     ],\n#     \"depth_1\": [\n#         read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n#         read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n#         read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n#         read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n#         read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n#         read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n#         read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n#         read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n#         read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n#         read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),   \n#     ],\n#     \"depth_2\": [\n#         read_files(TEST_DIR / \"test_applprev_2.parquet\", 2),#新加\n#         read_file(TEST_DIR / \"test_person_2.parquet\", 2),#新加\n#         read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n#         read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n        \n#     ]\n# }\n# # %% [code] {\"jupyter\":{\"outputs_hidden\":false},\"execution\":{\"iopub.status.busy\":\"2024-05-02T02:35:13.337192Z\",\"iopub.execute_input\":\"2024-05-02T02:35:13.337534Z\",\"iopub.status.idle\":\"2024-05-02T02:35:13.454908Z\",\"shell.execute_reply.started\":\"2024-05-02T02:35:13.337506Z\",\"shell.execute_reply\":\"2024-05-02T02:35:13.452272Z\"}}\n# df_test = feature_eng(**data_store_test)\n\n# print(\"test data shape:\\t\", df_test.shape)\n# df_test.write_parquet(\"test.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:33:51.834912Z","iopub.execute_input":"2024-10-27T06:33:51.835437Z","iopub.status.idle":"2024-10-27T06:33:51.842829Z","shell.execute_reply.started":"2024-10-27T06:33:51.835380Z","shell.execute_reply":"2024-10-27T06:33:51.841598Z"},"trusted":true},"execution_count":null,"outputs":[]}]}