{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8535067,"sourceType":"datasetVersion","datasetId":4926665}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, pickle, gc, time, math, re\nimport pandas as pd\nimport numpy as np\nimport polars as pl\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import Normalizer\n\ntrain_test = 'test'\n\non_kaggle = True\non_gcp = False\ndata_dir = '/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/'\nprefit_dir = '/kaggle/input/home-credit-data/'\noutdir = '/kaggle/input/home-credit-data/'\nrun_full = False\nforce_unsample_on_local = False\nadjust_inflation = False\n\ndef load1(path):\n    with open( path, 'rb' ) as pkl_file:\n        obj = pickle.load(pkl_file)\n    return obj","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-28T00:34:10.486079Z","iopub.execute_input":"2024-05-28T00:34:10.486973Z","iopub.status.idle":"2024-05-28T00:34:13.642775Z","shell.execute_reply.started":"2024-05-28T00:34:10.486919Z","shell.execute_reply":"2024-05-28T00:34:13.641614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if on_gcp:\n    os.system(\"gsutil cp gs://kaggle-417721/train.zip train.zip && unzip train.zip && rm train.zip\")\n    os.system(\"gsutil cp gs://kaggle-417721/cpiu.csv cpiu.csv\")\n\nagg = {\n    'applprev_1': ['case_id'],\n    'applprev_2': ['case_id'],\n    'tax_registry_a_1': ['case_id'],\n    'credit_bureau_b_1': ['case_id'],\n    'credit_bureau_b_2': ['case_id'],\n    'deposit_1': ['case_id'],\n    'person_1': ['case_id'],\n    'person_2': ['case_id'],\n    'debitcard_1': ['case_id'],\n    'tax_registry_b_1': ['case_id'],\n    'tax_registry_c_1': ['case_id'],\n    'credit_bureau_a_1': ['case_id'],\n    'credit_bureau_a_2': ['case_id']\n}\nagg_only = ['credit_bureau_a_1', 'credit_bureau_a_2']\n\ndropyearcols = {\n    'base': [],\n    'applprev_1': [],\n    'applprev_2': [],\n    'credit_bureau_a_1': [\n        'dpdmaxdateyear_596T', 'dpdmaxdateyear_896T', 'overdueamountmaxdateyear_2T', 'overdueamountmaxdateyear_994T'\n        # month:\n        'dpdmaxdatemonth_442T', 'dpdmaxdatemonth_89T', 'overdueamountmaxdatemonth_284T', 'overdueamountmaxdatemonth_365T'\n    ],\n    'credit_bureau_a_2': [\n        'pmts_year_1139T', 'pmts_year_507T'\n        'pmts_month_158T', 'pmts_month_706T'\n    ],\n    'credit_bureau_b_1': [\n        'dpdmaxdateyear_742T', 'overdueamountmaxdateyear_432T', \n        'dpdmaxdatemonth_804T', 'overdueamountmaxdatemonth_494T'\n    ],\n    'credit_bureau_b_2': [],\n    'debitcard_1': [],\n    'deposit_1': [],\n    'other_1': [],\n    'person_1': [],\n    'person_2': [],\n    'static_0': [],\n    'static_cb': [],\n    'tax_registry_a_1': [],\n    'tax_registry_b_1': [],\n    'tax_registry_c_1': []\n}\n\nclass Aggregator:\n    \n    def cols_of_type(of_type, coltypes, group_cols):\n        cols = [col for  col in coltypes[of_type] if col not in group_cols + ['case_id']]\n        return cols        \n    \n    def nums(df, group_cols, coltypes):\n        cols = Aggregator.cols_of_type('nums', coltypes, group_cols)\n        exprs = [pl.min(col).alias(f\"agg_min_{col}\").cast(pl.Float32)for col in cols] + \\\n            [pl.mean(col).alias(f\"agg_mean_{col}\").cast(pl.Float32) for col in cols] + \\\n            [pl.max(col).alias(f\"agg_max_{col}\").cast(pl.Float32) for col in cols]\n        return exprs\n    \n    def dates(df, group_cols, coltypes):\n        cols = Aggregator.cols_of_type('dates', coltypes, group_cols)\n        exprs = [pl.min(col).alias(f\"agg_min_{col}\") for col in cols] + \\\n            [pl.max(col).alias(f\"agg_max_{col}\") for col in cols]\n        return exprs\n    \n    def strings(df, group_cols, coltypes):\n        cols = Aggregator.cols_of_type('strings', coltypes, group_cols)\n        exprs = [pl.n_unique(col).alias(f\"agg_n_unique_{col}\").cast(pl.Int32) for col in cols]\n        return exprs\n    \n    def bools(df, group_cols, coltypes):\n        cols = Aggregator.cols_of_type('bools', coltypes, group_cols)\n        exprs = [pl.mean(col).alias(f\"agg_mean_{col}\").cast(pl.Float32) for col in cols]\n        return exprs\n    \n    def get_exprs(df, filename_key, group_cols):\n\n        # get column types.\n        if filename_key not in agg_col_types.keys():\n            coltypes = {'nums': [], 'dates': [], 'strings': [], 'bools': []}\n            for i in range(len(df.columns)):\n                itype = str(df.dtypes[i])\n                if itype in ['Int32', 'Float32']:\n                    coltypes['nums'].append(df.columns[i])\n                elif itype == 'Date':\n                    coltypes['dates'].append(df.columns[i])\n                elif itype == 'String':\n                    coltypes['strings'].append(df.columns[i])\n                elif itype == 'Boolean':\n                    coltypes['bools'].append(df.columns[i])\n                elif itype == 'Null':\n                    return []\n                else:\n                    raise Exception(f'Unhandled type: {itype}')\n            agg_col_types[filename_key] = coltypes\n        else:\n            coltypes = agg_col_types[filename_key]\n    \n        exprs = Aggregator.nums(df, group_cols, coltypes) + \\\n                Aggregator.dates(df, group_cols, coltypes) + \\\n                Aggregator.bools(df, group_cols, coltypes) + \\\n                Aggregator.strings(df, group_cols, coltypes)\n\n        return exprs\n\ndef set_types(idt, filename_key):\n    \n    if filename_key not in col_types.keys():\n        col_types[filename_key] = {'date': [], 'float': [], 'int': [], 'string': []}        \n    \n    for col in idt.columns:\n        \n        if on_kaggle:\n            \n            if (col in col_types[filename_key]['date']):\n                    idt = idt.with_columns(idt.get_column(col).cast(pl.Date).alias(col))    \n            elif (col in col_types[filename_key]['float']):\n                idt = idt.with_columns((idt[col]).cast(pl.Float32, strict=False).alias(col))            \n            elif (col in col_types[filename_key]['int']):\n                idt = idt.with_columns(idt[col].cast(pl.Int32, strict=False).alias(col)) \n            # ignore aggregate columns. these already have the correct types.\n            elif re.match(r'^agg_', col):\n                continue              \n            else:\n                idt = idt.with_columns(idt[col].cast(pl.String).alias(col))\n                \n        else:\n            \n            datecheck = False\n            if str(idt[col].dtype) == 'String':\n                colsample = idt.get_column(col).drop_nans().drop_nulls()\n                nsamples = 10 if len(colsample) > 10 else len(colsample)\n                colsample = colsample.sample(nsamples).to_list()\n                datecheck = np.all([True if re.fullmatch(r'[0-9]{4}-[0-9]{2}-[0-9]{2}', x) else False for x in colsample])\n    \n            # convert to types to save memory.        \n            # convert num_groups to bet handled list strings.\n            if ('num_group' in col) and ('n_unique' not in col):\n                idt = idt.with_columns(idt.get_column(col).cast(pl.String).alias(col))\n                col_types[filename_key]['string'].append(col)\n            \n            # dates.\n            elif datecheck:\n                    idt = idt.with_columns(idt.get_column(col).cast(pl.Date).alias(col))\n                    col_types[filename_key]['date'].append(col)\n            \n            # floats.\n            elif str(idt[col].dtype) == 'Float64':\n                    # idt = idt.with_columns((idt[col] * 100).cast(pl.Int32, strict=False).alias(col))\n                    idt = idt.with_columns((idt[col]).cast(pl.Float32, strict=False).alias(col))\n                    col_types[filename_key]['float'].append(col)\n            \n            # ints\n            elif (str(idt[col].dtype) == 'Int64') or (col == 'case_id'):\n                idt = idt.with_columns(idt[col].cast(pl.Int32, strict=False).alias(col))\n                col_types[filename_key]['int'].append(col)\n                \n            # ignore aggregate columns. these already have the correct types.\n            elif re.match(r'^agg_', col):\n                continue\n                \n            # everything else is a string.\n            else:\n                idt = idt.with_columns(idt[col].cast(pl.String).alias(col))\n                col_types[filename_key]['string'].append(col)\n        \n            \n    return idt\n\ndef sort_aggregate(idt, filename_key, group_cols):\n\n    idt_agg = []\n    group_cols = [x for x in group_cols if x in idt.columns]\n    extra_group_cols = [x for x in group_cols if x != 'case_id']\n    \n    # sort and create aggregation.\n    if filename_key in agg.keys():\n        for col in extra_group_cols:\n            if (col != 'case_id') and (str(idt[col].dtype) == \"Null\"):\n                idt = idt.with_columns(idt.get_column(col).cast(pl.String).alias(col))\n        idt = idt.sort(extra_group_cols)\n        idt_agg = idt.group_by(group_cols)\n        idt_agg = idt_agg.agg(Aggregator.get_exprs(idt, filename_key, extra_group_cols))\n        idt_agg = set_types(idt_agg, filename_key + '_agg')\n        \n        # we need to add new dates to col_types.\n        if not on_kaggle:\n            for col in idt_agg.select(pl.col(pl.Date)).columns:\n                if col not in col_types[filename_key + '_agg']['date']:\n                    col_types[filename_key + '_agg']['date'].append(col)\n                    # print(f'added to date for [{filename_key + \"_agg\"}]: [{col}]')\n        \n    return idt, idt_agg    \n\ndef readfiles(train_test, filename_key):\n    \n    dt = []\n    dt_agg = []\n    only_agg = (filename_key in agg_only)\n\n    filedir = data_dir + train_test + '/'\n    dofiles = [filedir + x for x in os.listdir(filedir) if filename_key in x]\n    for i in range(len(dofiles)):\n        \n        print(f'reading [{dofiles[i]}]')\n        \n        if (i == 0) and (not on_kaggle):\n    \n            # read the first file to select columns, identify agg cols, etc.\n            idt = pl.read_parquet(dofiles[i])\n            # idt.sample(1000).write_csv('out/sample-csvs/' + filename_key + '.csv')\n            \n            idt = set_types(idt, filename_key)\n            \n            # ignore columns without enough values to be interesting.\n            # or columns with too many values.\n            if filename_key != 'base':\n    \n                colcount = idt.select(pl.col(pl.String))\n                colcount =  pl.DataFrame({\n                    'col': colcount.columns,\n                    'count':[list(colcount.select(pl.approx_n_unique(x)).to_numpy())[0][0] for x in colcount.columns]\n                })\n            \n                # lets keep these and drop them later. they might have large categories.\n                # dropcols = colcount.filter(pl.col('count') == 1)['col'].to_list() + \\\n                #     colcount.filter(pl.col('count') >= 1000)['col'].to_list()                    \n                # if len(dropcols) > 0: \n                #     print('dropped:')\n                #     print(colcount.filter(pl.col('col').is_in(dropcols)))\n                #     idt = idt.drop(dropcols) \n                #     colcount = colcount.filter(~pl.col('col').is_in(dropcols))\n        \n            # get extra group cols.\n            if filename_key in agg.keys():\n                colcount = colcount.filter(pl.col('count') > 1, pl.col('count') < 10)\n                iextra_group_cols = colcount.sort('count')['col'].to_list()\n                extra_group_cols[filename_key] = iextra_group_cols\n                if len(iextra_group_cols) > 0:\n                    print('extra_group_cols:')\n                    print(colcount)\n            \n            idt.drop(dropyearcols[filename_key])\n            use_cols[filename_key] = idt.columns\n \n            group_cols = [] if (filename_key not in agg) else (['case_id'] + iextra_group_cols)\n            idt, idt_agg = sort_aggregate(idt, filename_key, group_cols)\n            \n        else:\n    \n            group_cols = [] if (filename_key not in agg) else (['case_id'] + extra_group_cols[filename_key])\n            idt = pl.read_parquet(dofiles[i])\n            idt = set_types(idt, filename_key)\n            idt, idt_agg = sort_aggregate(idt, filename_key, group_cols)\n            \n            \n        if not only_agg:\n            # we need to drop cols_not_used after reading the data, since we may keep aggregations of them.\n            idt = idt[[x for x in idt.columns if x not in cols_not_used]]\n            print(f'data: {idt.shape}')            \n            dt.append(idt)\n        \n        if isinstance(idt_agg, pl.DataFrame):\n            idt_agg = idt_agg[[x for x in idt_agg.columns if x not in cols_not_used]]\n            print(f'agg: {idt_agg.shape}')\n            dt_agg.append(idt_agg)\n    \n    if len(dt) > 0:\n        dt = pl.concat(dt, how=\"vertical_relaxed\")\n    \n    if len(dt_agg) > 0:\n        dt_agg = pl.concat(dt_agg, how=\"vertical_relaxed\")\n\n    return dt, dt_agg\n\ndef load1(path):\n    with open(path, 'rb' ) as pkl_file:\n        obj = pickle.load(pkl_file)\n    return obj\n\ndef save1(obj, path):\n    with open(path, 'wb') as output:\n        pickle.dump(obj, output, pickle.HIGHEST_PROTOCOL)\n\ndef unlist(x):\n    flatten = lambda *n: (e for a in n\n        for e in (flatten(*a) if isinstance(a, (tuple, list)) else (a,)))\n    return list(flatten(x))\n\nif force_unsample_on_local or on_gcp or (not os.path.exists('out/sampled-data.pkl')) or (on_kaggle):\n\n    dt = {}\n\n    if on_kaggle:\n        \n        try:\n            prefit = load1(prefit_dir + 'prefit.pkl')\n        except:\n            prefit = load1(prefit_dir + 'sample/prefit.pkl')\n        \n        adjust_inflation = prefit['adjust_inflation']\n        cpiu = prefit['cpiu']\n        use_cols = prefit['use_cols']\n        extra_group_cols = prefit['extra_group_cols']\n        agg_col_types = prefit['agg_col_types']\n        dtinfo = prefit['dtinfo']\n        colinfo = prefit['colinfo']\n        levels = prefit['levels']\n        col_types = prefit['col_types']\n        prepca_cols = prefit['prepca_cols']\n        table_transformers = prefit['table_transformers']\n        pregroupcols = prefit['pregroupcols']\n        cols_not_used = prefit['cols_not_used']\n        dotables = [x for x in dtinfo.name if '_agg' not in x] + ['base']\n        dotables += [x.replace('_agg', '') for x in dtinfo.name if x.replace('_agg', '') in agg_only]\n        \n        # fixes for current data. \n        cols_not_used = [x for x in cols_not_used if x not in ['num_group1', 'num_group2']]\n        use_cols['applprev_2'].append('num_group1')\n        use_cols['applprev_2'].append('num_group2')\n        \n    else:\n        \n        cols_not_used = []\n        use_cols = {}\n        extra_group_cols = {}\n        agg_col_types = {}\n        col_types = {}\n    \n        dotables = [\n            'base',\n            'static_0', \n            'static_cb', \n            'applprev_1', \n            'applprev_2', \n            'credit_bureau_a_1', \n            'credit_bureau_a_2',\n            'credit_bureau_b_1', \n            'credit_bureau_b_2', \n            'deposit_1', \n            'person_1', \n            'person_2', \n            'debitcard_1',\n            'other_1', \n            'tax_registry_a_1',\n            'tax_registry_b_1', \n            'tax_registry_c_1', \n        ]\n    \n    for itype in dotables: \n        \n        idt, idt_agg = readfiles(train_test, itype)\n        \n        if isinstance(idt, pl.DataFrame):\n            dt[itype] = idt\n        \n        if isinstance(idt_agg, pl.DataFrame):\n            dt[itype + '_agg'] = idt_agg\n            \n        if itype == 'applprev_2':\n            if 'num_group1' not in idt.columns:\n                raise Exception('Did not find expected column: num_group1')\n            \n        del itype, idt, idt_agg\n        \n        gc.collect()\n    \n    all_cases = dt['base'].get_column('case_id').to_numpy()\n    init_all_cases = all_cases\n    if not on_kaggle: targets = dt['base'].get_column('target').to_numpy()\n\n    cpiu = pl.read_csv(outdir + 'cpiu.csv')\n    \nelse:\n    \n    # read sampled data.\n    # if 'sampled_data' not in globals():\n    sampled_data = load1('out/sampled-data.pkl')\n    cpiu = sampled_data['cpiu']\n    dt = sampled_data['tables']\n    col_types = sampled_data['col_types']\n    all_cases = sampled_data['case_ids']\n    targets = sampled_data['targets']\n    use_cols = sampled_data['use_cols'] \n    cols_not_used = sampled_data['cols_not_used']\n    extra_group_cols = sampled_data['extra_group_cols'] \n    agg_col_types = sampled_data['agg_col_types']\n    del sampled_data\n\n    # for table in dt.keys():\n    #     if table != 'base':\n    #         dt[table] = dt[table].drop('date_decision')\n\n# on gcp we sometimes run out of storage. delete the training data since we don't need it anymore.\nif on_gcp:\n    shutil.rmtree('train')\n\n# sample data to work with locally.\n# we want to keep all the defaults and bring in non-defaults at a ration of 3 to 1.\nif (not force_unsample_on_local) and (not on_gcp) and (not on_kaggle):\n    \n    print('sampling to 200K')\n    dt_sample = dt['base'].sample(200000)\n    targets = dt_sample['target'].to_numpy()\n    all_cases  = dt_sample['case_id'].to_numpy()\n    \n    for table in dt.keys():\n        dt[table] = dt[table].filter(pl.col('case_id').is_in(all_cases)) \n    \n    save1({\n        'cpiu': cpiu,\n        'tables': dt, \n        'case_ids': all_cases, \n        'targets': targets,\n        'col_types': col_types,\n        'cols_not_used': cols_not_used,\n        'use_cols': use_cols,\n        'extra_group_cols': extra_group_cols,\n        'agg_col_types': agg_col_types,\n    }, 'out/sampled-data.pkl')\n\nif not on_kaggle and sample_further:\n    sample_cases = np.unique(np.random.choice(a=all_cases, size=5000, replace=False).tolist() + [explore_case])\n    sample_idxs = np.isin(all_cases, sample_cases)\n    targets = targets[sample_idxs]\n    all_cases = all_cases[sample_idxs]\n    for table in dt.keys():\n        dt[table] = dt[table].filter(pl.col('case_id').is_in(all_cases))\n    del sample_cases, sample_idxs\n    \n# attach data_decision to all tables. this will cause it to always have differences calculated.\ntojoin = dt['base'].select(['case_id', 'date_decision'])\nfor table in dt.keys():\n    if table != 'base':\n        dt[table] = dt[table].join(tojoin, on = 'case_id', how = 'left')\n        \n# adjustments for year. the test data is in a 2-year period after the train data.\n# so anything relating to year will cause predictions to get worse over time. \n# drop columns related to year. \nadjfromyear = {\n    'applprev_1': 'creationdate_885D',\n    # 'applprev_2': # no dates.\n    'credit_bureau_a_1': 'dateofcredend_353D',\n    # 'credit_bureau_a_2': # no dates.\n    'credit_bureau_b_1': 'contractdate_551D',\n    # 'credit_bureau_b_2': 'pmts_date_1107D', # doesn't seem appropriate to adjust.\n    # 'debitcard_1': 'openingdate_857D' # doesn't seem appropriate to adjust.\n    # 'deposit_1' # doesn't seem appropriate to adjust.\n    # 'other_1' # no dates\n    # 'person_1':  # no dollars.\n    # 'person_2': [], # no dollars.\n    'static_0': 'date_decision',\n    'static_cb': 'responsedate_4917613D',\n    'tax_registry_a_1': 'recorddate_4527225D',\n    'tax_registry_b_1': 'deductiondate_4917603D',\n    'tax_registry_c_1': 'processingdate_168D'    \n}\nadjcolstoyear = {\n    'applprev_1': ['actualdpd_943P', 'annuity_853A', 'credamount_590A', 'currdebt_94A', 'downpmt_134A', 'outstandingdebt_522A'],\n    'credit_bureau_a_1': [\n        'contractsum_5085717L', 'debtoutstand_525A', 'debtoverdue_47A', 'instlamount_768A', 'instlamount_852A',\n        'monthlyinstlamount_332A', 'monthlyinstlamount_674A', 'overdueamountmax2_398A', 'overdueamountmax_35A', 'overdueamountmax_155A',\n        'residualamount_856A', 'totalamount_6A', 'totalamount_996A', 'totaldebtoverduevalue_178A', \n        'totaldebtoverduevalue_718A', 'totaloutstanddebtvalue_39A', 'totaloutstanddebtvalue_668A'\n    ],\n    'credit_bureau_b_1': [\n        'amount_1115A', 'debtvalue_227A', 'installmentamount_644A', \n        'installmentamount_833A', 'instlamount_892A', 'overdueamountmax_950A', 'residualamount_1093A', 'residualamount_127A', 'residualamount_3940956A',         \n        'totalamount_503A', 'totalamount_881A'\n    ],\n    'static_0': [\n        'amtinstpaidbefduel24m_4187115A', 'annuity_780A', 'annuitynextmonth_57A', 'avgdbddpdlast24m_3658932P', 'avgdbdtollast24m_4525197P',\n         'avgdpdtolclosure24_3658938P', 'avginstallast24m_3658937A', 'avglnamtstart24m_4525187A', 'avgoutstandbalancel6m_4187114A', 'avgpmtlast12m_4525200A', \n         'currdebt_22A', 'currdebtcredtyperange_828A',  'disbursedcredamount_1113A', 'inittransactionamount_650A',  'maininc_215A',    'maxannuity_159A', \n         'maxdbddpdtollast12m_3658940P',    'maxdpdlast12m_727P', 'maxdpdlast24m_143P', 'maxdpdlast3m_392P', 'maxdpdlast6m_474P', \n         'maxdpdlast9m_1059P', 'maxdpdtolerance_374P', 'maxinstallast24m_3658928A', 'maxlnamtstart6m_4525199A', 'maxoutstandbalancel12m_4187113A', \n         'maxpmtlast3m_4525190A',\n         'sumoutstandtotal_3546847A', 'sumoutstandtotalest_4493215A', 'totaldebt_9A', 'totalsettled_863A', 'totinstallast1m_4525188A'\n    ],\n    'static_cb': ['pmtaverage_4955615A'],\n    'tax_registry_a_1': ['amount_4527230A'],\n    'tax_registry_b_1': ['amount_4917619A'],\n    'tax_registry_c_1': ['pmtamount_36A'],\n}\n\nadjust_to = cpiu['cpiu'].min() # this is the value for 2018-01-01. it doesn't matter when this is, just that everything gets adjusted to the same date.\ncpiu = cpiu.with_columns(pl.col('cpiu').cast(pl.Float32))\ncpiu = cpiu.with_columns(pl.col('cpiu_month').cast(pl.Date))\nif adjust_inflation:\n    for table in dt.keys():\n        if table in adjfromyear.keys():\n\n            # join to cpiu.\n            dt[table] = dt[table].with_columns(pl.col(adjfromyear[table]).dt.truncate(\"1mo\").alias('cpiu_month'))\n            dt[table] = dt[table].join(cpiu, on='cpiu_month', how='left')\n            dt[table] = dt[table].with_columns((pl.col('cpiu') / adjust_to).alias('cpiu')) # calculate the factor.\n\n            # make adjustments.\n            for col in adjcolstoyear[table]:\n                if col in dt[table].columns:\n                    dt[table] = dt[table].with_columns((pl.col(col) / pl.col('cpiu')).alias(col))\n\n            # drop cpiu columns.\n            dt[table].drop(['cpiu', 'cpiu_month'])\n\n        del table\n    \n# keep base but remove it from dt.\nbase = dt['base']\ndel dt['base']\n    \nif not on_kaggle:\n\n    # gather info about the tables.\n    dtinfo = []\n    dttarget = pl.DataFrame({'case_id': all_cases, 'target': targets}).filter(pl.col('target') == 1).drop('target')\n    for key in dt.keys():\n        joinedtotargets = dt[key].join(dttarget, on='case_id', how='inner')\n        dtinfo.append({\n            'name': key,\n            'rows': dt[key].shape[0],\n            'cols': dt[key].shape[1],\n            'pct_default_with_data': np.mean(np.isin(all_cases[targets == 1], dt[key].get_column('case_id').to_numpy())),\n            # 'rows_per_case_90pctl': int(joinedtotargets.get_column('case_id').value_counts().quantile(0.95).get_column('count').to_numpy()[0]) if dt[key].shape[0] > 0 else 0\n            'rows_per_case_90pctl': int(joinedtotargets.get_column('case_id').value_counts().quantile(0.95).get_column('count').to_numpy()[0])\n        })\n        del key, joinedtotargets\n    dtinfo = pd.DataFrame(dtinfo)\n    # print(dtinfo)\n    \n    # drop tables that have less that 10% pct_default_with_data\n    if drop_empty_tables:\n        for table in dtinfo.name[dtinfo.pct_default_with_data < inclusion_cutoff]:\n            print(f'dropped for no data: [{table}]')\n            del dt[table]\n        dtinfo = dtinfo[dtinfo.name.isin(dt.keys())]\n        dtinfo.to_csv('out/dtinfo.csv', index = False)\n        # print('selected tables:')\n        # print(dtinfo.name)\n\n    colinfo = []\n    print('')\n    for key in dt.keys():\n        \n        # drop columns with many null values.\n        joinedtotargets = dt[key].join(dttarget, on='case_id', how='inner')\n        null_count = (joinedtotargets.null_count()/joinedtotargets.shape[0]).to_numpy()[0]\n        for col_idx in range(dt[key].shape[1]):\n            if null_count[col_idx] > 0.9:\n                cols_not_used.append(dt[key].columns[col_idx])\n        \n        todrop = np.unique([x for x in cols_not_used if x in dt[key].columns])\n        if len(todrop) > 0:\n            dt[key] = dt[key].drop(todrop)\n            print(f'null columns dropped from [{key}]: {todrop}')\n            \n        del todrop\n        \n        for col in dt[key].columns:\n            colinfo.append({\n                'table': key,\n                'column': col,\n                'type': dt[key].get_column(col).dtype\n            })    \n\n    del key, col\n    colinfo = pd.DataFrame(colinfo)\n    colinfo.to_csv('out/colinfo.csv', index = False)\n\n    # output data for a case to help make decisions about sorting.\n    if (not on_gcp) and explore_case:\n        for table in dt.keys():\n            dt[table].filter(pl.col('case_id') == explore_case).write_csv('out/explore/case/' + table + '.csv')\n        del table, explore_case\n\n# perform sort.\n\nsortcols = {\n    'applprev_1': ['case_id', 'creationdate_885D', 'credtype_587L', 'cancelreason_3545846M'],\n    'applprev_2': ['case_id', 'num_group1', 'num_group2'],\n    'applprev_1_agg': ['case_id', 'credtype_587L'],\n    'applprev_2_agg': ['case_id'],\n    'credit_bureau_a_1': ['case_id', 'dateofcredstart_181D'],    \n    'credit_bureau_a_1_agg': ['case_id', 'subjectrole_182M', 'subjectrole_93M'],\n    'credit_bureau_a_2_agg': ['case_id', 'collater_typofvalofguarant_298M', 'subjectroles_name_838M'],\n    'credit_bureau_b_1': ['case_id', 'contractdate_551D'],\n    'credit_bureau_b_2': ['case_id', 'pmts_date_1107D'],\n    'debitcard_1': ['case_id', 'openingdate_857D'],\n    # 'debitcard_1_agg': ['case_id'], # only 1 row per case_id.\n    'deposit_1': ['case_id', 'openingdate_313D'],\n    # 'deposit_1_agg': ['case_id'], # only 1 row per case_id.\n    'person_1': ['case_id', 'birth_259D'],\n    'person_2': ['case_id', 'conts_role_79M'],\n    # person_2_agg only has case_id as grouping feature.\n    'tax_registry_a_1': ['case_id', 'recorddate_4527225D'],\n    # 'tax_registry_a_1_agg' only has case_id as grouping feature.\n    'tax_registry_b_1': ['case_id', 'deductiondate_4917603D'],\n    # 'tax_registry_b_1_agg' only has case_id as grouping feature.\n    'tax_registry_c_1': ['case_id', 'processingdate_168D'],\n    # 'tax_registry_c_1_agg' only has case_id as grouping feature.\n}\nfor table in dt.keys():\n    if table in sortcols.keys():\n        dt[table] = dt[table].sort(sortcols[table])\n    del table\n    gc.collect()\n\n# feature engineering.\nif not on_kaggle:\n    levels = {}\n\nfor table in dt.keys():\n    \n    cols_to_drop = []\n    if not on_kaggle:\n        levels[table] = {}\n    \n    print(f'dummies for: [{table}]')\n    \n    # convert column types.\n    for col_idx in range(len(dt[table].columns)):\n        \n        col = dt[table].columns[col_idx]\n        itype = str(dt[table].dtypes[col_idx])\n            \n        if (col == 'case_id') or (itype == 'Date'):\n            continue\n                    \n        elif itype in ['String', 'Null']:\n            \n            # identify levels with at least n% of the data. \n            # these are the levels we'll allow in the model.\n            istringcol = dt[table][col].cast(pl.String).fill_null('')\n\n            ilevels = []\n            if on_kaggle:\n                if (col not in cols_not_used) and (table in levels.keys()):\n                    dt[table] = dt[table].with_columns(pl.col(col).cast(pl.String))\n                    ilevels = np.unique(levels[table][col]) if col in levels[table].keys() else []\n\n            else:\n            \n                levels[table][col] = []\n\n                ilevels = istringcol.value_counts().filter(pl.col(col) != '')\n                if ilevels.shape[0] == 1:\n                    cols_not_used.append(col)\n                    cols_to_drop.append(col)\n                    continue # skip single-valued columns\n                icutoff = ilevels['count'].sum() * inclusion_cutoff\n                ilevels_large = ilevels.filter(pl.col('count') > icutoff)\n                del icutoff\n                \n                # if table == 'person_1' and col == 'registaddr_zipcode_184M':\n                #     stpoehre\n                    \n                if ilevels_large.shape[0] == 0:\n                    cols_not_used.append(col)\n                    cols_to_drop.append(col)\n                    print(f'string dropped: {col}')\n                    # print(ilevels)\n                    continue\n                \n                else:                    \n                    print(f'{col}: {ilevels_large.shape[0]}')\n                    \n                ilevels =  ilevels_large[col]\n            \n            for level in ilevels:\n                dt[table] = dt[table].with_columns((istringcol == level).cast(pl.Int8).alias(col + '_' + level))\n                if not on_kaggle: \n                    levels[table][col].append(level)\n                del level\n            \n            cols_to_drop.append(col)\n            del ilevels, istringcol\n            \n        elif itype == 'Boolean':\n            dt[table] = dt[table].with_columns(pl.col(col).cast(pl.Int8))\n            \n        elif itype not in ['Float32', 'Int32', 'Int8']:\n            raise Exception(f'Unhandled type: {itype}')\n    \n    dt[table] = dt[table].drop(cols_to_drop) #! this must happen after the loop so indexes don't get mixed up.\n    del col\n    gc.collect()\n\n# finish dates. \nprint('')\nfor table in dt.keys():\n    \n    print(f'dates for: [{table}]')\n    \n    idates = [x for x in col_types[table]['date'] if x in dt[table].columns] # dates might have been dropped. \n    idates.append('date_decision')\n    \n    # extract date parts.\n    for col in idates:\n\n        # dt[table] = dt[table].with_columns(pl.col(col).dt.year().alias(col + '_year'))\n        dt[table] = dt[table].with_columns(pl.col(col).dt.month().alias(col + '_month'))\n        dt[table] = dt[table].with_columns(pl.col(col).dt.weekday().alias(col + '_weekday'))\n        del col\n        \n    # get date differences. \n    for idate in range(len(idates)):\n        for jdate in range(idate + 1, len(idates)):\n            ijcol =  np.sort([idates[idate], idates[jdate]]).tolist()\n            ijcol = ijcol[0] + '_2_' + ijcol[1]\n            if ijcol in dt[table].columns:\n                continue\n            ijdiff = (dt[table][ idates[jdate] ] - dt[table][ idates[idate] ]).cast(pl.Float64())\n            ijdiff  = (ijdiff / 1000 / 60 / 60 / 24).cast(pl.Float32) # convert to days from ms.\n            dt[table] = dt[table].with_columns(ijdiff.alias(ijcol))\n            del jdate, ijcol, ijdiff\n        del idate\n        \n    # drop the dates. \n    dt[table] = dt[table].drop(idates)\n    del table\n    gc.collect()\n    \n# verify we have the exact same columns used during training.\n# use col select to ensure the same column order.\nif on_kaggle:\n    print('checking columns')\n    for table in dt.keys():    \n        extracols = [x for x in dt[table].columns if x not in prepca_cols[table]]\n        if len(extracols) > 0:\n            # raise Exception(f'extra cols: {extracols}')\n            print('extra cols:')\n            print((table, extracols))\n        dt[table] = dt[table][prepca_cols[table]]\n\n# capture the exact columns before sending to PCA. \n# we'll use this to check during inference that we have the exact same data.\nprint('')\nprint('fill nas')\nif not on_kaggle: prepca_cols = {}\nfor table in dt.keys():\n    if not on_kaggle: prepca_cols[table] = dt[table].columns\n    # fill NA prior to PCA\n    dt[table] = dt[table].fill_null(-1).fill_nan(-1)\n    del table\n\n# save pre-pca data for exploration.\nif not on_kaggle and sample_further:\n    print('')\n    print('save pre-pca data')\n    for table in dt.keys():\n        dt[table].write_csv('out/explore/prepca/' + table + '-prepca.csv')\n\nprint('')\nprint('pre-pca table sizes:')\nprint([(x, dt[x].shape) for x in dt.keys()])\nif np.any([dt[x].shape[1] > 1000 for x in dt.keys()]):\n    raise Exception('Table is too large. Please correct this.')\n\n# use PCA to reduce the size of the data.\nprint('')\nprint('PCA')\ncase_ids = {}\nif not on_kaggle: \n    table_transformers = {}\n    pca_sample_size = 50000 if on_gcp else 1000\n\ndef to_array_fix_na(x: pl.DataFrame):\n    x = x.to_numpy()\n    x[np.isnan(x)] = -1\n    return(x)\n\nfor table in dt:\n    \n    # case_ids need to be handled separately.\n    case_ids[table] = dt[table].get_column('case_id')\n    dt[table] = dt[table].drop('case_id')\n\n    # convert all remaining columns to floats.\n    dt[table] = dt[table].cast(pl.Float32)\n    \n    if dt[table].shape[0] == 0:\n        continue\n\n    if not on_kaggle:\n\n        # use a sample to estimate the number of components needed.\n        if dt[table].shape[0] > pca_sample_size:\n            idt_sample = dt[table].sample(pca_sample_size)\n        else:\n            idt_sample = dt[table]\n            \n        idt_sample = to_array_fix_na(idt_sample)\n\n        # scale and normalize prior to PCA.\n        table_transformers[table] = {'scaler': StandardScaler().fit(idt_sample)}\n        table_transformers[table]['normalizer'] = Normalizer().fit(table_transformers[table]['scaler'].transform(idt_sample))\n        idt_sample = table_transformers[table]['normalizer'].transform(table_transformers[table]['scaler'].transform(idt_sample))\n\n        # start at number cols minus one and drop columns until you get to 95% of the information.\n        n_components = idt_sample.shape[1]\n        if n_components > 1 and n_components < idt_sample.shape[0]:\n\n            while True:\n                pca = PCA(n_components = n_components)\n                pca = pca.fit(idt_sample)\n                if np.sum(pca.explained_variance_ratio_) < (1 - inclusion_cutoff):\n                    break\n                else:\n                    n_components = n_components - 1\n                    \n            # fit the final pca.\n            n_components += 1\n            #TODO: check n_components == idt_sample.shape[1] to avoid unnecessary PCA.\n            table_transformers[table]['pca'] = PCA(n_components = n_components).fit(idt_sample)    \n    \n    dt[table] = table_transformers[table]['normalizer'].transform(table_transformers[table]['scaler'].transform(to_array_fix_na(dt[table])))\n    if 'pca' in table_transformers[table].keys():\n        dt[table] = table_transformers[table]['pca'].transform(dt[table])\n        if not on_kaggle:\n            print({\n                'table': table, \n                'starting cols': idt_sample.shape[1], \n                'ending cols': n_components, \n                'explained': round(np.sum(table_transformers[table]['pca'].explained_variance_ratio_), 2)\n            })\n            del idt_sample, n_components\n\n    del table\n    gc.collect()\n\n# convert dt to a dict of DataFrames. \n# not needed if using a case_id indexed dict.\n# for table in dt.keys():\n#     dt[table] = pl.DataFrame(dt[table], [f'pca_{x}' for x in range(len(dt[table][0]))])\n#     dt[table] = dt[table].with_columns(case_ids[table].alias('case_id'))\n\n# before grouping, capture column counts.\nif not on_kaggle: \n    pregroupcols = {}\n    for table in dt.keys():\n        pregroupcols[table] = dt[table].shape[1]\n        del table\n\n# convert the data to 8-bit integers so save space.\n\n# split the data by case_id, this will make it easier to get the data we need later.\nprint('')\nprint('splitting by case id')\nbase = base.drop('MONTH', 'WEEK_NUM')\nif 'target' in base.columns:\n    base = base.with_columns(pl.col('target').cast(pl.Binary()))\nfor key, idt in dt.items():    \n    print(key)\n    # idt = np.array(np.round(dt[key] * 100, 0), dtype = np.int8)\n    idt = pl.DataFrame({'data': idt})\n    idt = idt.group_by(case_ids[key]).agg(pl.all())    \n    idt = idt.rename({'data': key})\n    base = base.join(idt, on = 'case_id', how = 'left')\n    del key, idt\n    \ndel dt\ngc.collect()\n\n# convert to input layers. \nprint('')\nprint('convert to input layers')\n\n# add a target column so we can use the same function on kaggle vs off. \nif 'target' not in base.columns:\n    base = base.insert_column(2,pl.Series('target', [-1]*base.shape[0]))\n\n### the next part goes much faster if we only rely on arrays/lists.\nmax_inputs = []\nmax_rows = []\ncols = []\nstart_at_indexes = []\nstart_at_index = 0\ntables = base.columns[3:]\nfor table in tables:\n    \n    imax_rows = dtinfo[dtinfo.name == table].rows_per_case_90pctl.tolist()[0]\n    imax_cols = pregroupcols[table]\n    imax_inputs = imax_rows * imax_cols\n    max_inputs.append(imax_inputs)\n    max_rows.append(imax_rows)\n    \n    # capture current row and move to the next start.\n    start_at_indexes.append(start_at_index)\n    start_at_index += imax_inputs + 1\n    \ntotal_len = start_at_index # this is now the total length.\n    \ndel imax_inputs, imax_rows, imax_cols, start_at_index, table\n\ndef get_input_layer(row):\n    input_layer = np.full((1,total_len), -1, dtype=np.float32)[0]\n    data = list(row[3:])\n    for table_idx in range(len(data)):\n        if data[table_idx]:\n            data[table_idx] = data[table_idx][:np.min([len(data[table_idx]), max_rows[table_idx]])] # cut off rows if there are more than max_rows.\n            data[table_idx] = np.reshape(data[table_idx],(1,-1))[0]\n            input_layer[\n                start_at_indexes[table_idx]:(start_at_indexes[table_idx] + len(data[table_idx])) \n            ] = data[table_idx]    \n    return row[:3] + (input_layer,)\n            \n    # code to validate all values made it into the input_layer:\n    # for table_idx in range(len(tables)):        \n    #     if case_id in dt[tables[table_idx]].keys():   \n    #         table_dt = dt[tables[table_idx]][case_id]\n    #         table_dt = table_dt[:np.min([table_dt.shape[0], max_rows[table_idx]])]\n    #         table_dt = np.reshape(table_dt, (1,-1))[0]    \n    #         for i in table_dt:\n    #             if i not in input_layer:\n    #                 raise Exception(f'Lost value: [{tables[table_idx]}] [{i}]')    \n    # return input_layer\n\n# we'll run out of memory if we process all at once.\ndt = base.with_row_index()\ndt = dt.with_columns((pl.col('index') / base.shape[0] * 15).floor())\ndt = dt.partition_by('index', maintain_order = True, include_key = False)\nprint(f'split to {len(dt)} tables for processing.')\n\nbase = []\nfor idt in dt:    \n    idt = idt.map_rows(get_input_layer)\n    idt.columns = ['case_id', 'date_decision', 'target', 'data']\n    base.append(idt)\n    del idt\n    gc.collect()\n    \nbase = pl.concat(base)\ndel dt\ngc.collect()\n\nprint('')\nif (on_gcp or force_unsample_on_local) and (not on_kaggle):\n\n    # save output.\n    print('saving data')\n    base = base.to_pandas() # necessary for saving. \n\n    prefit = {\n        'adjust_inflation': adjust_inflation,\n        'cpiu': cpiu,\n        'use_cols': use_cols,\n        'extra_group_cols': extra_group_cols,\n        'agg_col_types': agg_col_types,\n        'dtinfo': dtinfo,\n        'colinfo': colinfo,\n        'levels': levels,\n        'col_types': col_types,\n        'cols_not_used': cols_not_used,\n        'prepca_cols': prepca_cols,\n        'table_transformers': table_transformers,\n        'pregroupcols': pregroupcols\n    }    \n    save1(prefit, 'out/prefit.pkl')\n    base.to_parquet(f'out/dt.parquet')\n    \n    # save a sample dataset for testing.\n    print('saving sample')\n    base = base.sample(200000)\n    if not os.path.exists('out/sample'): os.makedirs('out/sample')\n    save1(prefit, 'out/sample/prefit.pkl')\n    base.to_parquet(f'out/sample/dt.parquet')\n    \n    # zip data and upload it to drive.    \n    if on_gcp:\n        print('zip and upload')\n        os.system(\"zip -r batch-output-preprocess.zip out && gsutil cp ./batch-output-preprocess.zip gs://kaggle-417721/batch-output-preprocess.zip\")\n    \nelif not on_kaggle:\n    \n    print('saving data')\n    base = base.to_pandas() # necessary for saving. \n    prefit = {\n        'adjust_inflation': adjust_inflation,\n        'cpiu': cpiu,\n        'use_cols': use_cols,\n        'extra_group_cols': extra_group_cols,\n        'agg_col_types': agg_col_types,\n        'dtinfo': dtinfo,\n        'colinfo': colinfo,\n        'col_types': col_types,\n        'levels': levels,\n        'cols_not_used': cols_not_used,\n        'prepca_cols': prepca_cols,\n        'table_transformers': table_transformers,\n        'pregroupcols': pregroupcols\n    }\n    save1(prefit, 'out/sample/prefit.pkl')\n    base.to_parquet(f'out/sample/dt.parquet')\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-28T00:34:13.645971Z","iopub.execute_input":"2024-05-28T00:34:13.646794Z","iopub.status.idle":"2024-05-28T00:34:31.621880Z","shell.execute_reply.started":"2024-05-28T00:34:13.646749Z","shell.execute_reply":"2024-05-28T00:34:31.620640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_input_layer(row):\n\n    input_layer = np.full((1,total_len), -1, dtype=np.float32)[0]\n    data = list(row[3:])\n    for table_idx in range(len(data)):\n        if data[table_idx]:\n            data[table_idx] = data[table_idx][:np.min([len(data[table_idx]), max_rows[table_idx]])] # cut off rows if there are more than max_rows.\n            data[table_idx] = np.reshape(data[table_idx],(1,-1))[0]\n            input_layer[\n                start_at_indexes[table_idx]:(start_at_indexes[table_idx] + len(data[table_idx])) \n            ] = data[table_idx]\n    \n    return row[:3] + (input_layer, )","metadata":{"execution":{"iopub.status.busy":"2024-05-28T00:34:31.623779Z","iopub.execute_input":"2024-05-28T00:34:31.624211Z","iopub.status.idle":"2024-05-28T00:34:31.634217Z","shell.execute_reply.started":"2024-05-28T00:34:31.624173Z","shell.execute_reply":"2024-05-28T00:34:31.632781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CreditDataset(Dataset):    \n    def __init__(self, dt):         \n        dt = dt.with_row_index()\n        dt = dt.with_columns((pl.col('index') / dt.shape[0] * 15).floor())\n        dt = dt.partition_by('index', maintain_order = True, include_key = False)\n        data = []\n        for idt in dt:\n            data+= [x for x in idt['data'].to_numpy()]\n            del idt\n            gc.collect()\n        data = torch.as_tensor(np.array(data), dtype=torch.float32)     \n        self.data = data        \n    def __len__(self):        \n        return len(self.data)\n    def __getitem__(self, idx):\n        return self.data[idx]","metadata":{"execution":{"iopub.status.busy":"2024-05-28T00:34:31.637853Z","iopub.execute_input":"2024-05-28T00:34:31.638589Z","iopub.status.idle":"2024-05-28T00:34:31.648315Z","shell.execute_reply.started":"2024-05-28T00:34:31.638554Z","shell.execute_reply":"2024-05-28T00:34:31.647120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pops = []\n# for idt in base.sample(5).iter_rows():\n#     ipop = np.mean(np.array(idt[3]) != -1)\n#     if ipop < 0.03:\n#         raise Exception(\"Layer is too empty.\")\n#     pops.append(ipop)\n\n# print('')\n# print(f'batches: {len(testloader)}')\n# print(f'input layer len: {len(idt[3])}')\n# print(f'mean pop: {round(np.mean(pops), 2)}')\n# print('')\n\n# del idt, ipop","metadata":{"execution":{"iopub.status.busy":"2024-05-28T00:34:31.650095Z","iopub.execute_input":"2024-05-28T00:34:31.650473Z","iopub.status.idle":"2024-05-28T00:34:31.657880Z","shell.execute_reply.started":"2024-05-28T00:34:31.650443Z","shell.execute_reply":"2024-05-28T00:34:31.656674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if np.any(init_all_cases != base['case_id'].to_numpy()):\n    raise Exception('Mixed up cases.')","metadata":{"execution":{"iopub.status.busy":"2024-05-28T00:34:31.659286Z","iopub.execute_input":"2024-05-28T00:34:31.661371Z","iopub.status.idle":"2024-05-28T00:34:31.667883Z","shell.execute_reply.started":"2024-05-28T00:34:31.661331Z","shell.execute_reply":"2024-05-28T00:34:31.666865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, ClassifierMixin\nfrom sklearn.linear_model import SGDClassifier\nimport xgboost as xgb\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.ensemble import RandomForestClassifier, VotingClassifier\n\nprint(f'cuda: {torch.cuda.is_available()}')\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nclass MLP_estimator(BaseEstimator, ClassifierMixin):\n    \n    def __init__(self, net_file):\n       self.net_file = net_file\n       self.net = self.load_model()\n       self.net = self.net.eval()\n       self.net = self.net.to(device)\n\n    def __getstate__(self):\n        return self.net_file\n    \n    def __setstate__(self, state):\n       self.net_file = state\n       self.net = self.load_model()\n       self.net = self.net.eval()\n       self.net = self.net.to(device)\n       \n    def fit(self, X, y):\n        return self\n    \n    def scores(self, X):\n        X = torch.tensor(np.array(X), dtype=torch.float32)\n        X = X.to(device)\n        with torch.no_grad():\n            outputs_test = self.net(X).cpu()\n        return np.array(unlist(outputs_test.tolist()))\n       \n    def predict(self, X):\n        scores = self.scores(X)\n        return [1 if x > 0.5 else 0 for x in scores]\n    \n    def predict_proba(self, X):\n        scores = self.scores(X)\n        return [[1-x, x] for x in scores]\n    \n    def load_model(self):\n        try:   \n            return torch.jit.load('out/sample/' + self.net_file)\n        except:\n            return torch.jit.load(prefit_dir + self.net_file)\n    \n    classes_ = np.array([0, 1])","metadata":{"execution":{"iopub.status.busy":"2024-05-28T00:34:31.669189Z","iopub.execute_input":"2024-05-28T00:34:31.669581Z","iopub.status.idle":"2024-05-28T00:34:31.745210Z","shell.execute_reply.started":"2024-05-28T00:34:31.669551Z","shell.execute_reply":"2024-05-28T00:34:31.743991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tqdm import tqdm\nuse_voting = False\n\nif not use_voting:\n    \n    testloader = DataLoader(CreditDataset(base), batch_size=5000, shuffle=False, num_workers=0)\n    \n    net = torch.jit.load('/kaggle/input/home-credit-data/model_scripted.pt')\n    net = net.eval()\n    net = net.to(device)\n\n    with torch.no_grad():\n\n        net_scores = []\n        for i, data in enumerate(testloader, 0):\n            data = data\n            data = data.to(device)\n            net_scores.append([x[0] for x in net(data).cpu().tolist()])\n\n        net_scores = np.array(unlist(net_scores))\n\n    scores = net_scores\n    \nelse:\n    \n    # sklearn will not run cuda. \n    # run predictions in batches so we don't run out of memory.\n    scores = []\n    m_evc = load1(f'{prefit_dir}/evc-model.pkl')\n    \n    X_list = base.with_row_index()\n    X_list = X_list.with_columns((pl.col('index') / base.shape[0] * 5000).floor())\n    X_list = X_list.partition_by('index', maintain_order = True, include_key = False)\n    \n    for iX in X_list:\n        X_test = []\n        for row in iX.iter_rows():\n            X_test.append(row[3])\n            del row\n        y_pred_test = m_evc.predict_proba(X_test)\n        y_pred_test = [x[1] for x in y_pred_test]\n        scores += y_pred_test","metadata":{"execution":{"iopub.status.busy":"2024-05-28T00:34:31.746591Z","iopub.execute_input":"2024-05-28T00:34:31.746969Z","iopub.status.idle":"2024-05-28T00:34:33.349255Z","shell.execute_reply.started":"2024-05-28T00:34:31.746940Z","shell.execute_reply":"2024-05-28T00:34:33.348314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scores = (y_pred_sgd + 2*y_pred_rf + 4*net_scores)/7\nsubmission = pd.DataFrame({\n    'case_id': all_cases,\n    'score': scores\n})\n        \nsubmission.set_index('case_id', inplace=True)\nsubmission.to_csv(\"./submission.csv\")\n\nprint(submission.head(10))\nprint(sum(scores))\n\n#             score\n# case_id          \n# 57543    0.006854\n# 57549    0.044152\n# 57551    0.005951\n# 57552    0.040276\n# 57569    0.065610\n# 57630    0.015043\n# 57631    0.002186\n# 57632    0.001539\n# 57633    0.002516\n# 57634    0.003114\n# 0.1872412117663771","metadata":{"execution":{"iopub.status.busy":"2024-05-28T00:34:33.350599Z","iopub.execute_input":"2024-05-28T00:34:33.351151Z","iopub.status.idle":"2024-05-28T00:34:33.364801Z","shell.execute_reply.started":"2024-05-28T00:34:33.351119Z","shell.execute_reply":"2024-05-28T00:34:33.363634Z"},"trusted":true},"execution_count":null,"outputs":[]}]}