{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"},{"sourceId":7634926,"sourceType":"datasetVersion","datasetId":4449013}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install /kaggle/input/packages/*","metadata":{"execution":{"iopub.status.busy":"2024-02-16T05:52:30.237674Z","iopub.execute_input":"2024-02-16T05:52:30.237984Z","iopub.status.idle":"2024-02-16T05:53:10.328961Z","shell.execute_reply.started":"2024-02-16T05:52:30.237962Z","shell.execute_reply":"2024-02-16T05:53:10.327997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, glob\nimport gc\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm \nfrom pathlib import Path\nfrom typing import Literal\nfrom sklearn.model_selection import train_test_split, cross_validate, StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb","metadata":{"execution":{"iopub.status.busy":"2024-02-16T05:36:43.637856Z","iopub.execute_input":"2024-02-16T05:36:43.638123Z","iopub.status.idle":"2024-02-16T05:36:45.458700Z","shell.execute_reply.started":"2024-02-16T05:36:43.638101Z","shell.execute_reply":"2024-02-16T05:36:45.457965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Helper functions\ndef reduce_mem_usage(df, int_cast=True, obj_to_category=False, subset=None):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\n    :param df: dataframe to reduce (pd.DataFrame)\n    :param int_cast: indicate if columns should be tried to be casted to int (bool)\n    :param obj_to_category: convert non-datetime related objects to category dtype (bool)\n    :param subset: subset of columns to analyse (list)\n    :return: dataset with the column dtypes adjusted (pd.DataFrame)\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024 ** 2;\n    gc.collect()\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n#     cols_none = subset if subset is  None else df.columns.tolist()\n#     for col_non in tqdm(cols_none):\n#         df[col_non] = df[col_non].fillna(-888)\n    \n    cols = subset if subset is not None else df.columns.tolist()\n\n    for col in tqdm(cols):\n        col_type = df[col].dtype\n\n        if col_type != object and col_type.name != 'category' and 'datetime' not in col_type.name:\n            df[col] = df[col].fillna(0)\n            c_min = df[col].min()\n            c_max = df[col].max()\n\n#             # test if column can be converted to an integer\n#             treat_as_int = str(col_type)[:3] == 'int'\n#             if int_cast and not treat_as_int:\n#                 treat_as_int = check_if_integer(df[col])\n                \n            treat_as_int = True\n            if treat_as_int:\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.uint8).min and c_max < np.iinfo(np.uint8).max:\n                    df[col] = df[col].astype(np.uint8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.uint16).min and c_max < np.iinfo(np.uint16).max:\n                    df[col] = df[col].astype(np.uint16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.uint32).min and c_max < np.iinfo(np.uint32).max:\n                    df[col] = df[col].astype(np.uint32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n                elif c_min > np.iinfo(np.uint64).min and c_max < np.iinfo(np.uint64).max:\n                    df[col] = df[col].astype(np.uint64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        elif 'datetime' not in col_type.name and obj_to_category:\n            df[col] = df[col].fillna('Mis')\n            df[col] = df[col].astype('category')\n    gc.collect()\n    end_mem = df.memory_usage().sum() / 1024 ** 2\n    print('Memory usage after optimization is: {:.3f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df\n\ndef date_column_depth_0(df):\n    date_columns = ['date_decision'] + [x for x in df.columns if x[-1] == 'D'] \n    df[date_columns] = df[date_columns].apply(pd.to_datetime, errors='coerce')\n    df_diff = df[date_columns].apply(lambda col: (df['date_decision'] - col).dt.days)\n    df_diff.columns = [f'Diff_{col}' for col in df_diff.columns]\n    df = pd.concat([df, df_diff], axis=1)\n    return df\n\ndef gini(x):\n    total = 0\n    for i, xi in enumerate(x[:-1], 1):\n        total += np.sum(np.abs(xi - x[i:]))\n    return total / (len(x)**2 * np.mean(x))\n","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:36:27.774084Z","iopub.execute_input":"2024-02-15T12:36:27.774412Z","iopub.status.idle":"2024-02-15T12:36:27.906799Z","shell.execute_reply.started":"2024-02-15T12:36:27.774383Z","shell.execute_reply":"2024-02-15T12:36:27.905355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_DATASET = \"/kaggle/input/home-credit-credit-risk-model-stability\"\nPATH_PARQUETS = PATH_DATASET + \"/parquet_files\"\nPATH_TRAIN = PATH_PARQUETS + \"/train\"\nPATH_TEST = PATH_PARQUETS + \"/test\"\n\npd.set_option('display.max_columns', 1000)\npd.set_option('display.max_rows', 1000)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:36:27.908868Z","iopub.execute_input":"2024-02-15T12:36:27.909192Z","iopub.status.idle":"2024-02-15T12:36:27.927854Z","shell.execute_reply.started":"2024-02-15T12:36:27.909165Z","shell.execute_reply":"2024-02-15T12:36:27.926765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path):\n    df = pd.read_parquet(path)\n    df = reduce_mem_usage(df)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:36:27.929871Z","iopub.execute_input":"2024-02-15T12:36:27.930272Z","iopub.status.idle":"2024-02-15T12:36:27.944361Z","shell.execute_reply.started":"2024-02-15T12:36:27.930233Z","shell.execute_reply":"2024-02-15T12:36:27.943292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_base = read_file(PATH_TRAIN + \"/train_base.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:36:27.948054Z","iopub.execute_input":"2024-02-15T12:36:27.948869Z","iopub.status.idle":"2024-02-15T12:36:28.659047Z","shell.execute_reply.started":"2024-02-15T12:36:27.948824Z","shell.execute_reply":"2024-02-15T12:36:28.657837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def multi_merge(base_data,train_vs_test,data_type):\n    if train_vs_test ==  'train':\n        file_path = PATH_TRAIN\n        list_parq =  [file_path + '/' + i for i in os.listdir(file_path) if data_type in i ] \n        \n    elif train_vs_test ==  'test':\n        file_path = PATH_TEST\n        list_parq =  [file_path + '/' + i for i in os.listdir(file_path) if data_type in i ] \n        \n    df_i_merged = pd.DataFrame()\n    \n    for i in list_parq:\n        print(i)\n        df_i = pd.read_parquet(i)\n        df_i = reduce_mem_usage(df_i)\n        if 'num_group1' in df_i.columns: \n            df_i = df_i[df_i['num_group1'] == 0 ]\n            df_i = df_i.drop(columns = 'num_group1')\n        df_i_merged = pd.concat([df_i_merged,df_i])\n        del df_i\n        gc.collect()\n    return df_i_merged   ","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:36:36.222511Z","iopub.execute_input":"2024-02-15T12:36:36.222927Z","iopub.status.idle":"2024-02-15T12:36:36.233202Z","shell.execute_reply.started":"2024-02-15T12:36:36.222898Z","shell.execute_reply":"2024-02-15T12:36:36.231805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged = train_base[['case_id']]\nvariable_type_list = ['train_static_0',\n                      'train_static_cb_0',\n                      'train_applprev_1',\n                      'train_credit_bureau_a_1',\n                     'train_credit_bureau_b_1',\n                     'train_debitcard_1',\n                     'train_deposit_1',\n                     'train_person_1',\n                     'train_tax_registry_a_1',\n                     'train_tax_registry_b_1',\n                     'train_tax_registry_c_1']\nfor k in variable_type_list:\n    df_k = multi_merge(train_base,'train',k)\n    df_merged = df_merged.merge(df_k,how = 'outer',on = 'case_id')\n    del df_k\n    gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_train = train_base.merge(df_merged,how = 'left',on = 'case_id')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:39:55.594708Z","iopub.execute_input":"2024-02-15T12:39:55.595045Z","iopub.status.idle":"2024-02-15T12:40:13.825272Z","shell.execute_reply.started":"2024-02-15T12:39:55.595018Z","shell.execute_reply":"2024-02-15T12:40:13.824053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_merged_train.columns)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:40:13.826466Z","iopub.execute_input":"2024-02-15T12:40:13.826797Z","iopub.status.idle":"2024-02-15T12:40:13.835142Z","shell.execute_reply.started":"2024-02-15T12:40:13.826769Z","shell.execute_reply":"2024-02-15T12:40:13.833901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_merged","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:40:13.838526Z","iopub.execute_input":"2024-02-15T12:40:13.839011Z","iopub.status.idle":"2024-02-15T12:40:14.733713Z","shell.execute_reply.started":"2024-02-15T12:40:13.838950Z","shell.execute_reply":"2024-02-15T12:40:14.732550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_percentage = (df_merged_train.isnull().sum()/df_merged_train.shape[0])*100\n\n# Below code gives list of columns having more than 60% null\ncol_to_drop = null_percentage[null_percentage>90].keys()\n\nprint(len(col_to_drop))\n\ndf_merged_train = df_merged_train.drop(col_to_drop, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:40:14.735449Z","iopub.execute_input":"2024-02-15T12:40:14.736082Z","iopub.status.idle":"2024-02-15T12:40:34.928898Z","shell.execute_reply.started":"2024-02-15T12:40:14.736038Z","shell.execute_reply":"2024-02-15T12:40:34.927709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select numerical columns\n\n#Convert date columns to difference\ndate_columns_train = [x for x in df_merged_train.columns if x[-1] == 'D']\ndf_merged_train = date_column_depth_0(df_merged_train)\n\n\ndf_merged_train = df_merged_train.drop(columns = date_columns_train)\ngc.collect()\n\nnumerical_columns = df_merged_train.select_dtypes(include='number').columns\ncat_cols = df_merged_train.select_dtypes(exclude='number').columns\ncat_cols = [col for col in cat_cols if col !='date_decision']\nnumerical_columns","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:40:34.930255Z","iopub.execute_input":"2024-02-15T12:40:34.930574Z","iopub.status.idle":"2024-02-15T12:41:01.565281Z","shell.execute_reply.started":"2024-02-15T12:40:34.930547Z","shell.execute_reply":"2024-02-15T12:41:01.564051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = df_merged_train.select_dtypes(exclude='number').columns\ncat_cols = [col for col in cat_cols if col !='date_decision']\nlen(cat_cols)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:41:01.566446Z","iopub.execute_input":"2024-02-15T12:41:01.566760Z","iopub.status.idle":"2024-02-15T12:41:03.098046Z","shell.execute_reply.started":"2024-02-15T12:41:01.566735Z","shell.execute_reply":"2024-02-15T12:41:03.096949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(numerical_columns),len(cat_cols))","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:41:03.099304Z","iopub.execute_input":"2024-02-15T12:41:03.099615Z","iopub.status.idle":"2024-02-15T12:41:03.105766Z","shell.execute_reply.started":"2024-02-15T12:41:03.099589Z","shell.execute_reply":"2024-02-15T12:41:03.104461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_merged_train.columns)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:41:03.107415Z","iopub.execute_input":"2024-02-15T12:41:03.107842Z","iopub.status.idle":"2024-02-15T12:41:03.125042Z","shell.execute_reply.started":"2024-02-15T12:41:03.107804Z","shell.execute_reply":"2024-02-15T12:41:03.124088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_train[numerical_columns] = df_merged_train[numerical_columns].fillna(-888)\ndf_merged_train[cat_cols] = df_merged_train[cat_cols].fillna('Mis')","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:41:03.129795Z","iopub.execute_input":"2024-02-15T12:41:03.130198Z","iopub.status.idle":"2024-02-15T12:41:25.083376Z","shell.execute_reply.started":"2024-02-15T12:41:03.130166Z","shell.execute_reply":"2024-02-15T12:41:25.082309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_cols = list(numerical_columns) + cat_cols\ntotal_cols = [col for col in total_cols if col not in ['case_id','MONTH','WEEK_NUM']]\nlen(total_cols)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:41:25.094347Z","iopub.execute_input":"2024-02-15T12:41:25.094757Z","iopub.status.idle":"2024-02-15T12:41:25.109547Z","shell.execute_reply.started":"2024-02-15T12:41:25.094719Z","shell.execute_reply":"2024-02-15T12:41:25.108390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from optbinning import OptimalBinning\n\niv_table = pd.DataFrame()\nfor i in total_cols:\n    if np.issubdtype(df_merged_train[i].dtype, np.number):\n        optb = OptimalBinning(name=i, dtype=\"numerical\", solver=\"cp\",special_codes = [-888])\n        optb.fit(df_merged_train[i].values, df_merged_train['target'].values)\n        binning_table = optb.binning_table.build()\n        binning_table['Variable'] = i\n        binning_table = binning_table.drop(index = 'Totals')\n        iv_table = pd.concat([iv_table,binning_table])\n        print(\"Numerical Added - \",i)\n    else:\n        optb = OptimalBinning(name=i, dtype=\"categorical\", solver=\"mip\",special_codes = ['Mis'])\n        optb.fit(df_merged_train[i].values, df_merged_train['target'].values)\n        binning_table = optb.binning_table.build()\n        binning_table['Variable'] = i\n        binning_table = binning_table.drop(index = 'Totals')\n        iv_table = pd.concat([iv_table,binning_table])\n        print(\"Categorical Added - \",i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iv_table['IV'] = iv_table['IV'].astype(float)\niv_sum_table = iv_table.groupby('Variable')['IV'].sum().reset_index()\nlen(iv_sum_table[iv_sum_table['IV']>0.05]['Variable'])","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:46:16.495414Z","iopub.execute_input":"2024-02-15T12:46:16.496715Z","iopub.status.idle":"2024-02-15T12:46:16.510390Z","shell.execute_reply.started":"2024-02-15T12:46:16.496671Z","shell.execute_reply":"2024-02-15T12:46:16.509198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iv_sum_table[iv_sum_table['IV']>0.05]['Variable'].values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_vars = ['case_id','WEEK_NUM','target','MONTH','Diff_approvaldate_319D', 'Diff_birth_259D', 'Diff_birthdate_574D',\n       'Diff_creationdate_885D', 'Diff_dateactivated_425D',\n       'Diff_datelastunpaid_3546854D', 'Diff_dateofbirth_337D',\n       'Diff_dateofcredend_353D', 'Diff_dateofcredstart_181D',\n       'Diff_dateofrealrepmt_138D', 'Diff_empl_employedfrom_271D',\n       'Diff_firstclxcampaign_1125D', 'Diff_firstnonzeroinstldate_307D',\n       'Diff_lastapplicationdate_877D', 'Diff_lastdelinqdate_224D',\n       'Diff_lastrejectdate_50D', 'Diff_lastupdate_388D',\n       'Diff_maxdpdinstldate_3546855D',\n       'Diff_numberofoverdueinstlmaxdat_148D',\n       'Diff_numberofoverdueinstlmaxdat_641D',\n       'Diff_overdueamountmax2date_1002D',\n       'Diff_overdueamountmax2date_1142D',\n       'amtinstpaidbefduel24m_4187115A', 'avgdbddpdlast24m_3658932P',\n       'avgdpdtolclosure24_3658938P', 'avgmaxdpdlast9m_3716943P',\n       'cancelreason_3545846M', 'classificationofcontr_400M',\n       'contaddr_district_15M', 'contaddr_zipcode_807M', 'days120_123L',\n       'days180_256L', 'days30_165L', 'days360_512L', 'days90_310L',\n       'daysoverduetolerancedd_3976961L', 'district_544M', 'dpdmax_139P',\n       'dpdmax_757P', 'dpdmaxdateyear_596T', 'dpdmaxdateyear_896T',\n       'education_1138M', 'empladdr_zipcode_114M', 'employername_160M',\n       'financialinstitution_382M', 'incometype_1044T',\n       'lastcancelreason_561M', 'lastrejectcredamount_222A',\n       'lastrejectreason_759M', 'lastrejectreasonclient_4145040M',\n       'lastst_736L', 'maxdbddpdlast1m_3658939P',\n       'maxdbddpdtollast12m_3658940P', 'maxdbddpdtollast6m_4187119P',\n       'maxdpdfrom6mto36m_3546853P', 'maxdpdinstlnum_3546846P',\n       'maxdpdlast12m_727P', 'maxdpdlast24m_143P', 'maxdpdlast3m_392P',\n       'maxdpdlast6m_474P', 'maxdpdlast9m_1059P', 'maxdpdtolerance_374P',\n       'mobilephncnt_593L', 'name_4527232M',\n       'numberofoverdueinstlmax_1039L', 'numberofoverdueinstlmax_1151L',\n       'numberofqueries_373L', 'numcontrs3months_479L',\n       'numinstlallpaidearly3d_817L', 'numinstlsallpaid_934L',\n       'numinstlswithdpd10_728L', 'numinstlswithdpd5_4187116L',\n       'numinstlswithoutdpd_562L', 'numinstmatpaidtearly2d_4499204L',\n       'numinstpaidearly3d_3546850L', 'numinstpaidearly3dest_4493216L',\n       'numinstpaidearly5d_1087L', 'numinstpaidearly5dobd_4499205L',\n       'numinstpaidearly_338L', 'numinstpaidearlyest_4493214L',\n       'numinstpaidlate1d_3546852L', 'numrejects9m_859L',\n       'overdueamountmax2_14A', 'overdueamountmax2_398A',\n       'overdueamountmax_155A', 'overdueamountmax_35A',\n       'overdueamountmaxdateyear_2T', 'overdueamountmaxdateyear_994T',\n       'pmtnum_254L', 'pmtnum_8L', 'purposeofcred_874M',\n       'registaddr_district_1083M', 'registaddr_zipcode_184M',\n       'rejectreason_755M', 'rejectreasonclient_4145042M',\n       'requesttype_4525192L', 'sellerplacecnt_915L', 'status_219L',\n       'tenor_203L']\n\ndf_merged_train_final = df_merged_train[selected_vars]","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:46:16.524291Z","iopub.execute_input":"2024-02-15T12:46:16.524655Z","iopub.status.idle":"2024-02-15T12:46:18.208617Z","shell.execute_reply.started":"2024-02-15T12:46:16.524627Z","shell.execute_reply":"2024-02-15T12:46:18.207457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_merged_train","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:47:57.350924Z","iopub.execute_input":"2024-02-15T12:47:57.351358Z","iopub.status.idle":"2024-02-15T12:47:57.780276Z","shell.execute_reply.started":"2024-02-15T12:47:57.351326Z","shell.execute_reply":"2024-02-15T12:47:57.779277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_train_final = df_merged_train_final.drop_duplicates(subset= 'case_id')\nprint(df_merged_train_final.shape)\n\n#Reindexing\ntarget = 'target'\n# Reindex\ndf_merged_train_final_2 = df_merged_train_final.set_index(['case_id','WEEK_NUM']) ","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:48:14.938399Z","iopub.execute_input":"2024-02-15T12:48:14.938790Z","iopub.status.idle":"2024-02-15T12:48:15.968719Z","shell.execute_reply.started":"2024-02-15T12:48:14.938760Z","shell.execute_reply":"2024-02-15T12:48:15.967347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Remove covid data\ncovid_weeks = list(np.arange(54,64))\ndf_merged_train_final_2 = df_merged_train_final_2[~df_merged_train_final_2.index.isin(covid_weeks,level = 1)]\nprint(df_merged_train_final_2.shape)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:48:55.875397Z","iopub.execute_input":"2024-02-15T12:48:55.875911Z","iopub.status.idle":"2024-02-15T12:48:56.568165Z","shell.execute_reply.started":"2024-02-15T12:48:55.875881Z","shell.execute_reply":"2024-02-15T12:48:56.566970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Define X,y\nidentifier_cols = ['MONTH']\nX = df_merged_train_final_2.drop(columns = identifier_cols + [target])\nX = X.select_dtypes(exclude=['object'])\ny = df_merged_train_final_2['target']\n#Delete data\ndel df_merged_train_final_2\ngc.collect()\n#Pick some weeks from starting and some weeks from end as OOT\noot_weeks = [0,  1,  2,  3, \n                        48, 49, 50, 51, 52,\n                        87, 88, 89,90, 91]\n#oot df\nX_oot = X[X.index.isin(oot_weeks,level = 1)]\ny_oot = y[y.index.isin(oot_weeks,level = 1)]\n\n#training df\nX = X[~X.index.isin(oot_weeks,level = 1)]\ny = y[~y.index.isin(oot_weeks,level = 1)]\n\n\n#Train test split(stratified with WEEK_NUM in index 1)\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.25, stratify= list(X.index.get_level_values(1)) , random_state=42)\nX_val, X_test, y_val, y_test = train_test_split(X_val, y_val,stratify= list(X_val.index.get_level_values(1)) ,test_size=0.50, random_state=42)\n#delete\ndel X,y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:50:07.604448Z","iopub.execute_input":"2024-02-15T12:50:07.604842Z","iopub.status.idle":"2024-02-15T12:50:12.008637Z","shell.execute_reply.started":"2024-02-15T12:50:07.604812Z","shell.execute_reply":"2024-02-15T12:50:12.007538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params= {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 3,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 1000,\n    \"colsample_bytree\": 0.4, \n    \"colsample_bynode\": 0.4,\n#     \"verbose\": 1,\n    \"random_state\": 42,\n    \"device\": \"cpu\",\n    \"early_stopping_round\": 100\n}\n\nmodel = lgb.LGBMClassifier(**params)\nmodel.fit(\n    X_train, y_train,\n    eval_set=[(X_val, y_val)])","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:50:20.478180Z","iopub.execute_input":"2024-02-15T12:50:20.478561Z","iopub.status.idle":"2024-02-15T12:53:36.902594Z","shell.execute_reply.started":"2024-02-15T12:50:20.478533Z","shell.execute_reply":"2024-02-15T12:53:36.901551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi_imp = pd.DataFrame([model.feature_name_,model.feature_importances_],index= ['F','FI']).T\nfi_imp.sort_values('FI',ascending= False).head(50)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:53:36.948389Z","iopub.execute_input":"2024-02-15T12:53:36.949294Z","iopub.status.idle":"2024-02-15T12:53:36.972589Z","shell.execute_reply.started":"2024-02-15T12:53:36.949242Z","shell.execute_reply":"2024-02-15T12:53:36.971492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Predictions\ny_train_pred = model.predict_proba(X_train)[:,1] \ny_val_pred = model.predict_proba(X_val)[:,1]\ny_test_pred = model.predict_proba(X_test)[:,1]\ny_oot_pred = model.predict_proba(X_oot)[:,1]","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:53:36.975203Z","iopub.execute_input":"2024-02-15T12:53:36.975554Z","iopub.status.idle":"2024-02-15T12:54:29.752770Z","shell.execute_reply.started":"2024-02-15T12:53:36.975525Z","shell.execute_reply":"2024-02-15T12:54:29.751620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\ndef predict_df(X,y):\n    preds = model.predict_proba(X)[:,1] \n    pred_df = pd.DataFrame(preds,columns = ['predict_proba'],index = X.index)\n    \n# #     pred_df.merge(y,how = 'left',on)\n# #     pred_df['Deciles'] = pd.qcut(train_pred['predict_proba'],q=10,labels = False)  \n#     pred_df = pred_df.set_index(['case_id','WEEK_NUM'])\n    pred_df = pred_df.merge(y,how= 'left',left_index = True,right_index = True) \n    pred_df = pred_df.reset_index(level = 1)\n    return pred_df\n\n\n\n\ndef gini_stability(base, score_col=\"score\", w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", score_col]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", score_col]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[score_col])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:54:29.754133Z","iopub.execute_input":"2024-02-15T12:54:29.754443Z","iopub.status.idle":"2024-02-15T12:54:29.765379Z","shell.execute_reply.started":"2024-02-15T12:54:29.754417Z","shell.execute_reply":"2024-02-15T12:54:29.764182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Train\ntrain_predict_df = predict_df(X_train,y_train)\ntrain_gini_stability = gini_stability(train_predict_df, score_col=\"predict_proba\", w_fallingrate=88.0, w_resstd=-0.5)\n\n\n#Val\nval_predict_df = predict_df(X_val,y_val)\nval_gini_stability = gini_stability(val_predict_df, score_col=\"predict_proba\", w_fallingrate=88.0, w_resstd=-0.5)\n\n#Test\ntest_predict_df = predict_df(X_test,y_test)\ntest_gini_stability = gini_stability(test_predict_df, score_col=\"predict_proba\", w_fallingrate=88.0, w_resstd=-0.5)\n\n#Oot\noot_predict_df = predict_df(X_oot,y_oot)\noot_gini_stability = gini_stability(oot_predict_df, score_col=\"predict_proba\", w_fallingrate=88.0, w_resstd=-0.5)\n\nprint(train_gini_stability)\nprint(val_gini_stability)\nprint(test_gini_stability)\nprint(oot_gini_stability)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:54:29.766695Z","iopub.execute_input":"2024-02-15T12:54:29.767533Z","iopub.status.idle":"2024-02-15T12:55:23.124679Z","shell.execute_reply.started":"2024-02-15T12:54:29.767497Z","shell.execute_reply":"2024-02-15T12:55:23.123636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc_train = roc_auc_score(y_train,y_train_pred)\nroc_auc_val = roc_auc_score(y_val,y_val_pred)\nroc_auc_test = roc_auc_score(y_test,y_test_pred)\nroc_auc_oot = roc_auc_score(y_oot,y_oot_pred)\n\n# Ginni\nginni_train = roc_auc_train * 2 - 1\nginni_val = roc_auc_val * 2 - 1\nginni_test = roc_auc_test * 2 - 1\nginni_oot = roc_auc_oot * 2 -1\n\nprint(roc_auc_train)\nprint(roc_auc_val)\nprint(roc_auc_test)\nprint(roc_auc_oot)\n\nprint(ginni_train)\nprint(ginni_val)\nprint(ginni_test)\nprint(ginni_oot)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:55:23.128545Z","iopub.execute_input":"2024-02-15T12:55:23.128887Z","iopub.status.idle":"2024-02-15T12:55:23.630065Z","shell.execute_reply.started":"2024-02-15T12:55:23.128860Z","shell.execute_reply":"2024-02-15T12:55:23.629173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Model features\nmodel_columns = model.feature_name_","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:55:23.631481Z","iopub.execute_input":"2024-02-15T12:55:23.632060Z","iopub.status.idle":"2024-02-15T12:55:23.636479Z","shell.execute_reply.started":"2024-02-15T12:55:23.632029Z","shell.execute_reply":"2024-02-15T12:55:23.635193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_columns","metadata":{"execution":{"iopub.status.busy":"2024-02-15T13:05:24.281526Z","iopub.execute_input":"2024-02-15T13:05:24.281933Z","iopub.status.idle":"2024-02-15T13:05:24.291077Z","shell.execute_reply.started":"2024-02-15T13:05:24.281898Z","shell.execute_reply":"2024-02-15T13:05:24.289677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Delete training data\ndel X_train,X_val,X_test,y_train,y_val,y_test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:55:23.638112Z","iopub.execute_input":"2024-02-15T12:55:23.639084Z","iopub.status.idle":"2024-02-15T12:55:24.271512Z","shell.execute_reply.started":"2024-02-15T12:55:23.639034Z","shell.execute_reply":"2024-02-15T12:55:24.270232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_TEST","metadata":{"execution":{"iopub.status.busy":"2024-02-15T12:57:19.907937Z","iopub.execute_input":"2024-02-15T12:57:19.908341Z","iopub.status.idle":"2024-02-15T12:57:19.915399Z","shell.execute_reply.started":"2024-02-15T12:57:19.908312Z","shell.execute_reply":"2024-02-15T12:57:19.914030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base = pd.read_parquet(PATH_TEST + '/test_base.parquet')\ntest_base = reduce_mem_usage(test_base)","metadata":{"execution":{"iopub.status.busy":"2024-02-15T13:06:59.565325Z","iopub.execute_input":"2024-02-15T13:06:59.565765Z","iopub.status.idle":"2024-02-15T13:07:00.824025Z","shell.execute_reply.started":"2024-02-15T13:06:59.565732Z","shell.execute_reply":"2024-02-15T13:07:00.823185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged = test_base[['case_id']]\nvariable_type_list = ['test_static_0',\n                      'test_static_cb_0',\n                      'test_applprev_1',\n                      'test_credit_bureau_a_1',\n                     'test_credit_bureau_b_1',\n                     'test_debitcard_1',\n                     'test_deposit_1',\n                     'test_person_1',\n                     'test_tax_registry_a_1',\n                     'test_tax_registry_b_1',\n                     'test_tax_registry_c_1']\nfor k in variable_type_list:\n    df_k = multi_merge(test_base,'test',k)\n    df_merged = df_merged.merge(df_k,how = 'outer',on = 'case_id')\n    del df_k\n    gc.collect()\n\ndf_merged_test = test_base.merge(df_merged,how = 'left',on = 'case_id')\n\ndel df_merged\n\n#Convert date columns to difference\ndate_columns_test = [x for x in df_merged_test.columns if x[-1] == 'D']\ndf_merged_test = date_column_depth_0(df_merged_test)\n\n\ndf_merged_test = df_merged_test.drop(columns = date_columns_test)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-15T13:07:00.825754Z","iopub.execute_input":"2024-02-15T13:07:00.826346Z","iopub.status.idle":"2024-02-15T13:07:45.339403Z","shell.execute_reply.started":"2024-02-15T13:07:00.826312Z","shell.execute_reply":"2024-02-15T13:07:45.338051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_columns = df_merged_test.select_dtypes(include='number').columns\ncat_cols = df_merged_test.select_dtypes(exclude='number').columns\ncat_cols = [col for col in cat_cols if col !='date_decision']\ncat_cols = df_merged_test.select_dtypes(exclude='number').columns\ncat_cols = [col for col in cat_cols if col !='date_decision']\nprint(len(numerical_columns), len(cat_cols))\n\ndf_merged_test[numerical_columns] = df_merged_test[numerical_columns].fillna(-888)\ndf_merged_test[cat_cols] = df_merged_test[cat_cols].fillna('Mis')\n\nselected_vars = ['case_id','WEEK_NUM','Diff_approvaldate_319D', 'Diff_birth_259D', 'Diff_birthdate_574D',\n       'Diff_creationdate_885D', 'Diff_dateactivated_425D',\n       'Diff_datelastunpaid_3546854D', 'Diff_dateofbirth_337D',\n       'Diff_dateofcredend_353D', 'Diff_dateofcredstart_181D',\n       'Diff_dateofrealrepmt_138D', 'Diff_empl_employedfrom_271D',\n       'Diff_firstclxcampaign_1125D', 'Diff_firstnonzeroinstldate_307D',\n       'Diff_lastapplicationdate_877D', 'Diff_lastdelinqdate_224D',\n       'Diff_lastrejectdate_50D', 'Diff_lastupdate_388D',\n       'Diff_maxdpdinstldate_3546855D',\n       'Diff_numberofoverdueinstlmaxdat_148D',\n       'Diff_numberofoverdueinstlmaxdat_641D',\n       'Diff_overdueamountmax2date_1002D',\n       'Diff_overdueamountmax2date_1142D',\n       'amtinstpaidbefduel24m_4187115A', 'avgdbddpdlast24m_3658932P',\n       'avgdpdtolclosure24_3658938P', 'avgmaxdpdlast9m_3716943P',\n       'cancelreason_3545846M', 'classificationofcontr_400M',\n       'contaddr_district_15M', 'contaddr_zipcode_807M', 'days120_123L',\n       'days180_256L', 'days30_165L', 'days360_512L', 'days90_310L',\n       'daysoverduetolerancedd_3976961L', 'district_544M', 'dpdmax_139P',\n       'dpdmax_757P', 'dpdmaxdateyear_596T', 'dpdmaxdateyear_896T',\n       'education_1138M', 'empladdr_zipcode_114M', 'employername_160M',\n       'financialinstitution_382M', 'incometype_1044T',\n       'lastcancelreason_561M', 'lastrejectcredamount_222A',\n       'lastrejectreason_759M', 'lastrejectreasonclient_4145040M',\n       'lastst_736L', 'maxdbddpdlast1m_3658939P',\n       'maxdbddpdtollast12m_3658940P', 'maxdbddpdtollast6m_4187119P',\n       'maxdpdfrom6mto36m_3546853P', 'maxdpdinstlnum_3546846P',\n       'maxdpdlast12m_727P', 'maxdpdlast24m_143P', 'maxdpdlast3m_392P',\n       'maxdpdlast6m_474P', 'maxdpdlast9m_1059P', 'maxdpdtolerance_374P',\n       'mobilephncnt_593L', 'name_4527232M',\n       'numberofoverdueinstlmax_1039L', 'numberofoverdueinstlmax_1151L',\n       'numberofqueries_373L', 'numcontrs3months_479L',\n       'numinstlallpaidearly3d_817L', 'numinstlsallpaid_934L',\n       'numinstlswithdpd10_728L', 'numinstlswithdpd5_4187116L',\n       'numinstlswithoutdpd_562L', 'numinstmatpaidtearly2d_4499204L',\n       'numinstpaidearly3d_3546850L', 'numinstpaidearly3dest_4493216L',\n       'numinstpaidearly5d_1087L', 'numinstpaidearly5dobd_4499205L',\n       'numinstpaidearly_338L', 'numinstpaidearlyest_4493214L',\n       'numinstpaidlate1d_3546852L', 'numrejects9m_859L',\n       'overdueamountmax2_14A', 'overdueamountmax2_398A',\n       'overdueamountmax_155A', 'overdueamountmax_35A',\n       'overdueamountmaxdateyear_2T', 'overdueamountmaxdateyear_994T',\n       'pmtnum_254L', 'pmtnum_8L', 'purposeofcred_874M',\n       'registaddr_district_1083M', 'registaddr_zipcode_184M',\n       'rejectreason_755M', 'rejectreasonclient_4145042M',\n       'requesttype_4525192L', 'sellerplacecnt_915L', 'status_219L',\n       'tenor_203L']\n\ndf_merged_test_final = df_merged_test[selected_vars]\ndel df_merged_test\ndf_merged_test_final = df_merged_test_final.drop_duplicates(subset= 'case_id')\nprint(df_merged_test_final.shape)\n\n#Reindexing\nidentifier_cols = ['MONTH']\ntarget = 'target'\n# Reindex\n","metadata":{"execution":{"iopub.status.busy":"2024-02-15T13:07:45.341487Z","iopub.execute_input":"2024-02-15T13:07:45.341831Z","iopub.status.idle":"2024-02-15T13:07:45.443482Z","shell.execute_reply.started":"2024-02-15T13:07:45.341801Z","shell.execute_reply":"2024-02-15T13:07:45.442676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_test_final_2 = df_merged_test_final.set_index(['case_id']) ","metadata":{"execution":{"iopub.status.busy":"2024-02-15T13:08:37.161509Z","iopub.execute_input":"2024-02-15T13:08:37.161962Z","iopub.status.idle":"2024-02-15T13:08:37.169453Z","shell.execute_reply.started":"2024-02-15T13:08:37.161928Z","shell.execute_reply":"2024-02-15T13:08:37.167837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_test_final_2 = df_merged_test_final_2[model_columns]","metadata":{"execution":{"iopub.status.busy":"2024-02-15T13:08:40.601692Z","iopub.execute_input":"2024-02-15T13:08:40.602287Z","iopub.status.idle":"2024-02-15T13:08:40.608354Z","shell.execute_reply.started":"2024-02-15T13:08:40.602253Z","shell.execute_reply":"2024-02-15T13:08:40.607444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_proba_sumbission = model.predict_proba(df_merged_test_final_2)[:,1]","metadata":{"execution":{"iopub.status.busy":"2024-02-15T13:08:40.840583Z","iopub.execute_input":"2024-02-15T13:08:40.841264Z","iopub.status.idle":"2024-02-15T13:08:40.848722Z","shell.execute_reply.started":"2024-02-15T13:08:40.841221Z","shell.execute_reply":"2024-02-15T13:08:40.847787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_proba_sumbission_df = pd.DataFrame(list(zip(list(df_merged_test_final_2.index),preds_proba_sumbission)),\n              columns=['case_id','score'])\npreds_proba_sumbission_df = preds_proba_sumbission_df.set_index('case_id')\npreds_proba_sumbission_df.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-02-15T13:08:41.038456Z","iopub.execute_input":"2024-02-15T13:08:41.039096Z","iopub.status.idle":"2024-02-15T13:08:41.046376Z","shell.execute_reply.started":"2024-02-15T13:08:41.039062Z","shell.execute_reply":"2024-02-15T13:08:41.045363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}