{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.0"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport psutil\nimport gc\nimport re\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport seaborn as sns\nimport lightgbm as lgb\nimport scipy.stats as ss\nimport missingno as msno\nimport matplotlib.pyplot as plt\n%matplotlib inline \nfrom scipy.stats import skew\nfrom scipy.stats import pearsonr\nfrom sklearn.utils import shuffle\nfrom scipy.stats import kendalltau\nfrom sklearn.metrics import roc_auc_score \nfrom sklearn.metrics import accuracy_score\nfrom pandas.api.types import is_numeric_dtype\nfrom sklearn.feature_selection import f_classif\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_selection import mutual_info_classif, chi2\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.tree import DecisionTreeClassifier \nfrom sklearn.feature_selection import SelectKBest\nfrom sklearn.ensemble import RandomForestClassifier\n\nle = LabelEncoder()\n\npar_path = '/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_'\npar_path_test = '/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_memory():\n    process = psutil.Process(os.getpid())\n    mem_info = process.memory_info()\n    print(f\"Memory used: {mem_info.rss / 1024**2:.2f} MB\")\n    if mem_info.rss / 1024**2 > 100:  # threshold in MB\n        gc.collect()\n\ncheck_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_cba = pd.read_parquet(par_path + \"base.parquet\", columns=['case_id','WEEK_NUM','target'])\nprint(base_cba.shape)\n\nbase_test_cba = pd.read_parquet(par_path_test + \"base.parquet\", columns=['case_id','WEEK_NUM'])\nprint(base_test_cba.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DATA AGGREGATION","metadata":{}},{"cell_type":"code","source":"\ndef filter_df(fname):\n    # Load the entire DataFrame from a Parquet file\n    df = pd.read_parquet(par_path + fname + '.parquet')\n\n    for col in df.columns:\n        # last letter of column name will help you determine the type\n        if col[-1] in (\"P\", \"A\"):\n            df[col] = df[col].astype('float32')\n\n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n\n    return df\n\n\ndef filter_df_test(fname):\n    # Load the entire DataFrame from a Parquet file\n    df = pd.read_parquet(par_path_test + fname + '.parquet')\n\n    for col in df.columns:\n        # last letter of column name will help you determine the type\n        if col[-1] in (\"P\", \"A\"):\n            df[col] = df[col].astype('float32')\n\n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n\n    return df\n\ndef depth1_feats(df:pd.DataFrame):\n    numeric_cols = df.select_dtypes(include=['number']).columns.tolist()\n    numeric_cols.remove('case_id')\n    numeric_cols.remove('num_group1')\n    aggfeats = df.groupby('case_id')[numeric_cols].agg('sum').reset_index()\n\n    notnum_cols = df.select_dtypes(exclude=['number']).columns.tolist()\n    notnum_cols.append('case_id')\n    filfeats = df[df['num_group1'] == 0]\n    filfeats = filfeats.drop('num_group1', axis=1)\n    filfeats = filfeats.filter(items=notnum_cols)\n    return pd.merge(filfeats, aggfeats, how='left', on='case_id')\n\ndef depth2_feats(df:pd.DataFrame):\n    numeric_cols = df.select_dtypes(include=['number']).columns.tolist()\n    numeric_cols.remove('case_id')\n    numeric_cols.remove('num_group1')\n    numeric_cols.remove('num_group2')\n    aggfeats = df.groupby('case_id')[numeric_cols].agg('sum').reset_index()\n\n    notnum_cols = df.select_dtypes(exclude=['number']).columns.tolist()\n    notnum_cols.append('case_id')\n    df = df[df['num_group1'] == 0]\n    df = df[df['num_group2'] == 0]\n    filterdf = df.drop(['num_group1', 'num_group2'], axis=1)\n    filterdf = filterdf.filter(items=notnum_cols)\n    return pd.merge(filterdf, aggfeats, how='left', on='case_id') ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Table of Depth: 2","metadata":{}},{"cell_type":"markdown","source":"### Train Data","metadata":{}},{"cell_type":"code","source":"# Files: train_credit_bureau_b_1_(0-10)\ncache = None\nfor id in range(2):\n    bureau_a_2 = filter_df('credit_bureau_a_2_' + str(id))\n    if cache is None:\n        cache = bureau_a_2\n    else:\n        cache = pd.concat([cache, bureau_a_2])\ntmp = depth2_feats(cache)    \ndel cache\ngc.collect()\n\ncache = None\nfor id in range(2, 4):\n    bureau_a_2 = filter_df('credit_bureau_a_2_' + str(id))\n    if cache is None:\n        cache = bureau_a_2\n    else:\n        cache = pd.concat([cache, bureau_a_2])\ntmp = pd.concat([tmp, depth2_feats(cache)])    \ndel cache\ngc.collect()      \n        \ncache = None\nfor id in range(4, 6):\n    bureau_a_2 = filter_df('credit_bureau_a_2_' + str(id))\n    if cache is None:\n        cache = bureau_a_2\n    else:\n        cache = pd.concat([cache, bureau_a_2])\ntmp = pd.concat([tmp, depth2_feats(cache)])    \ndel cache\ngc.collect()            \n        \ncache = None\nfor id in range(6, 8):\n    bureau_a_2 = filter_df('credit_bureau_a_2_' + str(id))\n    if cache is None:\n        cache = bureau_a_2\n    else:\n        cache = pd.concat([cache, bureau_a_2])\ntmp = pd.concat([tmp, depth2_feats(cache)])    \ndel cache\ngc.collect()     \n    \ncache = None\nfor id in range(8, 10):\n    bureau_a_2 = filter_df('credit_bureau_a_2_' + str(id))\n    if cache is None:\n        cache = bureau_a_2\n    else:\n        cache = pd.concat([cache, bureau_a_2])\ntmp = pd.concat([tmp, depth2_feats(cache)])    \ndel cache\ngc.collect()     \n    \ncache = None\nfor id in range(10, 11):\n    bureau_a_2 = filter_df('credit_bureau_a_2_' + str(id))\n    if cache is None:\n        cache = bureau_a_2\n    else:\n        cache = pd.concat([cache, bureau_a_2])\ntmp = pd.concat([tmp, depth2_feats(cache)])    \ndel cache\ngc.collect()      \n\nprint('merging...')\ndata_cba = pd.merge(base_cba, tmp, how=\"left\", on=\"case_id\")\ndel tmp\ngc.collect() \ndata_cba.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: train_applprev_2\napplprev_2 = filter_df('applprev_2')\ntmp = depth2_feats(applprev_2)  \ndel applprev_2\ngc.collect()   \n\nprint('merging...')\ndata_appl = pd.merge(base_cba, tmp, how=\"left\", on=\"case_id\")\ndel tmp\ngc.collect() \ndata_appl.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Data","metadata":{}},{"cell_type":"code","source":"# Files: test_credit_bureau_b_1_(0-10)\ncache = None\nfor id in range(11):\n    bureau_a_2 = filter_df_test('credit_bureau_a_2_' + str(id))\n    if cache is None:\n        cache = bureau_a_2\n    else:\n        cache = pd.concat([cache, bureau_a_2])\ntmp = depth2_feats(cache)    \ndel cache\ngc.collect()  \n\nprint('merging...')\ndata_test_cba = pd.merge(base_test_cba, tmp, how=\"left\", on=\"case_id\")\ndel tmp\ngc.collect() \ndata_test_cba.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: test_applprev_2\napplprev_2_test = filter_df_test('applprev_2')\ntmp = depth2_feats(applprev_2_test)  \ndel applprev_2_test\ngc.collect()   \n\nprint('merging...')\ndata_test_appl = pd.merge(base_test_cba, tmp, how=\"left\", on=\"case_id\")\ndel tmp\ngc.collect() \ndata_test_appl.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Table of Depth: 1","metadata":{}},{"cell_type":"markdown","source":"### Train Data","metadata":{}},{"cell_type":"code","source":"# Files: train_credit_bureau_a_1_(0-4)\ncache = None\nfor id in range(4):\n    cache = pd.concat([cache, filter_df('credit_bureau_a_1_' + str(id))])\n\ndata_cba = pd.merge(data_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect()\ndata_cba.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: train_applprev_1_(0-1)\ncache = None\nfor id in range(2):\n    cache = pd.concat([cache, filter_df('applprev_1_' + str(id))])\n\ndata_appl = pd.merge(data_appl, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect()\ndata_appl.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: train_debitcard_1\ncache = None\ncache = pd.concat([cache, filter_df('debitcard_1')])\n\ndata_debitcard_1_train = pd.merge(base_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_debitcard_1_train.shape}\")\nprint(f\"Columns in data_debitcard_1_train: {data_debitcard_1_train.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: train_deposit_1\ncache = None\ncache = pd.concat([cache, filter_df('deposit_1')])\n\ndata_deposit_1_train = pd.merge(base_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_deposit_1_train.shape}\")\nprint(f\"Columns in data_deposit_1_train: {data_deposit_1_train.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: train_tax_registry_a_1\n\ncache = None\ncache = pd.concat([cache, filter_df('tax_registry_a_1')])\n\ndata_tax_registry_a_1_train = pd.merge(base_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_tax_registry_a_1_train.shape}\")\nprint(f\"Columns in data_tax_registry_a_1_train: {data_tax_registry_a_1_train.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: train_tax_registry_b_1\n\ncache = None\ncache = pd.concat([cache, filter_df('tax_registry_b_1')])\n\ndata_tax_registry_b_1_train = pd.merge(base_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_tax_registry_b_1_train.shape}\")\nprint(f\"Columns in data_tax_registry_a_1_train: {data_tax_registry_b_1_train.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: train_tax_registry_c_1\ncache = None\ncache = pd.concat([cache, filter_df('tax_registry_c_1')])\n\ndata_tax_registry_c_1_train = pd.merge(base_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_tax_registry_c_1_train.shape}\")\nprint(f\"Columns in data_tax_registry_a_1_train: {data_tax_registry_c_1_train.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: train_person_1\ncache = None\n\ncache = pd.concat([cache, filter_df('person_1')])\n\ndata_person_1_train = pd.merge(base_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_person_1_train.shape}\")\nprint(f\"Columns in data_person_1_train: {data_person_1_train.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Data","metadata":{}},{"cell_type":"code","source":"# Files: test_credit_bureau_a_1_(0-4)\ncache = None\nfor id in range(4):\n    cache = pd.concat([cache, filter_df_test('credit_bureau_a_1_' + str(id))])\n\ndata_test_cba = pd.merge(data_test_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect()\ndata_test_cba.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: test_applprev_1_(0-1)\ncache = None\nfor id in range(2):\n    cache = pd.concat([cache, filter_df_test('applprev_1_' + str(id))])\n\ndata_test_appl = pd.merge(data_test_appl, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect()\ndata_test_appl.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: test_debitcard_1\ncache = None\ncache = pd.concat([cache, filter_df('debitcard_1')])\n\ndata_debitcard_1_test = pd.merge(base_test_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_debitcard_1_test.shape}\")\nprint(f\"Columns in data_debitcard_1_test: {data_debitcard_1_test.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: test_deposit_1\ncache = None\ncache = pd.concat([cache, filter_df('deposit_1')])\n\ndata_deposit_1_test = pd.merge(base_test_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_deposit_1_test.shape}\")\nprint(f\"Columns in data_debitcard_1_test: {data_deposit_1_test.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: test_tax_registry_a_1\ncache = None\n\ncache = pd.concat([cache, filter_df('tax_registry_a_1')])\n\ndata_tax_registry_a_1_test = pd.merge(base_test_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_tax_registry_a_1_test.shape}\")\nprint(f\"Columns in data_tax_registry_a_1_test: {data_tax_registry_a_1_test.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: test_tax_registry_b_1\ncache = None\ncache = pd.concat([cache, filter_df('tax_registry_b_1')])\n\ndata_tax_registry_b_1_test = pd.merge(base_test_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_tax_registry_b_1_test.shape}\")\nprint(f\"Columns in data_tax_registry_a_1_test: {data_tax_registry_b_1_test.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: test_tax_registry_c_1\ncache = None\ncache = pd.concat([cache, filter_df('tax_registry_c_1')])\n\ndata_tax_registry_c_1_test = pd.merge(base_test_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_tax_registry_c_1_test.shape}\")\nprint(f\"Columns in data_tax_registry_a_1_test: {data_tax_registry_c_1_test.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Files: test_person_1\ncache = None\ncache = pd.concat([cache, filter_df_test('person_1')])\n\ndata_person_1_test = pd.merge(base_test_cba, depth1_feats(cache), how=\"left\", on=\"case_id\")\ndel cache\ngc.collect() \n\n# Shape and columns of the merged DataFrame\nprint(f\"Shape of the merged DataFrame: {data_person_1_test.shape}\")\nprint(f\"Columns in data_person_1_test: {data_person_1_test.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Table of Depth: 0","metadata":{}},{"cell_type":"markdown","source":"### Train","metadata":{}},{"cell_type":"code","source":"# Files: train_static_0_0, train_static_0_1\n# Define the specific columns that wanted to include\ncolumns_to_keep = [\n   'case_id','annuity_780A', 'credamount_770A', 'disbursedcredamount_1113A', \n    'eir_270L', 'pmtnum_254L', 'lastst_736L', 'totalsettled_863A', \n    'numrejects9m_859L', 'currdebt_22A'\n]\n\ncache = None\nfor id in range(2):\n    static_0 = filter_df('static_0_' + str(id)) \n    static_0 = static_0[columns_to_keep] \n    if cache is None:\n        cache = static_0\n    else:\n        cache = pd.concat([cache, static_0])\n\nprint(static_0['case_id'].dtype)\n\n# Perform the merge\ndata_static_train = pd.merge(base_cba, cache, how=\"left\", on=\"case_id\")\ndel cache\ngc.collect()\nprint(\"Shape of the merged DataFrame:\", data_static_train.shape)\nprint(f\"Columns in data_static_train: {data_static_train.columns.tolist()}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test","metadata":{}},{"cell_type":"code","source":"# Files: test_static_0_0, test_static_0_1\n# Define the specific columns that wanted to include\ncolumns_to_keep = [\n   'case_id','annuity_780A', 'credamount_770A', 'disbursedcredamount_1113A', \n    'eir_270L', 'pmtnum_254L', 'lastst_736L', 'totalsettled_863A', \n    'numrejects9m_859L', 'currdebt_22A'\n]\n\ncache = None\nfor id in range(3):\n    static_0 = filter_df_test('static_0_' + str(id)) \n    static_0 = static_0[columns_to_keep] \n    if cache is None:\n        cache = static_0\n    else:\n        cache = pd.concat([cache, static_0])\n\n# Perform the merge\ndata_static_test = pd.merge(base_test_cba, cache, how=\"left\", on=\"case_id\")\ngc.collect()\nprint(\"Shape of the merged DataFrame (test):\", data_static_test.shape)\nprint(\"Columns in data_static_test: \", data_static_test.columns.tolist())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Copy\n- Used to read saved parquet files to prevent the need to re-merge files\n- Not used when submitted to kaggle\n- Instead, just renamed","metadata":{}},{"cell_type":"markdown","source":"### Train Data","metadata":{}},{"cell_type":"code","source":"# data_cba.to_parquet('all_data_cba.parquet', index=False)\n# data_cba = pd.read_parquet('all_data_cba.parquet')\ndata1_cba = data_cba\n\n# data_appl.to_parquet('all_data_appl.parquet', index=False)\n# data_appl = pd.read_parquet('all_data_appl.parquet')\ndata1_appl = data_appl\n\n# data_static_train.to_parquet('data_static_train.parquet', index=False)\n# data_static_train = pd.read_parquet('data_static_train.parquet')\ndata_static_train_1 = data_static_train\n\n# data_debitcard_1_train.to_parquet('data_debitcard_1_train.parquet', index=False)\n# data_debitcard_1_train = pd.read_parquet('data_debitcard_1_train.parquet')\ndata_debitcard_1_train_1 = data_debitcard_1_train\n\n# data_deposit_1_train.to_parquet('data_deposit_1_train.parquet', index=False)\n# data_deposit_1_train = pd.read_parquet('data_deposit_1_train.parquet')\ndata_deposit_1_train_1 = data_deposit_1_train\n\n# data_tax_registry_a_1_train.to_parquet('data_tax_registry_a_1_train.parquet', index=False)\n# data_tax_registry_a_1_train = pd.read_parquet('data_tax_registry_a_1_train.parquet')\ndata_tax_registry_a_1_train_1 = data_tax_registry_a_1_train\n\n# data_tax_registry_b_1_train.to_parquet('data_tax_registry_b_1_train.parquet', index=False)\n# data_tax_registry_b_1_train = pd.read_parquet('data_tax_registry_b_1_train.parquet')\ndata_tax_registry_b_1_train_1 = data_tax_registry_b_1_train\n\n# data_tax_registry_c_1_train.to_parquet('data_tax_registry_c_1_train.parquet', index=False)\n# data_tax_registry_c_1_train = pd.read_parquet('data_tax_registry_c_1_train.parquet')\ndata_tax_registry_c_1_train_1 = data_tax_registry_c_1_train\n\n# data_person_1_train.to_parquet('data_person_1_train.parquet', index=False)\n# data_person_1_train = pd.read_parquet('data_person_1_train.parquet')\ndata_person_1_train_1 = data_person_1_train","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Data","metadata":{}},{"cell_type":"code","source":"# data_test_cba.to_parquet('all_data_test_cba.parquet', index=False)\n# data_test_cba = pd.read_parquet('all_data_test_cba.parquet')\ndata2_cba = data_test_cba\n\n# data_test_appl.to_parquet('all_data_test_appl.parquet', index=False)\n# data_test_appl = pd.read_parquet('all_data_test_appl.parquet')\ndata2_appl = data_test_appl\n\n# data_static_test.to_parquet('data_static_test.parquet', index=False)\n# data_static_test = pd.read_parquet('data_static_test.parquet')\ndata_static_test_1 = data_static_test\n\n# data_debitcard_1_test.to_parquet('data_debitcard_1_test.parquet', index=False)\n# data_debitcard_1_test = pd.read_parquet('data_debitcard_1_test.parquet')\ndata_debitcard_1_test_1 = data_debitcard_1_test\n\n# data_deposit_1_test.to_parquet('data_deposit_1_test.parquet', index=False)\n# data_deposit_1_test = pd.read_parquet('data_deposit_1_test.parquet')\ndata_deposit_1_test_1 = data_deposit_1_test\n\n# data_tax_registry_a_1_test.to_parquet('data_tax_registry_a_1_test.parquet', index=False)\n# data_tax_registry_a_1_test = pd.read_parquet('data_tax_registry_a_1_test.parquet')\ndata_tax_registry_a_1_test_1 = data_tax_registry_a_1_test\n\n# data_tax_registry_b_1_test.to_parquet('data_tax_registry_b_1_test.parquet', index=False)\n# data_tax_registry_b_1_test = pd.read_parquet('data_tax_registry_b_1_test.parquet')\ndata_tax_registry_b_1_test_1 = data_tax_registry_b_1_test\n\n# data_tax_registry_c_1_test.to_parquet('data_tax_registry_c_1_test.parquet', index=False)\n# data_tax_registry_c_1_test = pd.read_parquet('data_tax_registry_c_1_test.parquet')\ndata_tax_registry_c_1_test_1 = data_tax_registry_c_1_test\n\n# data_person_1_test.to_parquet('data_person_1_test.parquet', index=False)\n# data_person_1_test = pd.read_parquet('data_person_1_test.parquet')\ndata_person_1_test_1 = data_person_1_test","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_cols_cba = []\nfor col in data1_cba.columns:\n    data_cols_cba.append(col)\nprint(data_cols_cba)\nprint(\"Number of columns:\", len(data_cols_cba))\ndata1_cba.shape\n\ndata_cols_appl = []\nfor col in data1_appl.columns:\n    data_cols_appl.append(col)\nprint(data_cols_appl)\nprint(\"Number of columns:\", len(data_cols_appl))\ndata1_appl.shape\n\ndata_cols_static = []\nfor col in data_static_train_1.columns:\n    data_cols_static.append(col)\nprint(data_cols_static)\nprint(\"Number of columns:\", len(data_cols_static))\ndata_static_train_1.shape\n\ndata_cols_debitcard_1 = []\nfor col in data_debitcard_1_train_1.columns:\n    data_cols_debitcard_1.append(col)\nprint(data_cols_debitcard_1)\nprint(\"Number of columns:\", len(data_cols_debitcard_1))\ndata_debitcard_1_train_1.shape\n\ndata_cols_deposit_1 = []\nfor col in data_deposit_1_train_1.columns:\n    data_cols_deposit_1.append(col)\nprint(data_cols_deposit_1)\nprint(\"Number of columns:\", len(data_cols_deposit_1))\ndata_deposit_1_train_1.shape\n\ndata_cols_tax_registry_a_1 = []\nfor col in data_tax_registry_a_1_train_1.columns:\n    data_cols_tax_registry_a_1.append(col)\nprint(data_cols_tax_registry_a_1)\nprint(\"Number of columns:\", len(data_cols_tax_registry_a_1))\ndata_tax_registry_a_1_train_1.shape\n\ndata_cols_tax_registry_b_1 = []\nfor col in data_tax_registry_b_1_train_1.columns:\n    data_cols_tax_registry_b_1.append(col)\nprint(data_cols_tax_registry_b_1)\nprint(\"Number of columns:\", len(data_cols_tax_registry_b_1))\ndata_tax_registry_b_1_train_1.shape\n\ndata_cols_tax_registry_c_1 = []\nfor col in data_tax_registry_c_1_train_1.columns:\n    data_cols_tax_registry_c_1.append(col)\nprint(data_cols_tax_registry_c_1)\nprint(\"Number of columns:\", len(data_cols_tax_registry_c_1))\ndata_tax_registry_c_1_train_1.shape\n\ndata_cols_person_1 = []\nfor col in data_person_1_train_1.columns:\n    data_cols_person_1.append(col)\nprint(data_cols_person_1)\nprint(\"Number of columns:\", len(data_cols_person_1))\ndata_person_1_train_1.shape\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test_cols_cba = []\nfor col in data2_cba.columns:\n    data_test_cols_cba.append(col)\nprint(data_test_cols_cba)\nprint(\"Number of columns:\", len(data_test_cols_cba))\ndata2_cba.shape\n\ndata_test_cols_appl = []\nfor col in data2_appl.columns:\n    data_test_cols_appl.append(col)\nprint(data_test_cols_appl)\nprint(\"Number of columns:\", len(data_test_cols_appl))\ndata2_appl.shape\n\ndata_cols_static = []\nfor col in data_static_test_1.columns:\n    data_cols_static.append(col)\nprint(data_cols_static)\nprint(\"Number of columns:\", len(data_cols_static))\ndata_static_test_1.shape\n\ndata_cols_debitcard_1 = []\nfor col in data_debitcard_1_test_1.columns:\n    data_cols_debitcard_1.append(col)\nprint(data_cols_debitcard_1)\nprint(\"Number of columns:\", len(data_cols_debitcard_1))\ndata_debitcard_1_test_1.shape\n\ndata_cols_deposit_1 = []\nfor col in data_deposit_1_test_1.columns:\n    data_cols_deposit_1.append(col)\nprint(data_cols_deposit_1)\nprint(\"Number of columns:\", len(data_cols_deposit_1))\ndata_deposit_1_test_1.shape\n\ndata_cols_tax_registry_a_1 = []\nfor col in data_tax_registry_a_1_test_1.columns:\n    data_cols_tax_registry_a_1.append(col)\nprint(data_cols_tax_registry_a_1)\nprint(\"Number of columns:\", len(data_cols_tax_registry_a_1))\ndata_tax_registry_a_1_test_1.shape\n\ndata_cols_tax_registry_b_1 = []\nfor col in data_tax_registry_b_1_test_1.columns:\n    data_cols_tax_registry_b_1.append(col)\nprint(data_cols_tax_registry_b_1)\nprint(\"Number of columns:\", len(data_cols_tax_registry_b_1))\ndata_tax_registry_b_1_test_1.shape\n\ndata_cols_tax_registry_c_1 = []\nfor col in data_tax_registry_c_1_test_1.columns:\n    data_cols_tax_registry_c_1.append(col)\nprint(data_cols_tax_registry_c_1)\nprint(\"Number of columns:\", len(data_cols_tax_registry_c_1))\ndata_tax_registry_c_1_test_1.shape\n\ndata_cols_person_1 = []\nfor col in data_person_1_test_1.columns:\n    data_cols_person_1.append(col)\nprint(data_cols_person_1)\nprint(\"Number of columns:\", len(data_cols_person_1))\ndata_person_1_test_1.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DATA OVERVIEW + SET TYPE","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)  \n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### a. Credit Bureau","metadata":{}},{"cell_type":"code","source":"data1_cba.describe()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data1_cba.info())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in data1_cba.columns:\n    if col.endswith('D'):\n        data1_cba[col] = pd.to_datetime(data1_cba[col], errors='coerce')\n\n\nfor col in data1_cba.select_dtypes(include=['object']).columns:\n    data1_cba[col] = data1_cba[col].astype('category')\n\n\nfor col in data2_cba.columns:\n    if col.endswith('D'):\n        data2_cba[col] = pd.to_datetime(data2_cba[col], errors='coerce')\n\n\nfor col in data2_cba.select_dtypes(include=['object']).columns:\n    data2_cba[col] = data2_cba[col].astype('category')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### b. Application Prev","metadata":{}},{"cell_type":"code","source":"data1_appl.describe()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data1_appl.info())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in data1_appl.columns:\n    if col.endswith('D'):\n        data1_appl[col] = pd.to_datetime(data1_appl[col], errors='coerce')\n\nfor col in data1_appl.select_dtypes(include=['object']).columns:\n    data1_appl[col] = data1_appl[col].astype('category')\n\nfor col in data2_appl.columns:\n    if col.endswith('D'):\n        data2_appl[col] = pd.to_datetime(data2_appl[col], errors='coerce')\n\nfor col in data2_appl.select_dtypes(include=['object']).columns:\n    data2_appl[col] = data2_appl[col].astype('category')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### c. Deposit","metadata":{}},{"cell_type":"code","source":"data_deposit_1_train_1.describe()\nprint(data_deposit_1_train_1.info())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_dataset\nfor col in data_deposit_1_train_1.columns:\n    if col.endswith('D'):\n        data_deposit_1_train_1[col] = pd.to_datetime(data_deposit_1_train_1[col], errors='coerce')\n\nfor col in data_deposit_1_train_1.select_dtypes(include=['object']).columns:\n    data_deposit_1_train_1[col] = data_deposit_1_train_1[col].astype('category')\n\n#test_dataset\nfor col in data_deposit_1_test_1.columns:\n    if col.endswith('D'):\n        data_deposit_1_test_1[col] = pd.to_datetime(data_deposit_1_test_1[col], errors='coerce')\n\nfor col in data_deposit_1_test_1.select_dtypes(include=['object']).columns:\n    data_deposit_1_test_1[col] = data_deposit_1_test_1[col].astype('category')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### d. Debit Card","metadata":{}},{"cell_type":"code","source":"data_debitcard_1_train_1.describe()\nprint(data_debitcard_1_train_1.info())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_dataset\nfor col in data_debitcard_1_train_1.columns:\n    if col.endswith('D'):\n        data_debitcard_1_train_1[col] = pd.to_datetime(data_debitcard_1_train_1[col], errors='coerce')\n\nfor col in data_debitcard_1_train_1.select_dtypes(include=['object']).columns:\n    data_debitcard_1_train_1[col] = data_debitcard_1_train_1[col].astype('category')\n\n#test_dataset\nfor col in data_debitcard_1_test_1.columns:\n    if col.endswith('D'):\n        data_debitcard_1_test_1[col] = pd.to_datetime(data_debitcard_1_test_1[col], errors='coerce')\n\nfor col in data_debitcard_1_test_1.select_dtypes(include=['object']).columns:\n    data_debitcard_1_test_1[col] = data_debitcard_1_test_1[col].astype('category')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### e. Tax Registry","metadata":{}},{"cell_type":"markdown","source":"#### i. tax registry a","metadata":{}},{"cell_type":"code","source":"data_tax_registry_a_1_train_1.describe()\nprint(data_tax_registry_a_1_train_1.info())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_dataset\nfor col in data_tax_registry_a_1_train_1.columns:\n    if col.endswith('D'):\n        data_tax_registry_a_1_train_1[col] = pd.to_datetime(data_tax_registry_a_1_train_1[col], errors='coerce')\n\nfor col in data_debitcard_1_train_1.select_dtypes(include=['object']).columns:\n    data_tax_registry_a_1_train_1[col] = data_tax_registry_a_1_train_1[col].astype('category')\n\n#test_dataset\nfor col in data_tax_registry_a_1_test_1.columns:\n    if col.endswith('D'):\n        data_tax_registry_a_1_test_1[col] = pd.to_datetime(data_tax_registry_a_1_test_1[col], errors='coerce')\n\nfor col in data_tax_registry_a_1_test_1.select_dtypes(include=['object']).columns:\n    data_tax_registry_a_1_test_1[col] = data_tax_registry_a_1_test_1[col].astype('category')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ii. tax registry b","metadata":{}},{"cell_type":"code","source":"data_tax_registry_b_1_train_1.describe()\nprint(data_tax_registry_b_1_train_1.info())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_dataset\nfor col in data_tax_registry_b_1_train_1.columns:\n    if col.endswith('D'):\n        data_tax_registry_b_1_train_1[col] = pd.to_datetime(data_tax_registry_b_1_train_1[col], errors='coerce')\n\nfor col in data_tax_registry_b_1_train_1.select_dtypes(include=['object']).columns:\n    data_tax_registry_b_1_train_1[col] = data_tax_registry_b_1_train_1[col].astype('category')\n\n#test_dataset\nfor col in data_tax_registry_b_1_test_1.columns:\n    if col.endswith('D'):\n        data_tax_registry_b_1_test_1[col] = pd.to_datetime(data_tax_registry_b_1_test_1[col], errors='coerce')\n\nfor col in data_tax_registry_b_1_test_1.select_dtypes(include=['object']).columns:\n    data_tax_registry_b_1_test_1[col] = data_tax_registry_b_1_test_1[col].astype('category')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### iii. tax registry c","metadata":{}},{"cell_type":"code","source":"data_tax_registry_c_1_train_1.describe()\nprint(data_tax_registry_c_1_train_1.info())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_dataset\nfor col in data_tax_registry_c_1_train_1.columns:\n    if col.endswith('D'):\n        data_tax_registry_c_1_train_1[col] = pd.to_datetime(data_tax_registry_c_1_train_1[col], errors='coerce')\n\nfor col in data_tax_registry_c_1_train_1.select_dtypes(include=['object']).columns:\n    data_tax_registry_c_1_train_1[col] = data_tax_registry_c_1_train_1[col].astype('category')\n\n#test_dataset\nfor col in data_tax_registry_c_1_test_1.columns:\n    if col.endswith('D'):\n        data_tax_registry_c_1_test_1[col] = pd.to_datetime(data_tax_registry_c_1_test_1[col], errors='coerce')\n\nfor col in data_tax_registry_c_1_test_1.select_dtypes(include=['object']).columns:\n    data_tax_registry_c_1_test_1[col] = data_tax_registry_c_1_test_1[col].astype('category')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### f. Static","metadata":{}},{"cell_type":"code","source":"data_static_train_1.describe()\nprint(data_static_train_1.info())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_dataset\nfor col in data_static_train_1.columns:\n    if col.endswith('D'):\n        data_static_train_1[col] = pd.to_datetime(data_static_train_1[col], errors='coerce')\n\nfor col in data_static_train_1.select_dtypes(include=['object']).columns:\n    data_static_train_1[col] = data_static_train_1[col].astype('category')\n\n#test_dataset\nfor col in data_static_test_1.columns:\n    if col.endswith('D'):\n        data_static_test_1[col] = pd.to_datetime(data_static_test_1[col], errors='coerce')\n\nfor col in data_static_test_1.select_dtypes(include=['object']).columns:\n    data_static_test_1[col] = data_static_test_1[col].astype('category')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### g. Person","metadata":{}},{"cell_type":"code","source":"data_person_1_train_1.describe()\nprint(data_person_1_train_1.info())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_dataset\nfor col in data_person_1_train_1.columns:\n    if col.endswith('D'):\n        data_person_1_train_1[col] = pd.to_datetime(data_person_1_train_1[col], errors='coerce')\n\nfor col in data_person_1_train_1.select_dtypes(include=['object']).columns:\n    data_person_1_train_1[col] = data_person_1_train_1[col].astype('category')\n\n#test_dataset\nfor col in data_person_1_test_1.columns:\n    if col.endswith('D'):\n        data_person_1_test_1[col] = pd.to_datetime(data_person_1_test_1[col], errors='coerce')\n\nfor col in data_person_1_test_1.select_dtypes(include=['object']).columns:\n    data_person_1_test_1[col] = data_person_1_test_1[col].astype('category')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## HANDLING MISSING DATA","metadata":{}},{"cell_type":"code","source":"def missing_values_table(df):\n    pd.set_option('display.max_rows', None) \n\n    # Standardize missing value indicators\n    df.replace(['na', 'NaN', '#########'], np.nan, inplace=True)\n    \n    # Calculate total missing values\n    mis_val = df.isnull().sum()\n    \n    # Calculate percentage of missing values\n    mis_val_percent = 100 * mis_val / len(df)\n    \n    # Create a table with the results\n    mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n    \n    # Rename the columns\n    mis_val_table_ren_columns = mis_val_table.rename(\n        columns={0: 'Missing Values', 1: '% of Total Values'})\n    \n    # Sort the table by percentage of missing descending\n    mis_val_table_ren_columns = mis_val_table_ren_columns[\n        mis_val_table_ren_columns.iloc[:, 1] != 0].sort_values(\n        '% of Total Values', ascending=False).round(1)\n    \n    # Print summary information\n    total_cols = df.shape[1]\n    total_missing_cols = mis_val_table_ren_columns.shape[0]\n    \n    print(f\"Your selected dataframe has {total_cols} columns.\")\n    print(f\"There are {total_missing_cols} columns that have missing values.\")\n    \n    # Return the dataframe with missing information\n    return mis_val_table_ren_columns","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### a. Credit Bureau","metadata":{}},{"cell_type":"code","source":"missing_values_table(data1_cba)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_table(data2_cba)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# List of specific columns to visualize\ncolumns_to_visualize = [\n    'overdueamountmax2date_1002D',\t\n    'numberofoverdueinstlmaxdat_148D',\t\n    'numberofoverdueinstlmaxdat_641D',\t\n    'overdueamountmax2date_1142D',\n    'dateofrealrepmt_138D',\n    'refreshdate_3813885D',\n    'lastupdate_388D',\n    'dateofcredstart_181D',\n    'dateofcredend_353D',\n    'lastupdate_1112D',\n    'dateofcredstart_739D',\n    'dateofcredend_289D'\n]\n\n# Filter the DataFrame to include only the columns of interest\ndf_filtered = data1_cba[columns_to_visualize]\n\n# Visualize missing data using a matrix\nmsno.matrix(df_filtered)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data1_cba.drop(columns_to_visualize, axis=1, inplace=True)\ndata2_cba.drop(columns_to_visualize, axis=1, inplace=True)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numcols_cba = []\ncatcols_cba = []\ndatecols_cba =[]\n\ndef divide(df):\n    exclude_columns = {'target', 'case_id', 'WEEK_NUM'}\n    for col in df.columns:\n        if col not in exclude_columns:\n            if pd.api.types.is_numeric_dtype(df[col]):\n                numcols_cba.append(col)\n            elif pd.api.types.is_datetime64_dtype(df[col]):\n                datecols_cba.append(col)\n            elif pd.api.types.is_categorical_dtype(df[col]):\n                catcols_cba.append(col)\n    return df\n\n# Assuming 'data1_cba' is your DataFrame\ndata1_cba = divide(data1_cba)\nprint(f'catcols: {len(catcols_cba)}, numcols: {len(numcols_cba)}, datecols: {len(datecols_cba)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in numcols_cba:\n    if data1_cba[col].isnull().sum() > 0:  \n        col_skewness = data1_cba[col].skew()\n        if abs(col_skewness) > 0.5:\n            print(f'Column {col} is skewed with skewness {col_skewness}')\n            median_value = data1_cba[col].median()\n            data1_cba[col] = data1_cba[col].fillna(median_value)\n        else:\n            mean_value = data1_cba[col].mean()\n            data1_cba[col] = data1_cba[col].fillna(mean_value)\n\nfor col in catcols_cba:\n    if data1_cba[col].isnull().sum() > 0:\n        mode_value = data1_cba[col].mode()[0]  # Compute mode; mode() returns a Series, take the first item\n        data1_cba[col] = data1_cba[col].fillna(mode_value)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_table(data1_cba)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### b. Application Prev","metadata":{}},{"cell_type":"code","source":"missing_values_table(data1_appl)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_table(data2_appl)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# List of specific columns to visualize\ncolumns_to_visualize = [\n    'credacc_cards_status_52L',\n    'credacc_status_367L',\n    'isdebitcard_527L',\n    'dtlastpmt_581D',\n    'employedfrom_700D',\n    'familystate_726L',\n    'dtlastpmtallstes_3545839D',\n    'dateactivated_425D',\n    'approvaldate_319D',\n    'firstnonzeroinstldate_307D'\n]\n\n# Filter the DataFrame to include only the columns of interest\ndf_filtered = data1_appl[columns_to_visualize]\n\n# Visualize missing data using a matrix\nmsno.matrix(df_filtered)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cramers_v(x, y):\n    \"\"\"Calculate Cramér's V statistic for categorical-categorical association.\"\"\"\n    confusion_matrix = pd.crosstab(x, y)\n    chi2 = ss.chi2_contingency(confusion_matrix)[0]\n    n = confusion_matrix.sum().sum()\n    phi2 = chi2 / n\n    r, k = confusion_matrix.shape\n    phi2_corr = max(0, phi2 - ((k-1)*(r-1))/(n-1))  # Correct for bias\n    r_corr = r - ((r-1)**2)/(n-1)\n    k_corr = k - ((k-1)**2)/(n-1)\n    return np.sqrt(phi2_corr / min((k_corr-1), (r_corr-1)))\n\n# DataFrame example load - you would replace this with your actual data load method\n# data = pd.read_csv('your_data_file.csv')\n\nresults = {}\nfor column in columns_to_visualize:\n    cramer_v_value = cramers_v(data1_appl[column], data1_appl['target'])\n    results[column] = cramer_v_value\n\n# Displaying results\nfor column, cv in results.items():\n    print(f\"Cramér's V between {column} and target: {cv:.3f}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data1_appl.drop(columns_to_visualize, axis=1, inplace=True)\ndata2_appl.drop(columns_to_visualize, axis=1, inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numcols_appl = []\ncatcols_appl = []\ndatecols_appl =[]\n\ndef divide(df):\n    exclude_columns = {'target', 'case_id', 'WEEK_NUM'}\n    for col in df.columns:\n        if col not in exclude_columns:\n            if pd.api.types.is_numeric_dtype(df[col]):\n                numcols_appl.append(col)\n            elif pd.api.types.is_datetime64_dtype(df[col]):\n                datecols_appl.append(col)\n            elif pd.api.types.is_categorical_dtype(df[col]):\n                catcols_appl.append(col)\n    return df\n\n# Assuming 'data1_cba' is your DataFrame\ndata1_appl = divide(data1_appl)\nprint(f'catcols: {len(catcols_appl)}, numcols: {len(numcols_appl)}, datecols: {len(datecols_appl)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in numcols_appl:\n    if data1_appl[col].isnull().sum() > 0:  # Check if there are any missing values\n        # Calculate skewness and decide on the imputation method\n        col_skewness = data1_appl[col].skew()\n        if abs(col_skewness) > 0.5:\n            print(f'Column {col} is skewed with skewness {col_skewness}')\n            median_value = data1_appl[col].median()\n            data1_appl[col] = data1_appl[col].fillna(median_value)\n        else:\n            mean_value = data1_appl[col].mean()\n            data1_appl[col] = data1_appl[col].fillna(mean_value)\n\nfor col in catcols_appl:\n    if data1_appl[col].isnull().sum() > 0:\n        mode_value = data1_appl[col].mode()[0]  # Compute mode; mode() returns a Series, take the first item\n        data1_appl[col] = data1_appl[col].fillna(mode_value)\n            ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_table(data1_appl)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### c. Deposit","metadata":{}},{"cell_type":"code","source":"missing_values_table(data_deposit_1_train_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_table(data_deposit_1_test_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter to include only columns with at least one missing value\nmissing_data_columns = data_deposit_1_train_1.columns[data_deposit_1_train_1.isnull().any()].tolist()\nmissing_data_binary = data_deposit_1_train_1[missing_data_columns].isnull().astype(float)\n\n# Compute a correlation matrix of missing values\nmissing_corr = missing_data_binary.corr()\n\n# Generate a heatmap\nplt.figure(figsize=(12, 10))\nsns.heatmap(missing_corr, cmap='viridis', linewidths=.5)\nplt.title('Correlation of Missing Data')\nplt.show()\nmsno.matrix(missing_corr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numcols_depo = []\ncatcols_depo = []\ndatecols_depo =[]\n\ndef divide(df):\n    exclude_columns = {'target', 'case_id', 'WEEK_NUM'}\n    for col in df.columns:\n        if col not in exclude_columns:\n            if pd.api.types.is_numeric_dtype(df[col]):\n                numcols_depo.append(col)\n            elif pd.api.types.is_datetime64_dtype(df[col]):\n                datecols_depo.append(col)\n            elif pd.api.types.is_categorical_dtype(df[col]):\n                catcols_depo.append(col)\n    return df\n\ndata_deposit_1_train_1 = divide(data_deposit_1_train_1)\nprint(f'catcols: {len(catcols_depo)}, numcols: {len(numcols_depo)}, datecols: {len(datecols_depo)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the skewness threshold for extremely high skewness\nskewness_threshold = 3\n\n# Calculate skewness for each column\nskewness_results = {}\ncolumns_with_high_skewness = []\n\nfor col in numcols_depo:\n    # Convert to numeric, coercing errors\n    numeric_col = pd.to_numeric(data_deposit_1_train_1[col], errors='coerce')\n    skewness_value = skew(numeric_col.dropna())\n    skewness_results[col] = skewness_value\n    \n    # Check if skewness is extremely high and mark the column\n    if abs(skewness_value) > skewness_threshold:\n        columns_with_high_skewness.append(col)\n\n# Drop columns with extremely high skewness\ndata_deposit_1_train_1 = data_deposit_1_train_1.drop(columns=columns_with_high_skewness)\n\n# Decide on imputation strategy based on skewness\nimputation_strategy = {}\n\nfor col, skew_value in skewness_results.items():\n    if col not in columns_with_high_skewness:  # Only consider columns not marked with high skewness\n        if abs(skew_value) > 1:  # Highly skewed data\n            imputation_strategy[col] = 'median'\n        else:  # Less skewed data\n            imputation_strategy[col] = 'mean'\n\n# Display skewness results and chosen imputation strategies\nskewness_df = pd.DataFrame.from_dict(skewness_results, orient='index', columns=['Skewness'])\nimputation_df = pd.DataFrame.from_dict(imputation_strategy, orient='index', columns=['Imputation Strategy'])\n\nprint(\"Skewness Results:\")\nprint(skewness_df)\n\nprint(\"\\nImputation Strategies:\")\nprint(imputation_df)\n\nprint(f\"\\nColumns with Extremely High Skewness (dropped): {columns_with_high_skewness}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### d. Debit Card\n","metadata":{}},{"cell_type":"code","source":"missing_values_table(data_debitcard_1_train_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_table(data_debitcard_1_train_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter to include only columns with at least one missing value\nmissing_data_columns = data_debitcard_1_train_1.columns[data_debitcard_1_train_1.isnull().any()].tolist()\nmissing_data_binary = data_debitcard_1_train_1[missing_data_columns].isnull().astype(float)\n\n# Compute a correlation matrix of missing values\nmissing_corr = missing_data_binary.corr()\n\n# Generate a heatmap\nplt.figure(figsize=(12, 10))\nsns.heatmap(missing_corr, cmap='viridis', linewidths=.5)\nplt.title('Correlation of Missing Data')\nmsno.matrix(missing_corr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numcols_dcard = []\ncatcols_dcard = []\ndatecols_dcard =[]\n\ndef divide(df):\n    exclude_columns = {'target', 'case_id', 'WEEK_NUM'}\n    for col in df.columns:\n        if col not in exclude_columns:\n            if pd.api.types.is_numeric_dtype(df[col]):\n                numcols_dcard.append(col)\n            elif pd.api.types.is_datetime64_dtype(df[col]):\n                catcols_dcard.append(col)\n            elif pd.api.types.is_categorical_dtype(df[col]):\n                datecols_dcard.append(col)\n    return df\n\ndata_debitcard_1_train_1 = divide(data_debitcard_1_train_1)\nprint(f'catcols: {len(catcols_dcard)}, numcols: {len(numcols_dcard)}, datecols: {len(datecols_dcard)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Relevant columns with missing values\ncolumns_to_check = [\n    'last180dayaveragebalance_704A',\n    'last180dayturnover_1134A',\n    'last30dayturnover_651A',\n]\n\n# Define the skewness threshold for extremely high skewness\nskewness_threshold = 2\n\n# Calculate skewness for each column\nskewness_results = {}\ncolumns_with_high_skewness = []\nfor col in columns_to_check:\n    if col in data_debitcard_1_train_1.columns:\n        # Convert to numeric, coercing errors\n        numeric_col = pd.to_numeric(data_debitcard_1_train_1[col], errors='coerce')\n        skewness_value = skew(numeric_col.dropna())\n        skewness_results[col] = skewness_value\n        # Check if skewness is extremely high and mark the column\n        if abs(skewness_value) > skewness_threshold:\n            columns_with_high_skewness.append(col)\n\n# Drop columns with extremely high skewness\ndata_debitcard_1_train_1 = data_debitcard_1_train_1.drop(columns=columns_with_high_skewness)\n\n# Decide on imputation strategy based on skewness\nimputation_strategy = {}\nfor col, skew_value in skewness_results.items():\n    if col not in columns_with_high_skewness:  # Only consider columns not marked with high skewness\n        if abs(skew_value) > 1:  # Highly skewed data\n            imputation_strategy[col] = 'median'\n        else:  # Less skewed data\n            imputation_strategy[col] = 'mean'\n\n# Display skewness results and chosen imputation strategies\nskewness_df = pd.DataFrame.from_dict(skewness_results, orient='index', columns=['Skewness'])\nimputation_df = pd.DataFrame.from_dict(imputation_strategy, orient='index', columns=['Imputation Strategy'])\n\nprint(\"Skewness Results:\")\nprint(skewness_df)\n\nprint(\"\\nImputation Strategies:\")\nprint(imputation_df)\n\nprint(f\"\\nColumns with Extremely High Skewness (dropped): {columns_with_high_skewness}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### e. Tax Registry\n\n\n","metadata":{}},{"cell_type":"markdown","source":"#### i. tax registry a","metadata":{}},{"cell_type":"code","source":"missing_values_table(data_tax_registry_a_1_train_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_table(data_tax_registry_a_1_test_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter to include only columns with at least one missing value\nmissing_data_columns = data_tax_registry_a_1_train_1.columns[data_tax_registry_a_1_train_1.isnull().any()].tolist()\nmissing_data_binary = data_tax_registry_a_1_train_1[missing_data_columns].isnull().astype(float)\n\n# Compute a correlation matrix of missing values\nmissing_corr = missing_data_binary.corr()\n\n# Generate a heatmap\nplt.figure(figsize=(12, 10))\nsns.heatmap(missing_corr, cmap='viridis', linewidths=.5)\nplt.title('Correlation of Missing Data')\nmsno.matrix(missing_corr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Relevant columns with missing values\ncolumns_to_check = [\n    'amount_4527230A',\n]\n\n# Define the skewness threshold for extremely high skewness\nskewness_threshold = 2\n\n# Calculate skewness for each column\nskewness_results = {}\ncolumns_with_high_skewness = []\nfor col in columns_to_check:\n    if col in data_tax_registry_a_1_train_1.columns:\n        # Convert to numeric, coercing errors\n        numeric_col = pd.to_numeric(data_tax_registry_a_1_train_1[col], errors='coerce')\n        skewness_value = skew(numeric_col.dropna())\n        skewness_results[col] = skewness_value\n        # Check if skewness is extremely high and mark the column\n        if abs(skewness_value) > skewness_threshold:\n            columns_with_high_skewness.append(col)\n\n# Drop columns with extremely high skewness\ndata_tax_registry_a_1_train_1 = data_tax_registry_a_1_train_1.drop(columns=columns_with_high_skewness)\n\n# Decide on imputation strategy based on skewness\nimputation_strategy = {}\nfor col, skew_value in skewness_results.items():\n    if col not in columns_with_high_skewness:  # Only consider columns not marked with high skewness\n        if abs(skew_value) > 1:  # Highly skewed data\n            imputation_strategy[col] = 'median'\n        else:  # Less skewed data\n            imputation_strategy[col] = 'mean'\n\n# Display skewness results and chosen imputation strategies\nskewness_df = pd.DataFrame.from_dict(skewness_results, orient='index', columns=['Skewness'])\nimputation_df = pd.DataFrame.from_dict(imputation_strategy, orient='index', columns=['Imputation Strategy'])\n\nprint(\"Skewness Results:\")\nprint(skewness_df)\n\nprint(\"\\nImputation Strategies:\")\nprint(imputation_df)\n\nprint(f\"\\nColumns with Extremely High Skewness (dropped): {columns_with_high_skewness}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numcols_tra = []\ncatcols_tra = []\ndatecols_tra =[]\n\ndef divide(df):\n    exclude_columns = {'target', 'case_id', 'WEEK_NUM'}\n    for col in df.columns:\n        if col not in exclude_columns:\n            if pd.api.types.is_numeric_dtype(df[col]):\n                numcols_tra.append(col)\n            elif pd.api.types.is_datetime64_dtype(df[col]):\n                datecols_tra.append(col)\n            elif pd.api.types.is_categorical_dtype(df[col]):\n                catcols_tra.append(col)\n    return df\n\ndata_tax_registry_a_1_train_1 = divide(data_tax_registry_a_1_train_1)\nprint(f'catcols: {len(catcols_tra)}, numcols: {len(numcols_tra)}, datecols: {len(datecols_tra)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ii. tax registry b\n","metadata":{}},{"cell_type":"code","source":"missing_values_table(data_tax_registry_b_1_train_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_table(data_tax_registry_b_1_test_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter to include only columns with at least one missing value\nmissing_data_columns = data_tax_registry_b_1_train_1.columns[data_tax_registry_b_1_train_1.isnull().any()].tolist()\nmissing_data_binary = data_tax_registry_b_1_train_1[missing_data_columns].isnull().astype(float)\n\n# Compute a correlation matrix of missing values\nmissing_corr = missing_data_binary.corr()\n\n# Generate a heatmap\nplt.figure(figsize=(12, 10))\nsns.heatmap(missing_corr, cmap='viridis', linewidths=.5)\nplt.title('Correlation of Missing Data')\nmsno.matrix(missing_corr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Relevant columns with missing values\ncolumns_to_check = [\n    'amount_4917619A',\n]\n\n# Define the skewness threshold for extremely high skewness\nskewness_threshold = 2\n\n# Calculate skewness for each column\nskewness_results = {}\ncolumns_with_high_skewness = []\nfor col in columns_to_check:\n    if col in data_tax_registry_b_1_train_1.columns:\n        # Convert to numeric, coercing errors\n        numeric_col = pd.to_numeric(data_tax_registry_b_1_train_1[col], errors='coerce')\n        skewness_value = skew(numeric_col.dropna())\n        skewness_results[col] = skewness_value\n        # Check if skewness is extremely high and mark the column\n        if abs(skewness_value) > skewness_threshold:\n            columns_with_high_skewness.append(col)\n\n# Drop columns with extremely high skewness\ndata_tax_registry_b_1_train_1 = data_tax_registry_b_1_train_1.drop(columns=columns_with_high_skewness)\n\n# Decide on imputation strategy based on skewness\nimputation_strategy = {}\nfor col, skew_value in skewness_results.items():\n    if col not in columns_with_high_skewness:  # Only consider columns not marked with high skewness\n        if abs(skew_value) > 1:  # Highly skewed data\n            imputation_strategy[col] = 'median'\n        else:  # Less skewed data\n            imputation_strategy[col] = 'mean'\n\n# Display skewness results and chosen imputation strategies\nskewness_df = pd.DataFrame.from_dict(skewness_results, orient='index', columns=['Skewness'])\nimputation_df = pd.DataFrame.from_dict(imputation_strategy, orient='index', columns=['Imputation Strategy'])\n\nprint(\"Skewness Results:\")\nprint(skewness_df)\n\nprint(\"\\nImputation Strategies:\")\nprint(imputation_df)\n\nprint(f\"\\nColumns with Extremely High Skewness (dropped): {columns_with_high_skewness}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numcols_trb = []\ncatcols_trb = []\ndatecols_trb =[]\n\ndef divide(df):\n    exclude_columns = {'target', 'case_id', 'WEEK_NUM'}\n    for col in df.columns:\n        if col not in exclude_columns:\n            if pd.api.types.is_numeric_dtype(df[col]):\n                numcols_trb.append(col)\n            elif pd.api.types.is_datetime64_dtype(df[col]):\n                datecols_trb.append(col)\n            elif pd.api.types.is_categorical_dtype(df[col]):\n                catcols_trb.append(col)\n    return df\n\ndata_tax_registry_b_1_train_1 = divide(data_tax_registry_b_1_train_1)\nprint(f'catcols: {len(catcols_trb)}, numcols: {len(numcols_trb)}, datecols: {len(datecols_trb)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### iii. tax registry c","metadata":{}},{"cell_type":"code","source":"missing_values_table(data_tax_registry_c_1_train_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_table(data_tax_registry_c_1_test_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter to include only columns with at least one missing value\nmissing_data_columns = data_tax_registry_c_1_train_1.columns[data_tax_registry_c_1_train_1.isnull().any()].tolist()\nmissing_data_binary = data_tax_registry_c_1_train_1[missing_data_columns].isnull().astype(float)\n\n# Compute a correlation matrix of missing values\nmissing_corr = missing_data_binary.corr()\n\n# Generate a heatmap\nplt.figure(figsize=(12, 10))\nsns.heatmap(missing_corr, cmap='viridis', linewidths=.5)\nplt.title('Correlation of Missing Data')\nmsno.matrix(missing_corr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Relevant columns with missing values\ncolumns_to_check = [\n    'pmtamount_36A',\n]\n\n# Define the skewness threshold for extremely high skewness\nskewness_threshold = 2\n\n# Calculate skewness for each column\nskewness_results = {}\ncolumns_with_high_skewness = []\nfor col in columns_to_check:\n    if col in data_tax_registry_c_1_train_1.columns:\n        # Convert to numeric, coercing errors\n        numeric_col = pd.to_numeric(data_tax_registry_c_1_train_1[col], errors='coerce')\n        skewness_value = skew(numeric_col.dropna())\n        skewness_results[col] = skewness_value\n        # Check if skewness is extremely high and mark the column\n        if abs(skewness_value) > skewness_threshold:\n            columns_with_high_skewness.append(col)\n\n# Drop columns with extremely high skewness\ndata_tax_registry_c_1_train_1 = data_tax_registry_c_1_train_1.drop(columns=columns_with_high_skewness)\n\n# Decide on imputation strategy based on skewness\nimputation_strategy = {}\nfor col, skew_value in skewness_results.items():\n    if col not in columns_with_high_skewness:  # Only consider columns not marked with high skewness\n        if abs(skew_value) > 1:  # Highly skewed data\n            imputation_strategy[col] = 'median'\n        else:  # Less skewed data\n            imputation_strategy[col] = 'mean'\n\n# Display skewness results and chosen imputation strategies\nskewness_df = pd.DataFrame.from_dict(skewness_results, orient='index', columns=['Skewness'])\nimputation_df = pd.DataFrame.from_dict(imputation_strategy, orient='index', columns=['Imputation Strategy'])\n\nprint(\"Skewness Results:\")\nprint(skewness_df)\n\nprint(\"\\nImputation Strategies:\")\nprint(imputation_df)\n\nprint(f\"\\nColumns with Extremely High Skewness (dropped): {columns_with_high_skewness}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numcols_trc = []\ncatcols_trc = []\ndatecols_trc =[]\n\ndef divide(df):\n    exclude_columns = {'target', 'case_id', 'WEEK_NUM'}\n    for col in df.columns:\n        if col not in exclude_columns:\n            if pd.api.types.is_numeric_dtype(df[col]):\n                numcols_trc.append(col)\n            elif pd.api.types.is_datetime64_dtype(df[col]):\n                datecols_trc.append(col)\n            elif pd.api.types.is_categorical_dtype(df[col]):\n                catcols_trc.append(col)\n    return df\n\ndata_tax_registry_c_1_train_1 = divide(data_tax_registry_c_1_train_1)\nprint(f'catcols: {len(catcols_trc)}, numcols: {len(numcols_trc)}, datecols: {len(datecols_trc)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### f. Static\n","metadata":{}},{"cell_type":"code","source":"missing_values_table(data_static_train_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_table(data_static_test_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter to include only columns with at least one missing value\nmissing_data_columns = data_static_train_1.columns[data_static_train_1.isnull().any()].tolist()\nmissing_data_binary = data_static_train_1[missing_data_columns].isnull().astype(float)\n\n# Compute a correlation matrix of missing values\nmissing_corr = missing_data_binary.corr()\n\n# Generate a heatmap\nplt.figure(figsize=(12, 10))\nsns.heatmap(missing_corr, cmap='viridis', linewidths=.5)\nplt.title('Correlation of Missing Data')\nmsno.matrix(missing_corr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numcols_static = []\ncatcols_static = []\ndatecols_static =[]\n\ndef divide(df):\n    exclude_columns = {'target', 'case_id', 'WEEK_NUM'}\n    for col in df.columns:\n        if col not in exclude_columns:\n            if pd.api.types.is_numeric_dtype(df[col]):\n                numcols_static.append(col)\n            elif pd.api.types.is_datetime64_dtype(df[col]):\n                datecols_static.append(col)\n            elif pd.api.types.is_categorical_dtype(df[col]):\n                catcols_static.append(col)\n    return df\n\ndata_static_train_1 = divide(data_static_train_1)\nprint(f'catcols: {len(catcols_static)}, numcols: {len(numcols_static)}, datecols: {len(datecols_static)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the skewness threshold for extremely high skewness\nskewness_threshold = 3\n\n# Calculate skewness for each column\nskewness_results = {}\ncolumns_with_high_skewness = []\n\nfor col in numcols_static:\n    # Convert to numeric, coercing errors\n    numeric_col = pd.to_numeric(data_static_train_1[col], errors='coerce')\n    skewness_value = skew(numeric_col.dropna())\n    skewness_results[col] = skewness_value\n    \n    # Check if skewness is extremely high and mark the column\n    if abs(skewness_value) > skewness_threshold:\n        columns_with_high_skewness.append(col)\n\n# Drop columns with extremely high skewness\ndata_static_train_1 = data_static_train_1.drop(columns=columns_with_high_skewness)\n\n# Decide on imputation strategy based on skewness\nimputation_strategy = {}\n\nfor col, skew_value in skewness_results.items():\n    if col not in columns_with_high_skewness:  # Only consider columns not marked with high skewness\n        if abs(skew_value) > 1:  # Highly skewed data\n            imputation_strategy[col] = 'median'\n        else:  # Less skewed data\n            imputation_strategy[col] = 'mean'\n\n# Display skewness results and chosen imputation strategies\nskewness_df = pd.DataFrame.from_dict(skewness_results, orient='index', columns=['Skewness'])\nimputation_df = pd.DataFrame.from_dict(imputation_strategy, orient='index', columns=['Imputation Strategy'])\n\nprint(\"Skewness Results:\")\nprint(skewness_df)\n\nprint(\"\\nImputation Strategies:\")\nprint(imputation_df)\n\nprint(f\"\\nColumns with Extremely High Skewness (dropped): {columns_with_high_skewness}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply imputation based on strategy\nfor col, strategy in imputation_strategy.items():\n    if strategy == 'median':\n        median_value = data_static_train_1[col].median()\n        print(f\"Median of {col}: {median_value}\")\n        imputer = SimpleImputer(strategy='median')\n    else:\n        mean_value = data_static_train_1[col].mean()\n        print(f\"Mean of {col}: {mean_value}\")\n        imputer = SimpleImputer(strategy='mean')\n    \n    data_static_train_1[[col]] = imputer.fit_transform(data_static_train_1[[col]])\n\n# Check the imputed DataFrame\nprint(\"Data after imputation:\")\nprint(data_static_train_1.head())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### g. Person","metadata":{}},{"cell_type":"code","source":"missing_values_table(data_person_1_train_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_table(data_person_1_test_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter to include only columns with at least one missing value\nmissing_data_columns = data_person_1_train_1.columns[data_person_1_train_1.isnull().any()].tolist()\nmissing_data_binary = data_person_1_train_1[missing_data_columns].isnull().astype(float)\n\n# Compute a correlation matrix of missing values\nmissing_corr = missing_data_binary.corr()\n\n# Generate a heatmap\nplt.figure(figsize=(12, 10))\nsns.heatmap(missing_corr, cmap='viridis', linewidths=.5)\nplt.title('Correlation of Missing Data')\nmsno.matrix(missing_corr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to remove columns with missing rate greater than 50% and print them\ndef remove_high_missing_columns(df, threshold=0.5):\n    missing_rate = df.isnull().mean()\n    cols_to_drop = missing_rate[missing_rate > threshold].index\n    print(f\"Columns removed due to missing rate > {threshold * 100}%: {list(cols_to_drop)}\")\n    df = df.drop(columns=cols_to_drop)\n    return df\n\ndata_person_1_train_1 = remove_high_missing_columns(data_person_1_train_1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numcols_person = []\ncatcols_person = []\ndatecols_person =[]\n\ndef divide(df):\n    exclude_columns = {'target', 'case_id', 'WEEK_NUM'}\n    for col in df.columns:\n        if col not in exclude_columns:\n            if pd.api.types.is_numeric_dtype(df[col]):\n                numcols_person.append(col)\n            elif pd.api.types.is_datetime64_dtype(df[col]):\n                datecols_person.append(col)\n            elif pd.api.types.is_categorical_dtype(df[col]):\n                catcols_person.append(col)\n    return df\n\ndata_person_1_train_1 = divide(data_person_1_train_1)\nprint(f'Initial catcols: {len(catcols_person)}, numcols: {len(numcols_person)}, datecols: {len(datecols_person)}')\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TARGET VARIABLE SEPARATION","metadata":{}},{"cell_type":"code","source":"#credit bureau\ny_cba = data1_cba['target']\ndata1_cba = data1_cba.drop('target', axis=1)\n\n#application prev\ny_appl = data1_appl['target']\ndata1_appl = data1_appl.drop('target', axis=1)\n\n#Static\ny_sta = data_static_train_1['target']\ndata_static_train_1 = data_static_train_1.drop('target', axis=1)\n\n#Person\ny_per = data_person_1_train_1['target']\ndata_person_1_train_1 = data_person_1_train_1.drop('target', axis=1)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CATEGORICAL DATA ENCODING","metadata":{}},{"cell_type":"code","source":"def encode_categorical_columns(train_data, test_data, categorical_columns):\n    # Initialize the LabelEncoder\n    label_encoders = {}\n\n    # For each categorical column in the training data\n    for col in categorical_columns:\n        le = LabelEncoder()\n        # Fit the encoder on the training data\n        train_data[col] = le.fit_transform(train_data[col])\n        # Store the encoder so it can be used on the test data\n        label_encoders[col] = le\n\n    for col in categorical_columns:\n        if col in label_encoders:\n            # Here we handle unknown categories by using 'transform' wrapped in a try-except block\n            try:\n                test_data[col] = label_encoders[col].transform(test_data[col])\n            except ValueError:\n                \n                test_data[col] = test_data[col].map(lambda s: label_encoders[col].classes_[0] if s not in label_encoders[col].classes_ else s)\n                test_data[col] = label_encoders[col].transform(test_data[col])\n    return train_data, test_data, label_encoders","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### a. Credit Bureau","metadata":{}},{"cell_type":"code","source":"numcols_cba = list(set(numcols_cba))\ncatcols_cba = list(set(catcols_cba))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data1_cba[catcols_cba].apply(pd.Series.nunique, axis = 0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sample unique values for columns with many unique categories\nfor col in catcols_cba:\n    unique_values = data1_cba[col].unique()\n    sample_size = min(10, len(unique_values))  # Change 10 to how many examples you want to see\n    print(f\"Sample of unique values in {col}: {unique_values[:sample_size]}\")\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data1_cba, data2_cba, catcols_cba = encode_categorical_columns(data1_cba, data2_cba, catcols_cba)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### b. Application Prev","metadata":{}},{"cell_type":"code","source":"data1_appl[catcols_appl].apply(pd.Series.nunique, axis = 0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sample unique values for columns with many unique categories\nfor col in catcols_appl:\n    unique_values = data1_appl[col].unique()\n    sample_size = min(10, len(unique_values))  # Change 10 to how many examples you want to see\n    print(f\"Sample of unique values in {col}: {unique_values[:sample_size]}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data1_appl, data2_appl, catcols_appl = encode_categorical_columns(data1_appl, data2_appl, catcols_appl)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### c. Static","metadata":{}},{"cell_type":"code","source":"data_static_train_1[catcols_static].apply(pd.Series.nunique, axis = 0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sample unique values for columns with many unique categories\nfor col in catcols_static:\n    unique_values = data_static_train_1[col].unique()\n    sample_size = min(10, len(unique_values))  # Change 10 to how many examples you want to see\n    print(f\"Sample of unique values in {col}: {unique_values[:sample_size]}\")\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_static_train_1, data_static_test_1, catcols_static = encode_categorical_columns(data_static_train_1, data_static_test_1, catcols_static)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Check and print missing values in the column before filling\n# print(\"Missing values in 'lastst_736L' before filling:\")\n# print(data_static_train_1['lastst_736L'].isnull().sum())\n\n# # Add a new category 'Unknown' to the categorical column if it doesn't already exist\n# if 'Unknown' not in data_static_train_1['lastst_736L'].cat.categories:\n#     data_static_train_1['lastst_736L'] = data_static_train_1['lastst_736L'].cat.add_categories(['Unknown'])\n\n# # Fill missing values with the new category\n# data_static_train_1['lastst_736L'] = data_static_train_1['lastst_736L'].fillna('Unknown')\n\n# # Verify that missing values have been filled\n# print(\"Missing values in 'lastst_736L' after filling:\")\n# print(data_static_train_1['lastst_736L'].isnull().sum())\n\n# # Apply one-hot encoding\n# df_encoded = pd.get_dummies(data_static_train_1, columns=['lastst_736L'])\n\n# # Convert one-hot encoded columns to uint8\n# one_hot_cols = [col for col in df_encoded.columns if 'lastst_736L' in col]\n# df_encoded[one_hot_cols] = df_encoded[one_hot_cols].astype('uint8')\n\n# # Display the encoded DataFrame structure and some of the first rows to verify\n# print(df_encoded.head())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Apply one-hot encoding to test data\n# df_encoded = pd.get_dummies(data_static_test_1, columns=['lastst_736L'])\n\n# # Convert one-hot encoded columns to uint8\n# one_hot_cols = [col for col in df_encoded.columns if 'lastst_736L' in col]\n# df_encoded[one_hot_cols] = df_encoded[one_hot_cols].astype('uint8')\n\n# # Display the encoded DataFrame structure and some of the first rows to verify\n# print(df_encoded.head())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unique_values = data_static_train_1['lastst_736L'].nunique()\n# print(\"Total number of unique values in 'lastst_736L':\", unique_values)\n# unique_values = data_static_train_1['lastst_736L'].unique()\n# print(\"Unique values in 'lastst_736L':\", unique_values)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### d. Person","metadata":{}},{"cell_type":"code","source":"data_person_1_train_1[catcols_person].apply(pd.Series.nunique, axis = 0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sample unique values for columns with many unique categories\nfor col in catcols_person:\n    unique_values = data_person_1_train_1[col].unique()\n    sample_size = min(10, len(unique_values))  # Change 10 to how many examples you want to see\n    print(f\"Sample of unique values in {col}: {unique_values[:sample_size]}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_person_1_train_1, data_person_1_test_1, catcols_person = encode_categorical_columns(data_person_1_train_1, data_person_1_test_1, catcols_person)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TARGET VARIABLE EDA","metadata":{}},{"cell_type":"code","source":"#Credit bureau\nprint(y_cba.value_counts())\n\n#Application prev\nprint(y_appl.value_counts())\n\n#Static\nprint(y_sta.value_counts())\n\n#Person\nprint(y_per.value_counts())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Credit bureau\ntemp = y_cba.value_counts()\ndf = pd.DataFrame({'labels': temp.index,\n                   'values': temp.values\n                  })\nplt.figure(figsize = (6,6))\nplt.title('Target')\nsns.set_color_codes(\"pastel\")\nsns.barplot(x = 'labels', y=\"values\", data=df)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Application prev\ntemp = y_appl.value_counts()\ndf = pd.DataFrame({'labels': temp.index,\n                   'values': temp.values\n                  })\nplt.figure(figsize = (6,6))\nplt.title('Target')\nsns.set_color_codes(\"pastel\")\nsns.barplot(x = 'labels', y=\"values\", data=df)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Person\ntemp = y_per.value_counts()\ndf = pd.DataFrame({'labels': temp.index,\n                   'values': temp.values\n                  })\nplt.figure(figsize = (6,6))\nplt.title('Target')\nsns.set_color_codes(\"pastel\")\nsns.barplot(x = 'labels', y=\"values\", data=df)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Static\ntemp = y_sta.value_counts()\ndf = pd.DataFrame({'labels': temp.index,\n                   'values': temp.values\n                  })\nplt.figure(figsize = (6,6))\nplt.title('Target')\nsns.set_color_codes(\"pastel\")\nsns.barplot(x = 'labels', y=\"values\", data=df)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. VARIANCE ANALYSIS","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### a. Credit bureau","metadata":{}},{"cell_type":"code","source":"for col in data1_cba.columns:\n    fraction_unique = data1_cba[col].unique().shape[0] / data1_cba.shape[0]\n    if fraction_unique > 0.5:\n        print(col)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndata1_cba.drop(['debtoutstand_525A', 'outstandingamount_362A', 'totaloutstanddebtvalue_39A'], axis=1, inplace=True)\ndata2_cba.drop(['debtoutstand_525A', 'outstandingamount_362A', 'totaloutstanddebtvalue_39A'], axis=1, inplace=True)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### b. Application prev","metadata":{}},{"cell_type":"code","source":"for col in data1_appl.columns:\n    fraction_unique = data1_appl[col].unique().shape[0] / data1_appl.shape[0]\n    if fraction_unique > 0.5:\n        print(col)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### c. Static\n","metadata":{}},{"cell_type":"code","source":"for col in data_static_train_1.columns:\n    fraction_unique = data_static_train_1[col].unique().shape[0] / data_static_train_1.shape[0]\n    if fraction_unique > 0.5:\n        print(col)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### d. Person","metadata":{}},{"cell_type":"code","source":"for col in data_person_1_train_1.columns:\n    fraction_unique = data_person_1_train_1[col].unique().shape[0] / data_person_1_train_1.shape[0]\n    if fraction_unique > 0.5:\n        print(col)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. FEATURE SELECTION","metadata":{}},{"cell_type":"code","source":"def evaluate_feature_significance(X_num, X_cat, y):\n    # Initialize lists to store results\n    results_num = []\n    results_cat = []\n\n    # Process numerical features\n    anova_f_values, _ = f_classif(X_num, y)\n    for i, col in enumerate(X_num.columns):\n        kendall_corr = kendalltau(X_num[col], y).correlation\n        results_num.append([col, anova_f_values[i], kendall_corr])\n    \n    # Convert results for numerical features to DataFrame\n    numerical_scores = pd.DataFrame(results_num, columns=['Feature', 'ANOVA F-value', 'Kendall Tau'])\n\n    # Process categorical features\n    chi2_values, _ = chi2(X_cat, y)\n    mutual_info_values = mutual_info_classif(X_cat, y)\n    for i, col in enumerate(X_cat.columns):\n        results_cat.append([col, chi2_values[i], mutual_info_values[i]])\n    \n    # Convert results for categorical features to DataFrame\n    categorical_scores = pd.DataFrame(results_cat, columns=['Feature', 'Chi2', 'Mutual Information'])\n\n    # Clean up intermediate variables to free memory\n    del results_num, results_cat, anova_f_values, chi2_values, mutual_info_values\n    gc.collect()  # Run garbage collection\n\n    return numerical_scores, categorical_scores\n\n\n\ndef evaluate_model(X, y, features):\n    model = DecisionTreeClassifier()\n    scores = cross_val_score(model, X[features], y, cv=3, scoring='accuracy', n_jobs=-1)  # Use fewer CV folds\n    return scores.mean()\n\n\n\ndef find_best_threshold(X, y, numerical_df, categorical_df, n_iter=10):\n    best_score = 0\n    best_thresholds = {}\n    \n    # Precompute maximum values to save computation in the loop\n    max_anova = numerical_df['ANOVA F-value'].max()\n    max_kendall = numerical_df['Kendall Tau'].abs().max()\n    max_chi2 = categorical_df['Chi2'].max()\n    max_mutual_info = categorical_df['Mutual Information'].max()\n    \n    for _ in range(n_iter):\n        # Sample thresholds once\n        anova_thresh = np.random.uniform(0, max_anova)\n        kendall_thresh = np.random.uniform(0, max_kendall)\n        chi2_thresh = np.random.uniform(0, max_chi2)\n        mutual_info_thresh = np.random.uniform(0, max_mutual_info)\n        \n        # Select numerical features\n        num_features = numerical_df[(numerical_df['ANOVA F-value'] > anova_thresh) | \n                                    (numerical_df['Kendall Tau'].abs() > kendall_thresh)]['Feature'].tolist()\n        \n        # Select categorical features\n        cat_features = categorical_df[(categorical_df['Chi2'] > chi2_thresh) | \n                                      (categorical_df['Mutual Information'] > mutual_info_thresh)]['Feature'].tolist()\n        \n        # Combine features\n        features = num_features + cat_features\n        \n        # Evaluate model performance\n        score = evaluate_model(X, y, features)\n        \n        # Update best score and thresholds if improved\n        if score > best_score:\n            best_score = score\n            best_thresholds = {\n                'anova_thresh': anova_thresh,\n                'kendall_thresh': kendall_thresh,\n                'chi2_thresh': chi2_thresh,\n                'mutual_info_thresh': mutual_info_thresh\n            }\n    \n    return best_thresholds, best_score","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### a. Credit bureau","metadata":{}},{"cell_type":"code","source":"# Create the plot\nplt.figure(figsize=(5.5, 17))\ncorrelation_series = data1_cba.corrwith(y_cba)\ncorrelation_series.plot.barh()\nplt.tight_layout()\n\n# Display the plot if you are in an interactive environment\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After the plot is no longer needed, close the plot and clear the variables\nplt.close()\n\n# Explicitly delete the variables holding large data\ndel correlation_series","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in data1_cba.columns:\n    if is_numeric_dtype(data1_cba[col]):\n        correlation, pvalue = pearsonr(data1_cba[col].values, y_cba.values)\n        print(f'{col: <40}: {correlation : .4f}, significant: {pvalue <= 0.05}')\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter the columns based on the p-value condition\nsignificant_columns = [col for col in data1_cba.columns if col in ['case_id', 'WEEK_NUM'] or pearsonr(data1_cba[col], y_cba)[1] <= 0.05]\n\n# Keep only the significant columns in the DataFrame\ndata1_cba = data1_cba[significant_columns]\ndata2_cba = data2_cba[significant_columns]\ndel significant_columns","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# Function to identify categorical and numerical columns\ndef identify_columns(dataframe, exclude_columns):\n    cat_features = [col for col in dataframe.columns if col in catcols_cba and col not in exclude_columns]\n    num_features = [col for col in dataframe.columns if col in numcols_cba and col not in exclude_columns]\n    \n    return cat_features, num_features\n\n# Columns to exclude\nexclude_columns = ['case_id', 'WEEK_NUM']\n\n# Identify categorical and numerical columns in data1\ncat_features_cba, num_features_cba = identify_columns(data1_cba, exclude_columns)\n\nprint(\"Categorical features:\", cat_features_cba)\nprint(\"Numerical features:\", num_features_cba)\n\n\ndata1_cba.shape\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nX_num = data1_cba[num_features_cba]\nX_cat = data1_cba[cat_features_cba]\nnumerical_scores, categorical_scores = evaluate_feature_significance(X_num, X_cat, y_cba)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the dataframes\nprint(\"Numerical Scores:\")\nprint(numerical_scores)\nprint(\"\\nCategorical Scores:\")\nprint(categorical_scores)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_thresholds, best_score = find_best_threshold(data1_cba, y_cba, numerical_scores, categorical_scores)\n\nprint(\"Best Thresholds:\", best_thresholds)\nprint(\"Best Cross-Validation Score:\", best_score)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_scores['Kendall Tau Abs'] = numerical_scores['Kendall Tau'].abs()\n\n# Apply thresholds in a more consolidated way\nselected_numerical_features = numerical_scores[\n    (numerical_scores['ANOVA F-value'] > best_thresholds['anova_thresh']) |\n    (numerical_scores['Kendall Tau Abs'] > best_thresholds['kendall_thresh'])\n]['Feature'].tolist()\n\nselected_categorical_features = categorical_scores[\n    (categorical_scores['Chi2'] > best_thresholds['chi2_thresh']) |\n    (categorical_scores['Mutual Information'] > best_thresholds['mutual_info_thresh'])\n]['Feature'].tolist()\n\n# Combine selected features\nselected_features = selected_numerical_features + selected_categorical_features\n\nprint(\"Final Selected Features:\", selected_features)\ndel selected_numerical_features\ndel selected_categorical_features\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features.extend(['case_id', 'WEEK_NUM'])\nselected_features = list(set(selected_features))\ndata_cba_selected_train = data1_cba[selected_features]\ndata_cba_selected_test = data2_cba[selected_features]\n\n# Display the first few rows of the filtered data\nprint(data_cba_selected_train.head())\nprint(data_cba_selected_test.head())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make sure y is a DataFrame and matches data_cba_selected_train\ny_cba = y_cba.reset_index(drop=True)\ndata_cba_selected_train = data_cba_selected_train.reset_index(drop=True)\n\n# Combine data_cba_selected_train with y\ndata_cba_selected_train['target'] = y_cba","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### b. Application prev","metadata":{}},{"cell_type":"code","source":"# Create the plot\nplt.figure(figsize=(5.5, 17))\ncorrelation_series = data1_appl.corrwith(y_appl)\ncorrelation_series.plot.barh()\nplt.tight_layout()\n\n# Display the plot if you are in an interactive environment\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After the plot is no longer needed, close the plot and clear the variables\nplt.close()\n\n# Explicitly delete the variables holding large data\ndel correlation_series","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in data1_appl.columns:\n    if is_numeric_dtype(data1_appl[col]):\n        correlation, pvalue = pearsonr(data1_appl[col].values, y_appl.values)\n        print(f'{col: <40}: {correlation : .4f}, significant: {pvalue <= 0.05}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zdate = data1_appl['creationdate_885D']\ndata1_appl = data1_appl.drop('creationdate_885D', axis=1)\n# Filter the columns based on the p-value condition\nsignificant_columns = [col for col in data1_appl.columns if col in ['case_id', 'WEEK_NUM'] or pearsonr(data1_appl[col], y_appl)[1] <= 0.05]\n\n# Keep only the significant columns in the DataFrame\ndata1_appl = data1_appl[significant_columns]\ndata2_appl = data2_appl[significant_columns]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to identify categorical and numerical columns\ndef identify_columns(dataframe, exclude_columns):\n    cat_features = [col for col in dataframe.columns if col in catcols_appl and col not in exclude_columns]\n    num_features = [col for col in dataframe.columns if col in numcols_appl and col not in exclude_columns]\n    \n    return cat_features, num_features\n\n\n# Columns to exclude\nexclude_columns = ['case_id', 'WEEK_NUM']\n\n# Identify categorical and numerical columns in data1\ncat_features_appl, num_features_appl = identify_columns(data1_appl, exclude_columns)\n\nprint(\"Categorical features:\", cat_features_appl)\nprint(\"Numerical features:\", num_features_appl)\n\n\ndata1_appl.shape\n\ndel catcols_appl\ndel numcols_appl","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare data for analysis\nX_num = data1_appl[num_features_appl]\nX_cat = data1_appl[cat_features_appl]\nnumerical_scores, categorical_scores = evaluate_feature_significance(X_num, X_cat, y_appl)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Numerical Scores:\")\nprint(numerical_scores)\nprint(\"\\nCategorical Scores:\")\nprint(categorical_scores)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find best thresholds\nbest_thresholds, best_score = find_best_threshold(data1_appl, y_appl, numerical_scores, categorical_scores)\n\nprint(\"Best Thresholds:\", best_thresholds)\nprint(\"Best Cross-Validation Score:\", best_score)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_scores['Kendall Tau Abs'] = numerical_scores['Kendall Tau'].abs()\n\n# Apply thresholds in a more consolidated way\nselected_numerical_features = numerical_scores[\n    (numerical_scores['ANOVA F-value'] > best_thresholds['anova_thresh']) |\n    (numerical_scores['Kendall Tau Abs'] > best_thresholds['kendall_thresh'])\n]['Feature'].tolist()\n\nselected_categorical_features = categorical_scores[\n    (categorical_scores['Chi2'] > best_thresholds['chi2_thresh']) |\n    (categorical_scores['Mutual Information'] > best_thresholds['mutual_info_thresh'])\n]['Feature'].tolist()\n\n# Combine selected features\nselected_features = selected_numerical_features + selected_categorical_features\n\nprint(\"Final Selected Features:\", selected_features)\ndel selected_numerical_features\ndel selected_categorical_features","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features.extend(['case_id', 'WEEK_NUM'])\nselected_features = list(set(selected_features))\ndata_appl_selected_train = data1_appl[selected_features]\ndata_appl_selected_test = data2_appl[selected_features]\n\n# Display the first few rows of the filtered data\nprint(data_appl_selected_train.head())\nprint(data_appl_selected_test.head())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_appl = y_appl.reset_index(drop=True)\ndata_appl_selected_train = data_appl_selected_train.reset_index(drop=True)\n\n# Combine data_cba_selected_train with y\ndata_appl_selected_train['target'] = y_appl","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### c. Static","metadata":{}},{"cell_type":"code","source":"# Identify the importance level of numerical features using Pearson correlation\nfor col in data_static_train_1.columns:\n    if pd.api.types.is_numeric_dtype(data_static_train_1[col]):\n        correlation, pvalue = pearsonr(data_static_train_1[col].values, y_sta.values)\n        print(f'{col: <40}: {correlation:.4f}, significant: {pvalue <= 0.05}')\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to identify categorical and numerical columns\ndef identify_columns(dataframe, exclude_columns):\n    cat_features = [col for col in dataframe.columns if col in catcols_static and col not in exclude_columns]\n    num_features = [col for col in dataframe.columns if col in numcols_static and col not in exclude_columns]\n    \n    return cat_features, num_features\n\n\n# Columns to exclude\nexclude_columns = ['case_id', 'WEEK_NUM']\n\n# Identify categorical and numerical columns in data1\ncat_features_sta, num_features_sta = identify_columns(data_static_train_1, exclude_columns)\n\nprint(\"Categorical features:\", cat_features_sta)\nprint(\"Numerical features:\", num_features_sta)\n\n\ndata_static_train_1.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_num = data_static_train_1[num_features_sta]\nX_cat = data_static_train_1[cat_features_sta]\nnumerical_scores, categorical_scores = evaluate_feature_significance(X_num, X_cat, y_sta)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Numerical Scores:\")\nprint(numerical_scores)\nprint(\"\\nCategorical Scores:\")\nprint(categorical_scores)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nbest_thresholds, best_score = find_best_threshold(data_static_train_1, y_sta, numerical_scores, categorical_scores)\n\nprint(\"Best Thresholds:\", best_thresholds)\nprint(\"Best Cross-Validation Score:\", best_score)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_scores['Kendall Tau Abs'] = numerical_scores['Kendall Tau'].abs()\n\n# Apply thresholds in a more consolidated way\nselected_numerical_features = numerical_scores[\n    (numerical_scores['ANOVA F-value'] > best_thresholds['anova_thresh']) |\n    (numerical_scores['Kendall Tau Abs'] > best_thresholds['kendall_thresh'])\n]['Feature'].tolist()\n\nselected_categorical_features = categorical_scores[\n    (categorical_scores['Chi2'] > best_thresholds['chi2_thresh']) |\n    (categorical_scores['Mutual Information'] > best_thresholds['mutual_info_thresh'])\n]['Feature'].tolist()\n\n# Combine selected features\nselected_features = selected_numerical_features + selected_categorical_features\n\nprint(\"Final Selected Features:\", selected_features)\ndel selected_numerical_features\ndel selected_categorical_features","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features.extend(['case_id', 'WEEK_NUM'])\nselected_features = list(set(selected_features))\ndata_sta_selected_train = data_static_train_1[selected_features]\ndata_sta_selected_test = data_static_test_1[selected_features]\n\n# Display the first few rows of the filtered data\nprint(data_sta_selected_train.head())\nprint(data_sta_selected_test.head())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_sta = y_sta.reset_index(drop=True)\ndata_sta_selected_train = data_sta_selected_train.reset_index(drop=True)\n\n# Combine data_cba_selected_train with y\ndata_sta_selected_train['target'] = y_sta","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### d. Person","metadata":{}},{"cell_type":"code","source":"# Function to identify categorical and numerical columns\ndef identify_columns(dataframe, exclude_columns):\n    cat_features = [col for col in dataframe.columns if col in catcols_person and col not in exclude_columns]\n    num_features = [col for col in dataframe.columns if col in numcols_person and col not in exclude_columns]\n    \n    return cat_features, num_features\n\n\n# Columns to exclude\nexclude_columns = ['case_id', 'WEEK_NUM']\n\n# Identify categorical and numerical columns in data1\ncat_features_per, num_features_per = identify_columns(data_person_1_train_1, exclude_columns)\n\nprint(\"Categorical features:\", cat_features_per)\nprint(\"Numerical features:\", num_features_per)\n\n\ndata_person_1_train_1.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare data for analysis\nX_num = data_person_1_train_1[num_features_per]\nX_cat = data_person_1_train_1[cat_features_per]\nnumerical_scores, categorical_scores = evaluate_feature_significance(X_num, X_cat, y_appl)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Numerical Scores:\")\nprint(numerical_scores)\nprint(\"\\nCategorical Scores:\")\nprint(categorical_scores)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nbest_thresholds, best_score = find_best_threshold(data_person_1_train_1, y_per, numerical_scores, categorical_scores)\n\nprint(\"Best Thresholds:\", best_thresholds)\nprint(\"Best Cross-Validation Score:\", best_score)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_scores['Kendall Tau Abs'] = numerical_scores['Kendall Tau'].abs()\n\n# Apply thresholds in a more consolidated way\nselected_numerical_features = numerical_scores[\n    (numerical_scores['ANOVA F-value'] > best_thresholds['anova_thresh']) |\n    (numerical_scores['Kendall Tau Abs'] > best_thresholds['kendall_thresh'])\n]['Feature'].tolist()\n\nselected_categorical_features = categorical_scores[\n    (categorical_scores['Chi2'] > best_thresholds['chi2_thresh']) |\n    (categorical_scores['Mutual Information'] > best_thresholds['mutual_info_thresh'])\n]['Feature'].tolist()\n\n# Combine selected features\nselected_features = selected_numerical_features + selected_categorical_features\n\nprint(\"Final Selected Features:\", selected_features)\ndel selected_numerical_features\ndel selected_categorical_features","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features.extend(['case_id', 'WEEK_NUM'])\nselected_features = list(set(selected_features))\ndata_per_selected_train = data_person_1_train_1[selected_features]\ndata_per_selected_test = data_person_1_test_1[selected_features]\n\n# Display the first few rows of the filtered data\nprint(data_per_selected_train.head())\nprint(data_per_selected_test.head())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_per = y_per.reset_index(drop=True)\ndata_per_selected_train = data_per_selected_train.reset_index(drop=True)\n\n# Combine data_cba_selected_train with y\ndata_per_selected_train['target'] = y_per","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. DATA SPLITTING","metadata":{}},{"cell_type":"code","source":"# Merge the training data tables on 'case_id' using inner joins\ndata_train_merged = pd.merge(data_cba_selected_train, data_appl_selected_train, on='case_id', how='inner', suffixes=('', '_drop'))\ndata_train_merged = pd.merge(data_train_merged, data_sta_selected_train, on='case_id', how='inner', suffixes=('', '_drop'))\ndata_train_merged = pd.merge(data_train_merged, data_per_selected_train, on='case_id', how='inner', suffixes=('', '_drop'))\n\n# Remove any duplicate 'target' columns that might have been included from both tables\ndata_train_merged = data_train_merged.loc[:, ~data_train_merged.columns.str.endswith('_drop')]\n\n# Similarly, merge the test data tables\ndata_test_merged = pd.merge(data_cba_selected_test, data_appl_selected_test, on='case_id', how='inner', suffixes=('', '_drop'))\ndata_test_merged = pd.merge(data_test_merged, data_sta_selected_test, on='case_id', how='inner', suffixes=('', '_drop'))\ndata_test_merged = pd.merge(data_test_merged, data_per_selected_test, on='case_id', how='inner', suffixes=('', '_drop'))\n\n# Now let's verify the final DataFrame\nprint(data_train_merged.head())\nprint(data_test_merged.head())\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Training columns:\", len(data_train_merged.columns))\nprint(\"Test columns:\", len(data_test_merged.columns))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"case_ids = data_train_merged[\"case_id\"].unique()\n\n# Shuffle the case_ids using numpy.random.shuffle\nnp.random.seed(1)  # Set seed for reproducibility\nnp.random.shuffle(case_ids)\n\n# Perform the split of the case IDs using the train_test_split function, keeping a random state for reproducibility.\ncase_ids_train, case_ids_test = train_test_split(list(case_ids), train_size=0.6, random_state=1)\ncase_ids_valid, case_ids_test = train_test_split(case_ids_test, train_size=0.5, random_state=1)\n\n# Prepare a list of predictive columns based on naming conventions (ending in uppercase, rest lowercase).\ncols_pred = []\nfor col in data_train_merged.columns:\n    if col[-1].isupper() and col[:-1].islower():\n        cols_pred.append(col)\n\nprint(cols_pred)\n\n# Filter data frames based on the case_ids in train, valid, and test splits\nbase_train = data_train_merged[data_train_merged['case_id'].isin(case_ids_train)]\nbase_valid = data_train_merged[data_train_merged['case_id'].isin(case_ids_valid)]\nbase_test = data_train_merged[data_train_merged['case_id'].isin(case_ids_test)]\n\n# Assuming 'target' is the name of your target column in the merged data frame\nX_train = base_train.drop(['target'], axis=1)\ny_train = base_train['target']\n\nX_valid = base_valid.drop(['target'], axis=1)\ny_valid = base_valid['target']\n\nX_test = base_test.drop(['target'], axis=1)\ny_test = base_test['target']\n\n# Optionally, print the shapes to verify\nprint(\"Training shapes:\", X_train.shape, y_train.shape)\nprint(\"Validation shapes:\", X_valid.shape, y_valid.shape)\nprint(\"Testing shapes:\", X_test.shape, y_test.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from imblearn.under_sampling import RandomUnderSampler\n# Perform undersampling\nrus = RandomUnderSampler(random_state=1)\nX_train_res, y_train_res = rus.fit_resample(X_train, y_train)\n\n# Optionally, print the shapes to verify undersampling\nprint(\"Resampled Training shapes:\", X_train_res.shape, y_train_res.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. MODEL TRAINING","metadata":{}},{"cell_type":"code","source":"lgb_train = lgb.Dataset(X_train_res, label=y_train_res)\nlgb_valid = lgb.Dataset(X_valid, label=y_valid, reference=lgb_train)\n\n# Define model parameters.\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 3,\n    \"num_leaves\": 31,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.9,\n    \"bagging_fraction\": 0.8,\n    \"bagging_freq\": 5,\n    \"n_estimators\": 1000,\n    \"verbose\": -1,\n}\n\n# Train the LightGBM model with early stopping based on validation set performance.\ngbm = lgb.train(\n    params,\n    lgb_train,\n    valid_sets=lgb_valid,\n    callbacks=[lgb.log_evaluation(50), lgb.early_stopping(10)]\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for base, X in [(base_train, X_train), (base_valid, X_valid), (base_test, X_test)]:\n    y_pred = gbm.predict(X, num_iteration=gbm.best_iteration)\n    base.loc[:, \"score\"] = y_pred\n\n# Print out the AUC scores for each set.\nprint(f'The AUC score on the train set is: {roc_auc_score(base_train[\"target\"], base_train[\"score\"])}') \nprint(f'The AUC score on the valid set is: {roc_auc_score(base_valid[\"target\"], base_valid[\"score\"])}') \nprint(f'The AUC score on the test set is: {roc_auc_score(base_test[\"target\"], base_test[\"score\"])}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. STABILITY","metadata":{}},{"cell_type":"code","source":"# Define a function to calculate the Gini stability score over time.\ndef gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n     # Calculate Gini scores over time and fit a linear model to determine the stability.\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\n# Calculate stability scores for each dataset.\nstability_score_train = gini_stability(base_train)\nstability_score_valid = gini_stability(base_valid)\nstability_score_test = gini_stability(base_test)\n\n# Print out the stability scores for each set.\nprint(f'The stability score on the train set is: {stability_score_train}') \nprint(f'The stability score on the valid set is: {stability_score_valid}') \nprint(f'The stability score on the test set is: {stability_score_test}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_submission = data_test_merged[cols_pred]\n\ny_submission_pred = gbm.predict(X_submission, num_iteration=gbm.best_iteration,predict_disable_shape_check=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. SUBMISSION","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": data_test_merged[\"case_id\"].to_numpy(),\n    \"score\": y_submission_pred\n}).set_index('case_id')\nsubmission.to_csv(\"./submission.csv\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_memory()","metadata":{},"execution_count":null,"outputs":[]}]}