{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8229779,"sourceType":"datasetVersion","datasetId":4481778}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Home Credit Load Train Data P3","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport gc, os, warnings\n\npathway = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:17:42.208985Z","iopub.execute_input":"2024-04-25T18:17:42.209894Z","iopub.status.idle":"2024-04-25T18:17:42.216553Z","shell.execute_reply.started":"2024-04-25T18:17:42.209852Z","shell.execute_reply":"2024-04-25T18:17:42.214921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame)-> pl.DataFrame:\n    for col in df.columns:\n        # Cast Transform DPD (Days past due, P) and Transform Amount (A) as Float64\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        # Cast Transform date (D) as Date\n        if col[-1] in (\"D\"):\n            df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        # Cast aggregated columns as Float64, tried combining sum and max, but did not work correctly\n        if col[-4:-1] in ('_sum'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        if col[-4:-1] in ('_max'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    return df\n\ndef convert_strings(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == pl.Utf8:\n            df = df.with_columns(pl.col(col).cast(pl.Categorical))\n    return df\n\n# Changed this function to work for Pandas\ndef missing_values(df, threshold = 0.9):\n    for col in df.columns:\n        decimal = (pd.isnull(df[col]).sum())/(len(df[col]))\n        if decimal > threshold:                                         \n            print(f\"{col}: {decimal}\")\n\n# Impute numeric columns with the median and cat with mode\ndef imputer(df:pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == 'float64':\n            df[col] = df[col].fillna(df[col].median())\n        if df[col].dtype.name in ['category','object'] and df[col].isnull().any():\n            mode_without_nan = df[col].dropna().mode().values[0]\n            df[col] = df[col].fillna(mode_without_nan)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:17:42.225783Z","iopub.execute_input":"2024-04-25T18:17:42.226858Z","iopub.status.idle":"2024-04-25T18:17:42.243759Z","shell.execute_reply.started":"2024-04-25T18:17:42.226785Z","shell.execute_reply":"2024-04-25T18:17:42.242224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_range_agg(series: pl.series):\n    custom_range = series.max() - series.min()\n    return custom_range\n\ndef compare_cols(df):\n    range_list = [element for e in df.columns for element in e.split() if element.endswith(\"_range\")]\n    sum_list = [name.replace('range', 'sum') for name in range_list]\n    length = len(df)\n\n    df=df.with_columns(*[pl.col(col).fill_null(strategy='zero') for col in range_list])\n    \n    for i in range(len(range_list)):\n        df = df.with_columns(check=\n        pl.when((df[range_list[i]] == df[sum_list[i]]) | (df[range_list[i]] == 0)).then(1).otherwise(0))\n\n        percent = df.select(pl.sum(\"check\")).item() / length\n        print(f\"Percent equal or 0 for {range_list[i]} = {percent:.3f}\")\n        \n    df = df.drop(\"check\")\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:17:42.246119Z","iopub.execute_input":"2024-04-25T18:17:42.246480Z","iopub.status.idle":"2024-04-25T18:17:42.261517Z","shell.execute_reply.started":"2024-04-25T18:17:42.246451Z","shell.execute_reply":"2024-04-25T18:17:42.259976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable = pl.read_csv(pathway + \"csv_files/train/train_base.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:17:42.263738Z","iopub.execute_input":"2024-04-25T18:17:42.264662Z","iopub.status.idle":"2024-04-25T18:17:43.169281Z","shell.execute_reply.started":"2024-04-25T18:17:42.264623Z","shell.execute_reply":"2024-04-25T18:17:43.168144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional training data, depth = 2\ntrain_applprev_2 = pl.read_csv(pathway + \"csv_files/train/train_applprev_2.csv\").pipe(set_table_dtypes)\n\ntrain_person_2 = pl.read_csv(pathway + \"csv_files/train/train_person_2.csv\").pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:17:43.170653Z","iopub.execute_input":"2024-04-25T18:17:43.171165Z","iopub.status.idle":"2024-04-25T18:17:47.087911Z","shell.execute_reply.started":"2024-04-25T18:17:43.171131Z","shell.execute_reply":"2024-04-25T18:17:47.086887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sel = ['case_id', 'collater_valueofguarantee_1124L','num_group1','num_group2','pmts_dpd_1073P','pmts_dpd_303P','pmts_overdue_1140A','pmts_overdue_1152A']\ntrain_credit_bureau_a_2 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/train/train_credit_bureau_a_2_0.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/train/train_credit_bureau_a_2_1.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/train/train_credit_bureau_a_2_2.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/train/train_credit_bureau_a_2_3.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/train/train_credit_bureau_a_2_4.csv\",columns=sel).pipe(set_table_dtypes),\n    \n    pl.read_csv(pathway + \"csv_files/train/train_credit_bureau_a_2_5.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/train/train_credit_bureau_a_2_6.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/train/train_credit_bureau_a_2_7.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/train/train_credit_bureau_a_2_8.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/train/train_credit_bureau_a_2_9.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/train/train_credit_bureau_a_2_10.csv\",columns=sel).pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:17:47.090754Z","iopub.execute_input":"2024-04-25T18:17:47.091856Z","iopub.status.idle":"2024-04-25T18:19:12.724983Z","shell.execute_reply.started":"2024-04-25T18:17:47.091751Z","shell.execute_reply":"2024-04-25T18:19:12.721424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Look at data","metadata":{}},{"cell_type":"code","source":"# selection = list_missing(train_applprev_2, threshold=0.90)\n#print(selection)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:12.728512Z","iopub.execute_input":"2024-04-25T18:19:12.729023Z","iopub.status.idle":"2024-04-25T18:19:12.737528Z","shell.execute_reply.started":"2024-04-25T18:19:12.728976Z","shell.execute_reply":"2024-04-25T18:19:12.734852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_applprev_2 = train_applprev_2.drop('credacc_cards_status_52L')","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:12.740987Z","iopub.execute_input":"2024-04-25T18:19:12.741895Z","iopub.status.idle":"2024-04-25T18:19:12.777333Z","shell.execute_reply.started":"2024-04-25T18:19:12.741797Z","shell.execute_reply":"2024-04-25T18:19:12.774486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Does not seem to be any useful features in this df\n#selection = list_missing(train_person_2, threshold=0.90)\nselection = ['addres_role_871L', 'empls_employedfrom_796D', 'relatedpersons_role_762T']\ntrain_person_2 = train_person_2.drop(selection)\ntrain_person_2.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:12.781915Z","iopub.execute_input":"2024-04-25T18:19:12.783047Z","iopub.status.idle":"2024-04-25T18:19:12.827382Z","shell.execute_reply.started":"2024-04-25T18:19:12.782919Z","shell.execute_reply":"2024-04-25T18:19:12.824695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#selection = list_missing(train_credit_bureau_a_2, threshold=0.90)\n#print(selection)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:12.831315Z","iopub.execute_input":"2024-04-25T18:19:12.832990Z","iopub.status.idle":"2024-04-25T18:19:12.844531Z","shell.execute_reply.started":"2024-04-25T18:19:12.832891Z","shell.execute_reply":"2024-04-25T18:19:12.841087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_a_2 = train_credit_bureau_a_2.drop('collater_valueofguarantee_1124L')","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:12.850879Z","iopub.execute_input":"2024-04-25T18:19:12.851686Z","iopub.status.idle":"2024-04-25T18:19:13.104325Z","shell.execute_reply.started":"2024-04-25T18:19:12.851602Z","shell.execute_reply":"2024-04-25T18:19:13.100438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{}},{"cell_type":"code","source":"train_applprev_2_feats = train_applprev_2.select([\"case_id\", \"num_group1\", \"num_group2\",\n    \"conts_type_509L\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\") \n\ntrain_credit_bureau_a_2_feats = train_credit_bureau_a_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_dpd_1073P\").sum().alias(\"pmts_dpd_1073P_sum\"),\n    custom_range_agg(pl.col(\"pmts_dpd_1073P\")).alias('pmts_dpd_1073P_range'),\n    pl.col(\"pmts_dpd_303P\").sum().alias(\"pmts_dpd_303P_sum\"),\n    custom_range_agg(pl.col(\"pmts_dpd_303P\")).alias('pmts_dpd_303P_range'),\n    pl.col(\"pmts_overdue_1140A\").sum().alias(\"pmts_overdue_1140A_sum\"),\n    custom_range_agg(pl.col(\"pmts_overdue_1140A\")).alias('pmts_overdue_1140A_range'),\n    pl.col(\"pmts_overdue_1152A\").sum().alias(\"pmts_overdue_1152A_sum\"),\n    custom_range_agg(pl.col(\"pmts_overdue_1152A\")).alias('pmts_overdue_1152A_range'))\n\ntrain_person_2_feats = train_person_2.select([\"case_id\", \"num_group1\", \"num_group2\", \"addres_zip_823M\",\n    \"addres_district_368M\",\"conts_role_79M\",\"empls_economicalst_849M\",\"empls_employer_name_740M\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\") ","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:13.109862Z","iopub.execute_input":"2024-04-25T18:19:13.110997Z","iopub.status.idle":"2024-04-25T18:19:23.784077Z","shell.execute_reply.started":"2024-04-25T18:19:13.110891Z","shell.execute_reply":"2024-04-25T18:19:23.782948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# No range cols removed\ntrain_credit_bureau_a_2_feats = compare_cols(train_credit_bureau_a_2_feats)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:23.785967Z","iopub.execute_input":"2024-04-25T18:19:23.786962Z","iopub.status.idle":"2024-04-25T18:19:23.922115Z","shell.execute_reply.started":"2024-04-25T18:19:23.786914Z","shell.execute_reply":"2024-04-25T18:19:23.919212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data3 = train_basetable.join(train_applprev_2_feats, how=\"left\", on=\"case_id\"\n).join(train_credit_bureau_a_2_feats, how=\"left\", on=\"case_id\").join(train_person_2_feats, how=\"left\", on=\"case_id\")","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:23.925166Z","iopub.execute_input":"2024-04-25T18:19:23.927302Z","iopub.status.idle":"2024-04-25T18:19:25.242981Z","shell.execute_reply.started":"2024-04-25T18:19:23.927202Z","shell.execute_reply":"2024-04-25T18:19:25.241895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data3 = join_data3.drop(['date_decision','MONTH','WEEK_NUM','target'])\njoin_data3.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:25.251548Z","iopub.execute_input":"2024-04-25T18:19:25.252013Z","iopub.status.idle":"2024-04-25T18:19:25.271730Z","shell.execute_reply.started":"2024-04-25T18:19:25.251976Z","shell.execute_reply":"2024-04-25T18:19:25.270615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values(join_data3,0.9)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:25.273233Z","iopub.execute_input":"2024-04-25T18:19:25.273978Z","iopub.status.idle":"2024-04-25T18:19:27.519196Z","shell.execute_reply.started":"2024-04-25T18:19:25.273944Z","shell.execute_reply":"2024-04-25T18:19:27.517896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Merge all previously joined dfs","metadata":{}},{"cell_type":"code","source":"join_data1 = pl.read_csv(\"/kaggle/input/joineddf/join_train_1_90.csv\").pipe(set_table_dtypes)\njoin_data2 = pl.read_csv(\"/kaggle/input/joineddf/join_train_2_90new.csv\").pipe(set_table_dtypes)\nprint(join_data1.shape)\nprint(join_data2.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:27.520652Z","iopub.execute_input":"2024-04-25T18:19:27.521010Z","iopub.status.idle":"2024-04-25T18:19:45.509701Z","shell.execute_reply.started":"2024-04-25T18:19:27.520982Z","shell.execute_reply":"2024-04-25T18:19:45.508372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = join_data1.join(join_data2, how=\"left\", on=\"case_id\"\n).join(join_data3, how=\"left\", on=\"case_id\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:45.511375Z","iopub.execute_input":"2024-04-25T18:19:45.512302Z","iopub.status.idle":"2024-04-25T18:19:47.842278Z","shell.execute_reply.started":"2024-04-25T18:19:45.512247Z","shell.execute_reply":"2024-04-25T18:19:47.840635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_basetable,train_person_2, train_person_2_feats,train_credit_bureau_a_2, train_credit_bureau_a_2_feats, train_applprev_2, train_applprev_2_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:47.843675Z","iopub.execute_input":"2024-04-25T18:19:47.844066Z","iopub.status.idle":"2024-04-25T18:19:48.985086Z","shell.execute_reply.started":"2024-04-25T18:19:47.844032Z","shell.execute_reply":"2024-04-25T18:19:48.983668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del join_data1, join_data2, join_data3 \ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:48.987000Z","iopub.execute_input":"2024-04-25T18:19:48.987381Z","iopub.status.idle":"2024-04-25T18:19:49.467194Z","shell.execute_reply.started":"2024-04-25T18:19:48.987350Z","shell.execute_reply":"2024-04-25T18:19:49.465869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.with_columns(pl.col('date_decision').cast(pl.Date)).with_columns(pl.col('birth_259D').cast(pl.Date))","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:49.468784Z","iopub.execute_input":"2024-04-25T18:19:49.469508Z","iopub.status.idle":"2024-04-25T18:19:49.892610Z","shell.execute_reply.started":"2024-04-25T18:19:49.469464Z","shell.execute_reply":"2024-04-25T18:19:49.891264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature engineer date columns\ndate_list = ['datefirstoffer_1144D', 'datelastunpaid_3546854D', 'dtlastpmtallstes_4499206D', 'firstclxcampaign_1125D', 'lastdelinqdate_224D', 'lastrejectdate_50D', 'maxdpdinstldate_3546855D', 'birthdate_574D', 'responsedate_1012D', 'responsedate_4527233D', 'responsedate_4917613D', 'empl_employedfrom_271D',\n            'firstdatedue_489D','lastactivateddate_801D','lastapplicationdate_877D', 'lastapprdate_640D', 'dateofbirth_337D','birth_259D']\n\nfor col in date_list:\n    train = train.with_columns(\n        ((pl.col(\"date_decision\") - pl.col(col)) / (24 * 60 * 60 * 1000)).cast(pl.Float64).alias(f\"{col}_diff\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:49.894215Z","iopub.execute_input":"2024-04-25T18:19:49.894701Z","iopub.status.idle":"2024-04-25T18:19:51.272645Z","shell.execute_reply.started":"2024-04-25T18:19:49.894657Z","shell.execute_reply":"2024-04-25T18:19:51.271421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop uneeded date columns after extraction\ndrop_list = ['date_decision','MONTH'] + date_list\ntrain = train.pipe(set_table_dtypes).pipe(convert_strings).drop(drop_list)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:51.275197Z","iopub.execute_input":"2024-04-25T18:19:51.275721Z","iopub.status.idle":"2024-04-25T18:19:55.917247Z","shell.execute_reply.started":"2024-04-25T18:19:51.275663Z","shell.execute_reply":"2024-04-25T18:19:55.915654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train.columns:\n        if col[-5:-1] in (\"range\"):\n            train = train.with_columns(pl.col(col).cast(pl.Float64).alias(col))","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:55.918719Z","iopub.execute_input":"2024-04-25T18:19:55.919093Z","iopub.status.idle":"2024-04-25T18:19:56.050879Z","shell.execute_reply.started":"2024-04-25T18:19:55.919063Z","shell.execute_reply":"2024-04-25T18:19:56.049388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:56.052928Z","iopub.execute_input":"2024-04-25T18:19:56.053323Z","iopub.status.idle":"2024-04-25T18:19:59.442588Z","shell.execute_reply.started":"2024-04-25T18:19:56.053289Z","shell.execute_reply":"2024-04-25T18:19:59.441241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_cols = train.select_dtypes(include='number').columns.tolist()\nfor col in numeric_cols:\n    if (train[col].max()-train[col].min()) == 0:\n        print(f'{col}: {train[col].max()-train[col].min()}')","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:19:59.444428Z","iopub.execute_input":"2024-04-25T18:19:59.444785Z","iopub.status.idle":"2024-04-25T18:20:03.918124Z","shell.execute_reply.started":"2024-04-25T18:19:59.444755Z","shell.execute_reply":"2024-04-25T18:20:03.916904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from above\nselection = ['commnoinclast6m_3546845L', 'deferredmnthsnum_166L', 'mastercontrelectronic_519L', 'mastercontrexist_109L']\ntrain = train.drop(selection, axis=1)\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:20:03.919795Z","iopub.execute_input":"2024-04-25T18:20:03.920294Z","iopub.status.idle":"2024-04-25T18:20:05.059097Z","shell.execute_reply.started":"2024-04-25T18:20:03.920239Z","shell.execute_reply":"2024-04-25T18:20:05.057503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bool_cols = train.select_dtypes(include=['bool']).columns.tolist() # only picked up isbidproduct_1095L\n# Convert boolean columns to 0 or 1 (False or True)\nfor col in bool_cols:\n    train[col] = train[col].astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:20:05.060794Z","iopub.execute_input":"2024-04-25T18:20:05.061308Z","iopub.status.idle":"2024-04-25T18:20:05.074216Z","shell.execute_reply.started":"2024-04-25T18:20:05.061263Z","shell.execute_reply":"2024-04-25T18:20:05.072539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# taken from manual inspection \n# Noticed boolean conversion converts nan's to 0, so leave it as categorical\nbool_cols = ['isdebitcard_729L', 'opencred_647L','safeguarantyflag_411L','isbidproduct_390L']\nfor col in bool_cols:\n    train[col] = train[col].astype(bool)\n    train[col] = train[col].astype(int)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:20:05.076468Z","iopub.execute_input":"2024-04-25T18:20:05.076984Z","iopub.status.idle":"2024-04-25T18:20:05.090017Z","shell.execute_reply.started":"2024-04-25T18:20:05.076929Z","shell.execute_reply":"2024-04-25T18:20:05.088221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_list = ['numinstlswithdpd5_4187116L', 'numinstmatpaidtearly2d_4499204L','numinstpaid_4499208L','numinstpaidearly3dest_4493216L','numinstpaidearly5dest_4493211L',\n           'numinstpaidearly5dobd_4499205L','numinstpaidearlyest_4493214L','numinstpaidlastcontr_4325080L','numinstregularpaidest_4493210L','numinsttopaygrest_4493213L','numinstunpaidmaxest_4493212L',\n           'contractssum_5085716L','days120_123L','days180_256L','days360_512L','firstquarter_103L','fourthquarter_440L','numberofqueries_373L','pmtscount_423L','secondquarter_766L','thirdquarter_1082L',\n           'days30_165L','days90_310L']\nfor col in num_list:\n    train[col]=train[col].astype('float64')","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:20:05.092873Z","iopub.execute_input":"2024-04-25T18:20:05.093242Z","iopub.status.idle":"2024-04-25T18:20:05.368179Z","shell.execute_reply.started":"2024-04-25T18:20:05.093210Z","shell.execute_reply":"2024-04-25T18:20:05.366856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows', None)\npd.set_option('display.max_columns', None)\n# train.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:20:05.369701Z","iopub.execute_input":"2024-04-25T18:20:05.370112Z","iopub.status.idle":"2024-04-25T18:20:05.375735Z","shell.execute_reply.started":"2024-04-25T18:20:05.370078Z","shell.execute_reply":"2024-04-25T18:20:05.374503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify all cat_cols should be categorical\ncat_cols = train.select_dtypes(include=['category','object']).columns.tolist()\ntrain[cat_cols].head(15)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:20:05.377834Z","iopub.execute_input":"2024-04-25T18:20:05.378246Z","iopub.status.idle":"2024-04-25T18:20:05.868963Z","shell.execute_reply.started":"2024-04-25T18:20:05.378213Z","shell.execute_reply":"2024-04-25T18:20:05.866964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Some numeric columns processed as cat, view cat_cols with # of cats and percent of distribution belonging to the top category\nwarnings.filterwarnings(\"ignore\")\ndrop_list =[]\n# View columns with > 150 unique categories for each variable\nfor col in (cat_cols):\n    if len(train[col].unique()) > 150:\n        drop_list.append(col)\n        print(f'{col}: {len(train[col].unique())}, {round(train[col].value_counts().sort_values(ascending=False)[0]/train[col].value_counts().sum(),4)}')\nwarnings.filterwarnings('default')","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:21:20.646702Z","iopub.execute_input":"2024-04-25T18:21:20.647297Z","iopub.status.idle":"2024-04-25T18:21:21.474263Z","shell.execute_reply.started":"2024-04-25T18:21:20.647250Z","shell.execute_reply":"2024-04-25T18:21:21.472933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# List from above\n#selection = ['lastapprcommoditytypec_5251766M','lastrejectcommodtypec_5251769M','previouscontdistrict_112M','addres_zip_823M', 'addres_district_368M']\nselection = ['addres_zip_823M']\ntrain = train.drop(selection, axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:22:57.280048Z","iopub.execute_input":"2024-04-25T18:22:57.280513Z","iopub.status.idle":"2024-04-25T18:22:58.811732Z","shell.execute_reply.started":"2024-04-25T18:22:57.280483Z","shell.execute_reply":"2024-04-25T18:22:58.810480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Impute here if needed\nprint(np.count_nonzero(train.isnull()))\ntrain = imputer(train)\nprint(np.count_nonzero(train.isnull()))\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:23:13.959704Z","iopub.execute_input":"2024-04-25T18:23:13.960147Z","iopub.status.idle":"2024-04-25T18:23:28.663020Z","shell.execute_reply.started":"2024-04-25T18:23:13.960116Z","shell.execute_reply":"2024-04-25T18:23:28.661706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n%%time\ntrain = pd.get_dummies(train, dtype=int, columns=cat_cols, sparse=True, drop_first=False)\ntrain.head()\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Final check no numeric columns are all the same\nnumeric_cols = train.select_dtypes(include='number').columns.tolist()\nfor col in numeric_cols:\n    if (train[col].max()-train[col].min()) == 0:\n        print(f'{col}: {train[col].max()-train[col].min()}')","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:23:35.395573Z","iopub.execute_input":"2024-04-25T18:23:35.396022Z","iopub.status.idle":"2024-04-25T18:23:41.457313Z","shell.execute_reply.started":"2024-04-25T18:23:35.395988Z","shell.execute_reply":"2024-04-25T18:23:41.456314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values(train, 0.9)\ntrain.shape # 255 cols","metadata":{"execution":{"iopub.status.busy":"2024-04-25T18:23:41.458838Z","iopub.execute_input":"2024-04-25T18:23:41.459417Z","iopub.status.idle":"2024-04-25T18:23:42.059507Z","shell.execute_reply.started":"2024-04-25T18:23:41.459385Z","shell.execute_reply":"2024-04-25T18:23:42.058105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain.to_csv(\"train255_noimpute.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}