{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7869421,"sourceType":"datasetVersion","datasetId":4506020}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Home Credit EDA\n### Kevin Wang and Andrea Adams","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nimport os","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:10.711763Z","iopub.execute_input":"2024-03-19T16:41:10.712249Z","iopub.status.idle":"2024-03-19T16:41:12.560555Z","shell.execute_reply.started":"2024-03-19T16:41:10.712208Z","shell.execute_reply":"2024-03-19T16:41:12.559289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Code to delete all output in kaggle\ndef remove_folder_contents(folder):\n    for the_file in os.listdir(folder):\n        file_path = os.path.join(folder, the_file)\n        try:\n            if os.path.isfile(file_path):\n                os.unlink(file_path)\n            elif os.path.isdir(file_path):\n                remove_folder_contents(file_path)\n                os.rmdir(file_path)\n        except Exception as e:\n            print(e)\n\nfolder_path = '/kaggle/working'\nremove_folder_contents(folder_path)\nos.rmdir(folder_path)\n\"\"\"\n","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:12.562847Z","iopub.execute_input":"2024-03-19T16:41:12.563346Z","iopub.status.idle":"2024-03-19T16:41:12.574408Z","shell.execute_reply.started":"2024-03-19T16:41:12.563314Z","shell.execute_reply":"2024-03-19T16:41:12.572488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## UDFS","metadata":{}},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame)-> pl.DataFrame:\n    for col in df.columns:\n        # Cast Transform DPD (Days past due, P) and Transform Amount (A) as Float64\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        # Cast Transform date (D) as Date\n        if col[-1] in (\"D\"):\n            df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        # Cast aggregated columns as Float64, tried combining sum and max, but did not work correctly\n        if col[-4:-1] in ('_sum'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        if col[-4:-1] in ('_max'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    return df\n\ndef convert_strings(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == pl.Utf8:\n            df = df.with_columns(pl.col(col).cast(pl.Categorical))\n    return df\n\n# Changed this function to work for Pandas\ndef missing_values(df, threshold = 0.0):\n    for col in df.columns:\n        decimal = (pd.isnull(train[col]).sum())/(len(train[col]))\n        if decimal > threshold:                                         \n            print(f\"{col}: {decimal}\")\n\n# Impute numeric columns with the median and cat with mode\ndef imputer(df:pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == 'float64':\n            df[col] = df[col].fillna(df[col].median())\n        if df[col].dtype.name in ['category','object'] and df[col].isnull().any():\n            mode_without_nan = df[col].dropna().mode().values[0]\n            df[col] = df[col].fillna(mode_without_nan)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:12.575724Z","iopub.execute_input":"2024-03-19T16:41:12.576202Z","iopub.status.idle":"2024-03-19T16:41:12.596445Z","shell.execute_reply.started":"2024-03-19T16:41:12.576134Z","shell.execute_reply":"2024-03-19T16:41:12.594968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pl.read_csv('/kaggle/input/training/train_final.csv').with_columns(pl.col('date_decision').cast(pl.Date)).with_columns(pl.col('birth_259D').cast(pl.Date))","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:12.599424Z","iopub.execute_input":"2024-03-19T16:41:12.599825Z","iopub.status.idle":"2024-03-19T16:41:28.992530Z","shell.execute_reply.started":"2024-03-19T16:41:12.599794Z","shell.execute_reply":"2024-03-19T16:41:28.991243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.with_columns(\n    ((pl.col(\"date_decision\") - pl.col(\"birth_259D\")) / (24 * 60 * 60 * 1000)).cast(pl.Float64).alias(\"date_diff\"))","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:28.993760Z","iopub.execute_input":"2024-03-19T16:41:28.994094Z","iopub.status.idle":"2024-03-19T16:41:29.108173Z","shell.execute_reply.started":"2024-03-19T16:41:28.994064Z","shell.execute_reply":"2024-03-19T16:41:29.106738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Subtract birth date from decision date and cast as a float (diff in days)\ntrain = train.with_columns(\n    ((pl.col(\"date_decision\") - pl.col(\"birth_259D\")) / (24 * 60 * 60 * 1000)).cast(pl.Float64).alias(\"date_diff\"))\n\n# Drop uneeded date columns after extraction\ndate_list = ['date_decision','MONTH','firstdatedue_489D','lastactivateddate_801D',\n            'lastapplicationdate_877D', 'lastapprdate_640D', 'dateofbirth_337D','birth_259D']\ntrain = train.pipe(set_table_dtypes).pipe(convert_strings).drop(date_list)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:29.109740Z","iopub.execute_input":"2024-03-19T16:41:29.110186Z","iopub.status.idle":"2024-03-19T16:41:34.945663Z","shell.execute_reply.started":"2024-03-19T16:41:29.110143Z","shell.execute_reply":"2024-03-19T16:41:34.943850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert polars df to pandas so pandas specific methods/attributes work later\ntrain = train.to_pandas()\n\n# Noticed some of these numeric variables were parsed as strings, changed them all to int \nnum_list = ['days120_123L','days180_256L','days30_165L','days360_512L','days90_310L','firstquarter_103L','numinstpaidlastcontr_4325080L',\n            'fourthquarter_440L','numinstlswithdpd5_4187116L','numberofqueries_373L', 'secondquarter_766L','thirdquarter_1082L']\nfor col in num_list:\n    train[col]=train[col].astype('float64')\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:34.947164Z","iopub.execute_input":"2024-03-19T16:41:34.947513Z","iopub.status.idle":"2024-03-19T16:41:38.597135Z","shell.execute_reply.started":"2024-03-19T16:41:34.947483Z","shell.execute_reply":"2024-03-19T16:41:38.595836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check target distribution\ntrain.target.value_counts().sort_values().plot(kind = 'bar')","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:38.598618Z","iopub.execute_input":"2024-03-19T16:41:38.600011Z","iopub.status.idle":"2024-03-19T16:41:38.947380Z","shell.execute_reply.started":"2024-03-19T16:41:38.599965Z","shell.execute_reply":"2024-03-19T16:41:38.946154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Target 1 represents about 3.14% of the outcomes\nprint(round(train.target.value_counts()[1]/train.target.value_counts().sum(),4))","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:38.948906Z","iopub.execute_input":"2024-03-19T16:41:38.949326Z","iopub.status.idle":"2024-03-19T16:41:38.988293Z","shell.execute_reply.started":"2024-03-19T16:41:38.949295Z","shell.execute_reply":"2024-03-19T16:41:38.986989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# median of all numeric columns before imputation, should be same after\nnumeric_cols = train.select_dtypes(include='number').columns.tolist()\ntrain[numeric_cols].median()","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:38.992445Z","iopub.execute_input":"2024-03-19T16:41:38.992839Z","iopub.status.idle":"2024-03-19T16:41:46.468299Z","shell.execute_reply.started":"2024-03-19T16:41:38.992808Z","shell.execute_reply":"2024-03-19T16:41:46.467035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in numeric_cols:\n    if (train[col].max()-train[col].min()) == 0:\n        print(f'{col}: {train[col].max()-train[col].min()}')","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:46.470317Z","iopub.execute_input":"2024-03-19T16:41:46.470927Z","iopub.status.idle":"2024-03-19T16:41:48.871589Z","shell.execute_reply.started":"2024-03-19T16:41:46.470892Z","shell.execute_reply":"2024-03-19T16:41:48.870216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Numeric columns with all same value (0 range), drop\nselection = ['commnoinclast6m_3546845L', 'deferredmnthsnum_166L', 'mastercontrelectronic_519L', 'mastercontrexist_109L']\ntrain.drop(columns=selection, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:48.873265Z","iopub.execute_input":"2024-03-19T16:41:48.873663Z","iopub.status.idle":"2024-03-19T16:41:49.692534Z","shell.execute_reply.started":"2024-03-19T16:41:48.873605Z","shell.execute_reply":"2024-03-19T16:41:49.690842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = train.select_dtypes(include=['category']).columns.tolist()\n#print the mode of each categorical/object variable\ntrain[cat_cols].mode()","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:49.694048Z","iopub.execute_input":"2024-03-19T16:41:49.694434Z","iopub.status.idle":"2024-03-19T16:41:50.192567Z","shell.execute_reply.started":"2024-03-19T16:41:49.694402Z","shell.execute_reply":"2024-03-19T16:41:50.191174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count of missing values in DF\nnp.count_nonzero(train.isnull())","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:50.193907Z","iopub.execute_input":"2024-03-19T16:41:50.194253Z","iopub.status.idle":"2024-03-19T16:41:51.675240Z","shell.execute_reply.started":"2024-03-19T16:41:50.194222Z","shell.execute_reply":"2024-03-19T16:41:51.674095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Medians and modes after imputation are the same, warning message does not seem to be an issue\ntrain = imputer(train)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:51.677223Z","iopub.execute_input":"2024-03-19T16:41:51.677769Z","iopub.status.idle":"2024-03-19T16:41:59.943817Z","shell.execute_reply.started":"2024-03-19T16:41:51.677728Z","shell.execute_reply":"2024-03-19T16:41:59.942476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count of missing values in DF\nnp.count_nonzero(train.isnull())","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:41:59.945360Z","iopub.execute_input":"2024-03-19T16:41:59.945811Z","iopub.status.idle":"2024-03-19T16:42:01.122246Z","shell.execute_reply.started":"2024-03-19T16:41:59.945771Z","shell.execute_reply":"2024-03-19T16:42:01.120973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# See boolean columns\nbool_cols = train.select_dtypes(include=['bool']).columns.tolist()\n# Convert boolean columns to 0 or 1 (False or True)\nfor col in bool_cols:\n    train[col] = train[col].astype(int)\n# Check unique values of bool_cols\nfor col in bool_cols:\n    print(train[col].unique())","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:42:01.123633Z","iopub.execute_input":"2024-03-19T16:42:01.123988Z","iopub.status.idle":"2024-03-19T16:42:01.204225Z","shell.execute_reply.started":"2024-03-19T16:42:01.123959Z","shell.execute_reply":"2024-03-19T16:42:01.202625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ignore warnings\nwarnings.filterwarnings(\"ignore\")\ndrop_list =[]\n# View columns with > 13 unique categories for each variable\nfor col in (cat_cols):\n    if len(train[col].unique()) > 13:\n        drop_list.append(col)\n        print(f'{col}: {len(train[col].unique())}, {round(train[col].value_counts().sort_values(ascending=False)[0]/train[col].value_counts().sum(),4)}')\nwarnings.filterwarnings('default')","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:42:01.205779Z","iopub.execute_input":"2024-03-19T16:42:01.206197Z","iopub.status.idle":"2024-03-19T16:42:01.772913Z","shell.execute_reply.started":"2024-03-19T16:42:01.206164Z","shell.execute_reply":"2024-03-19T16:42:01.770867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Inspected output above for the drop_list. Dropped every column above.  ","metadata":{}},{"cell_type":"code","source":"drop_list = ['lastapprcommoditytypec_5251766M', 'lastrejectcommodtypec_5251769M','lastrejectcommoditycat_161M','lastrejectreasonclient_4145040M',\n            'previouscontdistrict_112M','lastapprcommoditycat_1041M', 'lastcancelreason_561M','lastrejectreason_759M', 'addres_zip_823M', 'addres_district_368M']\ntrain.drop(columns=drop_list, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:42:01.774478Z","iopub.execute_input":"2024-03-19T16:42:01.774919Z","iopub.status.idle":"2024-03-19T16:42:03.047053Z","shell.execute_reply.started":"2024-03-19T16:42:01.774884Z","shell.execute_reply":"2024-03-19T16:42:03.045638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\n# Verify all cat_cols should be categorical\ncat_cols = train.select_dtypes(include=['category']).columns.tolist()\ntrain[cat_cols].head()","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:42:03.048428Z","iopub.execute_input":"2024-03-19T16:42:03.049593Z","iopub.status.idle":"2024-03-19T16:42:03.105574Z","shell.execute_reply.started":"2024-03-19T16:42:03.049558Z","shell.execute_reply":"2024-03-19T16:42:03.104233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', 20)\n# Note we now have 254 columns as we created dummies for all categorical columns.\n# drop_first = False to keep columns consistent between training and test\ntrain = pd.get_dummies(train, dtype=int, columns=cat_cols, sparse=True, drop_first=False)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:42:03.106936Z","iopub.execute_input":"2024-03-19T16:42:03.107271Z","iopub.status.idle":"2024-03-19T16:42:31.737403Z","shell.execute_reply.started":"2024-03-19T16:42:03.107234Z","shell.execute_reply":"2024-03-19T16:42:31.736010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:42:31.738985Z","iopub.execute_input":"2024-03-19T16:42:31.740092Z","iopub.status.idle":"2024-03-19T16:42:31.748747Z","shell.execute_reply.started":"2024-03-19T16:42:31.740045Z","shell.execute_reply":"2024-03-19T16:42:31.747025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Final check no numeric columns are all the same\nnumeric_cols = train.select_dtypes(include='number').columns.tolist()\nfor col in numeric_cols:\n    if (train[col].max()-train[col].min()) == 0:\n        print(f'{col}: {train[col].max()-train[col].min()}')","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:42:31.750773Z","iopub.execute_input":"2024-03-19T16:42:31.751157Z","iopub.status.idle":"2024-03-19T16:42:33.647378Z","shell.execute_reply.started":"2024-03-19T16:42:31.751124Z","shell.execute_reply":"2024-03-19T16:42:33.646370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain.to_csv('train_dummy.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T16:42:33.648855Z","iopub.execute_input":"2024-03-19T16:42:33.649749Z","iopub.status.idle":"2024-03-19T16:52:36.868135Z","shell.execute_reply.started":"2024-03-19T16:42:33.649706Z","shell.execute_reply":"2024-03-19T16:52:36.866866Z"},"trusted":true},"execution_count":null,"outputs":[]}]}