{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Table of Contents\n* [Feature Definitions](#definitions)\n* [File Overview](#files)\n* [Import Data and Preview](#import)\n* [EDA](#EDA)\n    * [Features of depth 0 - internal](#features_0_int)\n        * [Numerical](#features_0_int_n)\n        * [Categorical](#features_0_int_c)\n    * [Features of depth 0 - external](#features_0_ext)\n        * [Numerical](#features_0_ext_n)\n        * [Categorical](#features_0_ext_c)","metadata":{}},{"cell_type":"code","source":"# packages\n\n# standard\nimport numpy as np\nimport pandas as pd\nimport time\n\n# plot\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2024-03-11T18:56:45.435551Z","iopub.execute_input":"2024-03-11T18:56:45.437140Z","iopub.status.idle":"2024-03-11T18:56:49.434801Z","shell.execute_reply.started":"2024-03-11T18:56:45.437084Z","shell.execute_reply":"2024-03-11T18:56:49.433402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# configs\npd.set_option('display.max_columns', None) # we want to display all columns in this notebook\npd.set_option('display.max_rows', 500)\n\n# aesthetics\ndefault_color_1 = 'darkblue'\ndefault_color_2 = 'darkgreen'\ndefault_color_3 = 'darkred'","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:49.438234Z","iopub.execute_input":"2024-03-11T18:56:49.439246Z","iopub.status.idle":"2024-03-11T18:56:49.445974Z","shell.execute_reply.started":"2024-03-11T18:56:49.439195Z","shell.execute_reply":"2024-03-11T18:56:49.444945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='definitions'></a>\n# Feature Definitions","metadata":{}},{"cell_type":"code","source":"# load feature definitions\nfeature_defs = pd.read_csv('../input/home-credit-credit-risk-model-stability/feature_definitions.csv')\nfeature_defs # show all rows","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-11T18:56:49.447471Z","iopub.execute_input":"2024-03-11T18:56:49.448407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='files'></a>\n# File Overview","metadata":{}},{"cell_type":"code","source":"ls -l '../input/home-credit-credit-risk-model-stability'","metadata":{"execution":{"iopub.execute_input":"2024-03-11T18:56:49.573869Z","iopub.status.idle":"2024-03-11T18:56:50.929113Z","shell.execute_reply.started":"2024-03-11T18:56:49.573829Z","shell.execute_reply":"2024-03-11T18:56:50.927422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CSV Files","metadata":{}},{"cell_type":"code","source":"ls -l '../input/home-credit-credit-risk-model-stability/csv_files/train'","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:50.931534Z","iopub.execute_input":"2024-03-11T18:56:50.932713Z","iopub.status.idle":"2024-03-11T18:56:52.097757Z","shell.execute_reply.started":"2024-03-11T18:56:50.932642Z","shell.execute_reply":"2024-03-11T18:56:52.095962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls -l '../input/home-credit-credit-risk-model-stability/csv_files/test'","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:52.100248Z","iopub.execute_input":"2024-03-11T18:56:52.100823Z","iopub.status.idle":"2024-03-11T18:56:53.222773Z","shell.execute_reply.started":"2024-03-11T18:56:52.100771Z","shell.execute_reply":"2024-03-11T18:56:53.220756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Parquet Files","metadata":{}},{"cell_type":"code","source":"ls -l '../input/home-credit-credit-risk-model-stability/parquet_files/train'","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:53.225440Z","iopub.execute_input":"2024-03-11T18:56:53.225946Z","iopub.status.idle":"2024-03-11T18:56:54.359743Z","shell.execute_reply.started":"2024-03-11T18:56:53.225898Z","shell.execute_reply":"2024-03-11T18:56:54.358176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls -l '../input/home-credit-credit-risk-model-stability/parquet_files/test'","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:54.361817Z","iopub.execute_input":"2024-03-11T18:56:54.362313Z","iopub.status.idle":"2024-03-11T18:56:55.511016Z","shell.execute_reply.started":"2024-03-11T18:56:54.362263Z","shell.execute_reply":"2024-03-11T18:56:55.509798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='import'></a>\n# Import Data and Preview","metadata":{}},{"cell_type":"code","source":"# import base data\ndf_train = pd.read_csv('../input/home-credit-credit-risk-model-stability/csv_files/train/train_base.csv')\ndf_test = pd.read_csv('../input/home-credit-credit-risk-model-stability/csv_files/test/test_base.csv')\ndf_sub = pd.read_csv('../input/home-credit-credit-risk-model-stability/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:55.512613Z","iopub.execute_input":"2024-03-11T18:56:55.513476Z","iopub.status.idle":"2024-03-11T18:56:57.137603Z","shell.execute_reply.started":"2024-03-11T18:56:55.513436Z","shell.execute_reply":"2024-03-11T18:56:57.135671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview - train\ndf_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:57.143328Z","iopub.execute_input":"2024-03-11T18:56:57.143828Z","iopub.status.idle":"2024-03-11T18:56:57.162005Z","shell.execute_reply.started":"2024-03-11T18:56:57.143781Z","shell.execute_reply":"2024-03-11T18:56:57.160622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# type conversions\ndf_train.MONTH = df_train.MONTH.astype('string')\ndf_test.MONTH = df_test.MONTH.astype('string')\n\ndf_train.date_decision = pd.to_datetime(df_train.date_decision)\ndf_test.date_decision = pd.to_datetime(df_test.date_decision)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:57.163726Z","iopub.execute_input":"2024-03-11T18:56:57.164194Z","iopub.status.idle":"2024-03-11T18:56:58.430730Z","shell.execute_reply.started":"2024-03-11T18:56:57.164144Z","shell.execute_reply":"2024-03-11T18:56:58.429484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure - train\ndf_train.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:58.432215Z","iopub.execute_input":"2024-03-11T18:56:58.432591Z","iopub.status.idle":"2024-03-11T18:56:58.628416Z","shell.execute_reply.started":"2024-03-11T18:56:58.432551Z","shell.execute_reply":"2024-03-11T18:56:58.627109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview - test\ndf_test.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:58.630432Z","iopub.execute_input":"2024-03-11T18:56:58.630881Z","iopub.status.idle":"2024-03-11T18:56:58.644778Z","shell.execute_reply.started":"2024-03-11T18:56:58.630842Z","shell.execute_reply":"2024-03-11T18:56:58.643463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure - test\ndf_test.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:58.646443Z","iopub.execute_input":"2024-03-11T18:56:58.646824Z","iopub.status.idle":"2024-03-11T18:56:58.666426Z","shell.execute_reply.started":"2024-03-11T18:56:58.646792Z","shell.execute_reply":"2024-03-11T18:56:58.665072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Depth 0, internal data:","metadata":{}},{"cell_type":"code","source":"# load and combined data\ndf1 = pd.read_parquet('../input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_0.parquet')\ndf2 = pd.read_parquet('../input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_1.parquet')\ndf_train_static = pd.concat([df1,df2])\ndel(df1,df2)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:56:58.667912Z","iopub.execute_input":"2024-03-11T18:56:58.669745Z","iopub.status.idle":"2024-03-11T18:57:08.706129Z","shell.execute_reply.started":"2024-03-11T18:56:58.669699Z","shell.execute_reply":"2024-03-11T18:57:08.704755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf_train_static.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:08.707832Z","iopub.execute_input":"2024-03-11T18:57:08.709199Z","iopub.status.idle":"2024-03-11T18:57:09.025845Z","shell.execute_reply.started":"2024-03-11T18:57:08.709157Z","shell.execute_reply":"2024-03-11T18:57:09.024198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show structure\ndf_train_static.info(verbose=True, show_counts=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:09.027563Z","iopub.execute_input":"2024-03-11T18:57:09.028516Z","iopub.status.idle":"2024-03-11T18:57:13.742837Z","shell.execute_reply.started":"2024-03-11T18:57:09.028477Z","shell.execute_reply":"2024-03-11T18:57:13.741440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Depth 0, external data:","metadata":{}},{"cell_type":"code","source":"# load/preview\ndf_train_static_x = pd.read_parquet('../input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_cb_0.parquet')\ndf_train_static_x.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:13.745538Z","iopub.execute_input":"2024-03-11T18:57:13.746038Z","iopub.status.idle":"2024-03-11T18:57:15.273539Z","shell.execute_reply.started":"2024-03-11T18:57:13.745993Z","shell.execute_reply":"2024-03-11T18:57:15.272320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show structure\ndf_train_static_x.info(verbose=True, show_counts=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:15.276500Z","iopub.execute_input":"2024-03-11T18:57:15.277684Z","iopub.status.idle":"2024-03-11T18:57:16.926192Z","shell.execute_reply.started":"2024-03-11T18:57:15.277615Z","shell.execute_reply":"2024-03-11T18:57:16.924896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='EDA'></a>\n# EDA","metadata":{}},{"cell_type":"code","source":"# basic stats\ndf_train.describe(include='all')","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:16.927730Z","iopub.execute_input":"2024-03-11T18:57:16.928097Z","iopub.status.idle":"2024-03-11T18:57:17.565337Z","shell.execute_reply.started":"2024-03-11T18:57:16.928064Z","shell.execute_reply":"2024-03-11T18:57:17.564005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot MONTH distribution\nplt.figure(figsize=(8,3))\ndf_train.MONTH.value_counts().sort_index().plot(kind='bar', color=default_color_1)\nplt.title('MONTH')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:17.566933Z","iopub.execute_input":"2024-03-11T18:57:17.567360Z","iopub.status.idle":"2024-03-11T18:57:18.295547Z","shell.execute_reply.started":"2024-03-11T18:57:17.567325Z","shell.execute_reply":"2024-03-11T18:57:18.294087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot WEEK_NUM distribution\nplt.figure(figsize=(14,3))\ndf_train.WEEK_NUM.value_counts().sort_index().plot(kind='bar', color=default_color_1)\nplt.title('WEEK_NUM')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:18.297265Z","iopub.execute_input":"2024-03-11T18:57:18.297652Z","iopub.status.idle":"2024-03-11T18:57:19.255209Z","shell.execute_reply.started":"2024-03-11T18:57:18.297619Z","shell.execute_reply":"2024-03-11T18:57:19.253857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target distribution\nplt.figure(figsize=(6,3))\ndf_train.target.plot(kind='hist', color=default_color_3)\nplt.title('Target')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:19.257443Z","iopub.execute_input":"2024-03-11T18:57:19.257849Z","iopub.status.idle":"2024-03-11T18:57:19.610726Z","shell.execute_reply.started":"2024-03-11T18:57:19.257815Z","shell.execute_reply":"2024-03-11T18:57:19.609358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target by week\ntarget_by_week = df_train.groupby(by=['WEEK_NUM'], observed=True)['target'].mean()\n\nplt.figure(figsize=(10,3))\nplt.bar(x=target_by_week.index, height=target_by_week,\n        color=default_color_3)\nplt.title('Target mean by week')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:19.612535Z","iopub.execute_input":"2024-03-11T18:57:19.612960Z","iopub.status.idle":"2024-03-11T18:57:20.030126Z","shell.execute_reply.started":"2024-03-11T18:57:19.612925Z","shell.execute_reply":"2024-03-11T18:57:20.028693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target by month\ntarget_by_month = df_train.groupby(by=['MONTH'], observed=True)['target'].mean()\n\nplt.figure(figsize=(14,3))\nplt.bar(x=target_by_month.index, height=target_by_month,\n        color=default_color_3)\nplt.xticks(rotation=90)\nplt.title('Target mean by month')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:20.032307Z","iopub.execute_input":"2024-03-11T18:57:20.032714Z","iopub.status.idle":"2024-03-11T18:57:20.762656Z","shell.execute_reply.started":"2024-03-11T18:57:20.032680Z","shell.execute_reply":"2024-03-11T18:57:20.760866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic stats\ndf_train_static.describe(include='all')","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:20.764508Z","iopub.execute_input":"2024-03-11T18:57:20.764907Z","iopub.status.idle":"2024-03-11T18:57:40.508844Z","shell.execute_reply.started":"2024-03-11T18:57:20.764873Z","shell.execute_reply":"2024-03-11T18:57:40.507681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='features_0_int'></a>\n## Features of depth 0, internal:","metadata":{}},{"cell_type":"code","source":"# extract numerical features\nfeatures_num = list(df_train_static.select_dtypes('number'))\nfeatures_num.remove('case_id')\nprint(features_num)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:40.510420Z","iopub.execute_input":"2024-03-11T18:57:40.511163Z","iopub.status.idle":"2024-03-11T18:57:45.797346Z","shell.execute_reply.started":"2024-03-11T18:57:40.511110Z","shell.execute_reply":"2024-03-11T18:57:45.795952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract categorical features\nfeatures_cat = df_train_static.columns.tolist()\nfeatures_cat = [el for el in features_cat if el not in features_num]\nfeatures_cat.remove('case_id')\nprint(features_cat)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:57:45.803641Z","iopub.execute_input":"2024-03-11T18:57:45.804066Z","iopub.status.idle":"2024-03-11T18:57:45.811227Z","shell.execute_reply.started":"2024-03-11T18:57:45.804031Z","shell.execute_reply":"2024-03-11T18:57:45.809791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='features_0_int_n'></a>\n### Numerical Features:","metadata":{}},{"cell_type":"code","source":"# plot distributions\nfor f in features_num:\n    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8,4))\n    # histogram\n    ax1.hist(df_train_static[f], bins=100,\n             color=default_color_1)\n    ax1.grid()\n    ax1.set_title(f)\n    # boxplot\n    ax2.boxplot(df_train_static[f].dropna(), vert=False)\n    ax2.grid()       \n    plt.show()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-11T18:57:45.812879Z","iopub.execute_input":"2024-03-11T18:57:45.814089Z","iopub.status.idle":"2024-03-11T18:59:24.132228Z","shell.execute_reply.started":"2024-03-11T18:57:45.814044Z","shell.execute_reply":"2024-03-11T18:59:24.130954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='features_0_int_c'></a>\n### Categorical Features:","metadata":{}},{"cell_type":"code","source":"for f in features_cat:\n    plt.figure(figsize=(8,2))\n    plotdata = df_train_static[f].value_counts()[0:25] # show only most frequent categories!\n    plotdata.plot(kind='bar', color=default_color_1)\n    plt.title(f)\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:59:24.133887Z","iopub.execute_input":"2024-03-11T18:59:24.134256Z","iopub.status.idle":"2024-03-11T18:59:42.336745Z","shell.execute_reply.started":"2024-03-11T18:59:24.134223Z","shell.execute_reply":"2024-03-11T18:59:42.334906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='features_0_ext'></a>\n## Features of depth 0, external:","metadata":{}},{"cell_type":"code","source":"# extract numerical features\nfeatures_num_x = list(df_train_static_x.select_dtypes('number'))\nfeatures_num_x.remove('case_id')\nprint(features_num_x)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:59:42.338670Z","iopub.execute_input":"2024-03-11T18:59:42.342239Z","iopub.status.idle":"2024-03-11T18:59:42.658748Z","shell.execute_reply.started":"2024-03-11T18:59:42.342189Z","shell.execute_reply":"2024-03-11T18:59:42.657361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract categorical features\nfeatures_cat_x = df_train_static_x.columns.tolist()\nfeatures_cat_x = [el for el in features_cat_x if el not in features_num_x]\nfeatures_cat_x.remove('case_id')\nprint(features_cat_x)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T18:59:42.660628Z","iopub.execute_input":"2024-03-11T18:59:42.662737Z","iopub.status.idle":"2024-03-11T18:59:42.673711Z","shell.execute_reply.started":"2024-03-11T18:59:42.662682Z","shell.execute_reply":"2024-03-11T18:59:42.671802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='features_0_ext_n'></a>\n### Numerical Features:","metadata":{}},{"cell_type":"code","source":"# plot distributions\nfor f in features_num_x:\n    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8,4))\n    # histogram\n    ax1.hist(df_train_static_x[f], bins=100,\n             color=default_color_1)\n    ax1.grid()\n    ax1.set_title(f)\n    # boxplot\n    ax2.boxplot(df_train_static_x[f].dropna(), vert=False)\n    ax2.grid()       \n    plt.show()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-11T18:59:42.675395Z","iopub.execute_input":"2024-03-11T18:59:42.676420Z","iopub.status.idle":"2024-03-11T19:00:04.433810Z","shell.execute_reply.started":"2024-03-11T18:59:42.676352Z","shell.execute_reply":"2024-03-11T19:00:04.432438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='features_0_ext_c'></a>\n### Categorical Features","metadata":{}},{"cell_type":"code","source":"for f in features_cat_x:\n    plt.figure(figsize=(8,2))\n    plotdata = df_train_static_x[f].value_counts()[0:25] # show only most frequent categories!\n    plotdata.plot(kind='bar', color=default_color_1)\n    plt.title(f)\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T19:00:22.658978Z","iopub.execute_input":"2024-03-11T19:00:22.659404Z","iopub.status.idle":"2024-03-11T19:00:29.740917Z","shell.execute_reply.started":"2024-03-11T19:00:22.659354Z","shell.execute_reply":"2024-03-11T19:00:29.739345Z"},"trusted":true},"execution_count":null,"outputs":[]}]}