{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"},{"sourceId":7704822,"sourceType":"datasetVersion","datasetId":4498128},{"sourceId":7705311,"sourceType":"datasetVersion","datasetId":4498480},{"sourceId":7703109,"sourceType":"datasetVersion","datasetId":4431895},{"sourceId":7605313,"sourceType":"datasetVersion","datasetId":4408860}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"load data\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-26T12:06:39.804409Z","iopub.execute_input":"2024-02-26T12:06:39.805210Z","iopub.status.idle":"2024-02-26T12:06:39.813311Z","shell.execute_reply.started":"2024-02-26T12:06:39.805173Z","shell.execute_reply":"2024-02-26T12:06:39.812493Z"}}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nimport os\nimport glob\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm_notebook as tqdm\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nfrom datetime import datetime\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-26T22:15:32.389597Z","iopub.execute_input":"2024-02-26T22:15:32.390102Z","iopub.status.idle":"2024-02-26T22:15:32.396644Z","shell.execute_reply.started":"2024-02-26T22:15:32.390053Z","shell.execute_reply":"2024-02-26T22:15:32.395706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\nfrom pathlib import Path\n# define a configuration class to hold directory paths\nclass CFG:\n    # root directory where the dataset is stored\n    root_dir = '/kaggle/input/home-credit-credit-risk-model-stability'\n    # directory where training data files are located\n    train_dir = os.path.join(root_dir, 'parquet_files', 'train')\n    # directory where test data files are located\n    test_dir = os.path.join(root_dir, 'parquet_files', 'test')\n# load feature definitions and display\nfeature_definitions_df = pd.read_csv(os.path.join(CFG.root_dir, \"feature_definitions.csv\"))\n# display the relative infomation of the dataset\ndisplay(feature_definitions_df.info())\ndisplay(feature_definitions_df.head())\n\nclass CFG:\n    # Root directory where the dataset is stored\n    root_dir = '/kaggle/input/home-credit-credit-risk-model-stability'\n    # Directory where training data files are located\n    train_dir = Path(root_dir) / 'parquet_files' / 'train'\n    # Directory where test data files are located\n    test_dir = Path(root_dir) / 'parquet_files' / 'test'\n\ndef load_feature_definitions():\n    try:\n        # Load feature definitions CSV file\n        feature_definitions_df = pd.read_csv(Path(CFG.root_dir) / \"feature_definitions.csv\")\n        return feature_definitions_df\n    except FileNotFoundError:\n        print(\"Error: Feature definitions file not found.\")\n        return None\n    except Exception as e:\n        print(f\"Error occurred while loading feature definitions: {e}\")\n        return None\n\ndef display_info(df):\n    # Display information about the DataFrame\n    if df is not None:\n        display(df.info())\n        display(df.head())\n    else:\n        print(\"DataFrame is None. Cannot display info.\")\n\nif __name__ == \"__main__\":\n    # Load feature definitions and display\n    feature_definitions_df = load_feature_definitions()\n    display_info(feature_definitions_df)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-26T22:15:32.416447Z","iopub.execute_input":"2024-02-26T22:15:32.416806Z","iopub.status.idle":"2024-02-26T22:15:32.469427Z","shell.execute_reply.started":"2024-02-26T22:15:32.416780Z","shell.execute_reply":"2024-02-26T22:15:32.468444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data Collection and Preprocessing","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport os\n\nclass Pipeline:\n    @staticmethod\n    def set_table_dtypes(df):\n        try:\n            for col in df.columns:\n                if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                    df[col] = df[col].astype('Int64')\n                elif col in [\"date_decision\"]:\n                    df[col] = pd.to_datetime(df[col])\n                elif col[-1] in (\"P\", \"A\"):\n                    df[col] = df[col].astype('float64')\n                elif col[-1] in (\"M\",):\n                    df[col] = df[col].astype('string')\n                elif col[-1] in (\"D\",):\n                    df[col] = pd.to_datetime(df[col])\n        except Exception as e:\n            print(f\"Error in set_table_dtypes: {e}\")\n        return df\n    \n    @staticmethod\n    def handle_dates(df):\n        try:\n            date_cols = [col for col in df.columns if col[-1] in (\"D\",)]\n            for col in date_cols:\n                df[col] = (df[col] - df[\"date_decision\"]).dt.days\n            df = df.drop(columns=[\"date_decision\", \"MONTH\"])\n        except Exception as e:\n            print(f\"Error in handle_dates: {e}\")\n        return df\n    \n    @staticmethod\n    def filter_cols(df):\n        try:\n            cols_to_drop = [col for col in df.columns if col not in [\"target\", \"case_id\", \"WEEK_NUM\"] and df[col].isnull().mean() > 0.95]\n            df = df.drop(columns=cols_to_drop)\n\n            cols_to_drop = [col for col in df.columns if col not in [\"target\", \"case_id\", \"WEEK_NUM\"] and df[col].dtype == object and (df[col].nunique() == 1 or df[col].nunique() > 200)]\n            df = df.drop(columns=cols_to_drop)\n        except Exception as e:\n            print(f\"Error in filter_cols: {e}\")\n        return df\n\nclass Aggregator:\n    num_aggregators = ['max', 'min', 'mean']\n    str_aggregators = ['max', 'min']\n    \n    @staticmethod\n    def num_expr(df):\n        try:\n            num_cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n            expr_all = {col: Aggregator.num_aggregators for col in num_cols}\n            return expr_all\n        except Exception as e:\n            print(f\"Error in num_expr: {e}\")\n            return {}\n\n    @staticmethod\n    def date_expr(df):\n        try:\n            date_cols = [col for col in df.columns if col[-1] in (\"D\",)]\n            expr_all = {col: Aggregator.num_aggregators for col in date_cols}\n            return expr_all\n        except Exception as e:\n            print(f\"Error in date_expr: {e}\")\n            return {}\n\n    @staticmethod\n    def str_expr(df):\n        try:\n            str_cols = [col for col in df.columns if col[-1] in (\"M\",)]\n            expr_all = {col: Aggregator.str_aggregators for col in str_cols}\n            return expr_all\n        except Exception as e:\n            print(f\"Error in str_expr: {e}\")\n            return {}\n\n    @staticmethod\n    def get_exprs(df):\n        try:\n            exprs = {**Aggregator.num_expr(df),\n                     **Aggregator.date_expr(df),\n                     **Aggregator.str_expr(df)}\n            return exprs\n        except Exception as e:\n            print(f\"Error in get_exprs: {e}\")\n            return {}\n\n    @staticmethod\n    def aggregate(df, groupby_cols):\n        try:\n            exprs = Aggregator.get_exprs(df)\n            aggregated_df = df.groupby(groupby_cols).agg(exprs)\n            return aggregated_df\n        except Exception as e:\n            print(f\"Error in aggregate: {e}\")\n            return None\n\nclass CFG:\n    root_dir = '/kaggle/input/home-credit-credit-risk-model-stability'\n    train_dir = os.path.join(root_dir, 'parquet_files', 'train')\n    test_dir = os.path.join(root_dir, 'parquet_files', 'test')\n\nfeature_definitions_df = pd.read_csv(os.path.join(CFG.root_dir, \"feature_definitions.csv\"))\n","metadata":{"execution":{"iopub.status.busy":"2024-02-26T22:15:32.471345Z","iopub.execute_input":"2024-02-26T22:15:32.471652Z","iopub.status.idle":"2024-02-26T22:15:32.494973Z","shell.execute_reply.started":"2024-02-26T22:15:32.471627Z","shell.execute_reply":"2024-02-26T22:15:32.493944Z"},"trusted":true},"execution_count":null,"outputs":[]}]}