{"metadata":{"colab":{"provenance":[],"collapsed_sections":["Gaygxot8p9K0"]},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7880793,"sourceType":"datasetVersion","datasetId":4625392}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# PASO 01 - IMPORTAR LIBRERIAS DE TRABAJO","metadata":{"id":"UqPEdBDLjtcd"}},{"cell_type":"code","source":"import seaborn as sns              # Importar la librería Seaborn\nimport missingno as msno           # Importar la librería Missingno\nimport polars as pl                # Importar la librería Polars\nimport numpy as np                 # Importar la librería Numpy\nimport pandas as pd                # Importar la librería Pandas\nimport matplotlib.pyplot as plt    # Importar la librería mMtplotlib\nimport xgboost as xgb              # Importar la librería XGBoost\nimport lightgbm as lgb             # Importar la librería LightGBM\nfrom sklearn.model_selection import TimeSeriesSplit, GroupKFold, StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.model_selection import KFold\n#import CatBoostClassifier as catboost\n\n# Importa la función train_test_split del módulo model_selection de la librería scikit-learn.\n# Esta función se utiliza para dividir un conjunto de datos en conjuntos de entrenamiento y prueba.\nfrom sklearn.model_selection import train_test_split   # cross_val_score Delete\n\n# Importa la función roc_auc_score del módulo metrics de la librería scikit-learn.\n# Esta función se utiliza para calcular el área bajo la curva ROC (AUC) para problemas de clasificación binaria.\nfrom sklearn.metrics import roc_auc_score\n# from sklearn.model_selection import KFold\n","metadata":{"id":"URUIm3T2GfoI","execution":{"iopub.status.busy":"2024-03-19T03:02:16.383038Z","iopub.execute_input":"2024-03-19T03:02:16.383412Z","iopub.status.idle":"2024-03-19T03:02:19.501535Z","shell.execute_reply.started":"2024-03-19T03:02:16.383378Z","shell.execute_reply":"2024-03-19T03:02:19.500419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PASO 02 - CONECTAR AL DATA SET\n","metadata":{"id":"g_TF8yFSmR0u"}},{"cell_type":"code","source":"#from google.colab import drive                # Importando libreria del Drive\n#drive.mount('/content/drive')                 # Montar data del Drive para montarlo a Google Colab\n#dataPath = \"/content/drive/MyDrive/Kaggle/\"   # Ruta donde se encuentra la data\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"id":"rF39ZVV8mV8G","outputId":"19492eb4-9af4-4d09-8d5d-62468eaeece8","execution":{"iopub.status.busy":"2024-03-19T03:02:01.622527Z","iopub.execute_input":"2024-03-19T03:02:01.623935Z","iopub.status.idle":"2024-03-19T03:02:01.670835Z","shell.execute_reply.started":"2024-03-19T03:02:01.623846Z","shell.execute_reply":"2024-03-19T03:02:01.669057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PASO 03 - CREAR FUNCIONES","metadata":{"id":"vcRnv6SYkAV_"}},{"cell_type":"code","source":"# Esta función se utiliza para ajustar los tipos de datos de las columnas de un DataFrame según su criterio específico,\n# convierte las columnas que terminan su nombre en \"P\" o \"A\" al tipo de dato \"Float64\"\ndef set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n\n    for col in df.columns:\n\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n\n    return df\n\n# Función para convertir las columnas de tipo \"object\" o \"string\" a tipo \"category\", si el tipo es diferente a lo mencionado\n# convierte a la categoría \"Unknown\" y devuelve el DataFrame modificado.\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n\n    for col in df.columns:\n\n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n\n    return df\n\n# Función que proporciona una manera de limpiar los valores nulos en un DataFrame de Polars, utilizando la moda de cada columna como valor de reemplazo\ndef replace_null_with_moda(df: pl.DataFrame):\n    for col in df.columns:\n        df = df.with_columns(\n          pl.col(col).fill_null(pl.lit(df[col].mode()))\n      )\n        return df","metadata":{"id":"hxSZTy6JG-OG","execution":{"iopub.status.busy":"2024-03-19T03:03:03.229793Z","iopub.execute_input":"2024-03-19T03:03:03.230971Z","iopub.status.idle":"2024-03-19T03:03:03.242455Z","shell.execute_reply.started":"2024-03-19T03:03:03.230921Z","shell.execute_reply":"2024-03-19T03:03:03.240846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PASO 04 - IMPORTACION DE DATOS / TRAIN","metadata":{"id":"8V2xglnIkPPl"}},{"cell_type":"code","source":"# Se encarga de leer el archivo CSV llamado \"train_base.csv\" y lo almacena el DataFrames en la variable \"train_basetable\" mediante la libreria Polars\ntrain_basetable = pl.read_csv(dataPath + \"csv_files/train/train_base.csv\")\n\n# Se encarga de leer el archivo CSV llamado \"train_static_0_0.csv\" y \"train_static_0_1.csv\" y se concatena los DataFrames generados mediante la libreria Polars,\n# el nuevo DataFrames resultante se almacena en la variable \"train_static\"\ntrain_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_1.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\n# Se encarga de leer el archivo CSV llamado \"train_static_cb_0.csv\" y lo almacena el DataFrames en la variable \"train_static_cb\" mediante la libreria Polars\ntrain_static_cb = pl.read_csv(dataPath + \"csv_files/train/train_static_cb_0.csv\").pipe(set_table_dtypes)\n\n# Se encarga de leer el archivo CSV llamado \"train_person_1.csv\" y lo almacena el DataFrames en la variable \"train_person_1\" mediante la libreria Polars\ntrain_person_1 = pl.read_csv(dataPath + \"csv_files/train/train_person_1.csv\").pipe(set_table_dtypes)\n\n# Se encarga de leer el archivo CSV llamado \"train_credit_bureau_b_2.csv\" y lo almacena el DataFrames en la variable \"train_credit_bureau_b_2\" mediante la libreria Polars\ntrain_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_b_2.csv\").pipe(set_table_dtypes)","metadata":{"id":"Cmwkv8v5HFJO","execution":{"iopub.status.busy":"2024-03-19T03:03:07.776671Z","iopub.execute_input":"2024-03-19T03:03:07.777618Z","iopub.status.idle":"2024-03-19T03:03:25.865575Z","shell.execute_reply.started":"2024-03-19T03:03:07.777580Z","shell.execute_reply":"2024-03-19T03:03:25.864461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PASO 05 - IMPORTACION DE DATOS / TEST","metadata":{"id":"Hdd4X7N9k4Vs"}},{"cell_type":"code","source":"# Se encarga de leer el archivo CSV llamado \"test_base.csv\" y lo almacena el DataFrames en la variable \"test_basetable\" mediante la libreria Polars\ntest_basetable = pl.read_csv(dataPath + \"csv_files/test/test_base.csv\")\n\n# Se encarga de leer el archivo CSV llamado \"test_static_0_0.csv\", \"test_static_0_1.csv\" y \"test_static_0_2.csv\", se concatena los DataFrames generados mediante la libreria Polars,\n# el nuevo DataFrames resultante se almacena en la variable \"test_static\"\ntest_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\n# Se encarga de leer el archivo CSV llamado \"test_static_cb_0.csv\" y lo almacena el DataFrames en la variable \"test_static_cb\" mediante la libreria Polars\ntest_static_cb = pl.read_csv(dataPath + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\n\n# Se encarga de leer el archivo CSV llamado \"test_person_1.csv\" y lo almacena el DataFrames en la variable \"test_person_1\" mediante la libreria Polars\ntest_person_1 = pl.read_csv(dataPath + \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes)\n\n# Se encarga de leer el archivo CSV llamado \"test_credit_bureau_b_2.csv\" y lo almacena el DataFrames en la variable \"test_credit_bureau_b_2\" mediante la libreria Polars\ntest_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes)","metadata":{"id":"skrww3HYlg6N","execution":{"iopub.status.busy":"2024-03-19T03:03:33.860630Z","iopub.execute_input":"2024-03-19T03:03:33.861094Z","iopub.status.idle":"2024-03-19T03:03:33.930129Z","shell.execute_reply.started":"2024-03-19T03:03:33.861060Z","shell.execute_reply":"2024-03-19T03:03:33.928922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PASO 06 - ANALISIS EXPLORATORIO / TRAIN\n","metadata":{"id":"K_7JIee6ltUt"}},{"cell_type":"code","source":"# Mostrando el tipo de la variable \"train_basetable\"\nprint(type(train_basetable))","metadata":{"id":"D1myHzKlrDZ-","outputId":"9bee8834-b022-4366-822f-4cbe2cc03b2f","execution":{"iopub.status.busy":"2024-03-19T03:03:39.908355Z","iopub.execute_input":"2024-03-19T03:03:39.908804Z","iopub.status.idle":"2024-03-19T03:03:39.915296Z","shell.execute_reply.started":"2024-03-19T03:03:39.908774Z","shell.execute_reply":"2024-03-19T03:03:39.914064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra las dimensiones del DataFrames (filas, columnas)\ntrain_basetable.shape","metadata":{"id":"o9Fj2KFjQekU","outputId":"4caaaa67-534f-4726-cb8e-57be3265f902","execution":{"iopub.status.busy":"2024-03-19T03:03:43.470395Z","iopub.execute_input":"2024-03-19T03:03:43.470808Z","iopub.status.idle":"2024-03-19T03:03:43.480428Z","shell.execute_reply.started":"2024-03-19T03:03:43.470776Z","shell.execute_reply":"2024-03-19T03:03:43.478929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestras las columnas del DataFrames\ntrain_basetable.columns","metadata":{"id":"ttMhxepMP7gc","outputId":"44672f99-4ded-40d1-c172-5b749180a1be","execution":{"iopub.status.busy":"2024-03-19T03:03:46.230400Z","iopub.execute_input":"2024-03-19T03:03:46.230843Z","iopub.status.idle":"2024-03-19T03:03:46.238931Z","shell.execute_reply.started":"2024-03-19T03:03:46.230806Z","shell.execute_reply":"2024-03-19T03:03:46.237166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra las 10 primeras filas del DataFrames\ntrain_basetable.head(10)","metadata":{"id":"qbq4_7AUHQvv","outputId":"5ca62ded-b63f-4454-d792-4915424f86e2","execution":{"iopub.status.busy":"2024-03-19T03:03:51.229378Z","iopub.execute_input":"2024-03-19T03:03:51.230180Z","iopub.status.idle":"2024-03-19T03:03:51.249106Z","shell.execute_reply.started":"2024-03-19T03:03:51.230123Z","shell.execute_reply":"2024-03-19T03:03:51.247289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra datos descriptivas de las columnas numéricas del DataFrame\ntrain_basetable.describe()","metadata":{"id":"e2U6UyoWe8Za","outputId":"47aa0917-f6da-4f61-ffc3-c4241b50db28","execution":{"iopub.status.busy":"2024-03-19T03:03:56.603841Z","iopub.execute_input":"2024-03-19T03:03:56.604250Z","iopub.status.idle":"2024-03-19T03:03:56.724606Z","shell.execute_reply.started":"2024-03-19T03:03:56.604219Z","shell.execute_reply":"2024-03-19T03:03:56.723647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convierte el DataFrame train_basetable de Polars a un DataFrames de Pandas\ndata_df_pd = train_basetable.to_pandas()\n\n# Obteniendo la cantidad de columnas y almacenando en la variable \"serie\"\nserie = train_basetable.columns\n\n# Generando el histograma y mostrando el grafico\ndata_df_pd[serie].hist(bins=20, figsize=(20,10))\nplt.show()","metadata":{"id":"d-anKCaOomEP","outputId":"f97866a3-c473-49d5-f90a-a8343f852b35","execution":{"iopub.status.busy":"2024-03-19T03:04:02.788142Z","iopub.execute_input":"2024-03-19T03:04:02.788565Z","iopub.status.idle":"2024-03-19T03:04:04.556154Z","shell.execute_reply.started":"2024-03-19T03:04:02.788536Z","shell.execute_reply":"2024-03-19T03:04:04.554820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra información concisa del DataFrames de Panda\ndata_df_pd.info()","metadata":{"id":"PCT23jZHpmCO","outputId":"7880837f-71aa-46f8-90e1-687322976b71","execution":{"iopub.status.busy":"2024-03-19T03:04:12.328068Z","iopub.execute_input":"2024-03-19T03:04:12.329368Z","iopub.status.idle":"2024-03-19T03:04:12.522516Z","shell.execute_reply.started":"2024-03-19T03:04:12.329325Z","shell.execute_reply":"2024-03-19T03:04:12.520304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra la cantidad de nulos que tiene mi variables del DataFrames de Panda\ndata_df_pd.isna().sum()","metadata":{"id":"OAV0u6RDqBkR","outputId":"e2398d46-9505-495d-db0f-5fe8a13d86a4","execution":{"iopub.status.busy":"2024-03-19T03:04:18.734441Z","iopub.execute_input":"2024-03-19T03:04:18.735381Z","iopub.status.idle":"2024-03-19T03:04:18.918241Z","shell.execute_reply.started":"2024-03-19T03:04:18.735335Z","shell.execute_reply":"2024-03-19T03:04:18.916983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mostrando el tipo de la variable \"train_static\"\nprint(type(train_static))","metadata":{"id":"cuEaC-E4tivX","outputId":"6501afa9-0926-4037-80eb-d594dc6ceefe","execution":{"iopub.status.busy":"2024-03-19T03:04:22.876387Z","iopub.execute_input":"2024-03-19T03:04:22.876792Z","iopub.status.idle":"2024-03-19T03:04:22.883316Z","shell.execute_reply.started":"2024-03-19T03:04:22.876759Z","shell.execute_reply":"2024-03-19T03:04:22.881782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra las dimensiones del DataFrames (filas, columnas)\ntrain_static.shape","metadata":{"id":"22NyjPt_t4tF","outputId":"9d091063-87bd-4273-f48e-127d3516c11b","execution":{"iopub.status.busy":"2024-03-19T03:04:26.114489Z","iopub.execute_input":"2024-03-19T03:04:26.114912Z","iopub.status.idle":"2024-03-19T03:04:26.122057Z","shell.execute_reply.started":"2024-03-19T03:04:26.114856Z","shell.execute_reply":"2024-03-19T03:04:26.120909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestras las columnas del DataFrames\ntrain_static.columns","metadata":{"id":"Yu2qK42cttrs","outputId":"3db1f4bc-73de-417e-ce66-e17f7d307512","execution":{"iopub.status.busy":"2024-03-19T03:04:29.865943Z","iopub.execute_input":"2024-03-19T03:04:29.866394Z","iopub.status.idle":"2024-03-19T03:04:29.878045Z","shell.execute_reply.started":"2024-03-19T03:04:29.866359Z","shell.execute_reply":"2024-03-19T03:04:29.876370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra las 10 primeras filas del DataFrames\ntrain_static.head(10)","metadata":{"id":"EXCWRyT0t1Mm","outputId":"a3f1b1d8-7505-41f2-9c52-c258c3d33e9a","execution":{"iopub.status.busy":"2024-03-19T03:04:36.605021Z","iopub.execute_input":"2024-03-19T03:04:36.605496Z","iopub.status.idle":"2024-03-19T03:04:36.642003Z","shell.execute_reply.started":"2024-03-19T03:04:36.605463Z","shell.execute_reply":"2024-03-19T03:04:36.639965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra datos descriptivas de las columnas numéricas del DataFrame\ntrain_static.describe()","metadata":{"id":"wN9bfLwmt9Qm","outputId":"ad5ddc9b-a84f-44b7-c21c-15452b207711","execution":{"iopub.status.busy":"2024-03-19T03:04:42.051547Z","iopub.execute_input":"2024-03-19T03:04:42.052001Z","iopub.status.idle":"2024-03-19T03:04:45.188597Z","shell.execute_reply.started":"2024-03-19T03:04:42.051968Z","shell.execute_reply":"2024-03-19T03:04:45.186738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convierte el DataFrame train_basetable de Polars a un DataFrames de Pandas\ndata_df_pd_02 = train_static.to_pandas()\n\n# Obteniendo la cantidad de columnas y almacenando en la variable \"serie\"\nserie = train_static.columns\n\n# Generando el histograma y mostrando el grafico\ndata_df_pd_02[serie].hist(bins=20, figsize=(20,10))\nplt.show()","metadata":{"id":"Cvk0gof_uKrQ","outputId":"c84c726c-6983-4813-9d8b-953d49fcdfc9","execution":{"iopub.status.busy":"2024-03-19T03:04:53.094448Z","iopub.execute_input":"2024-03-19T03:04:53.094815Z","iopub.status.idle":"2024-03-19T03:05:30.795745Z","shell.execute_reply.started":"2024-03-19T03:04:53.094786Z","shell.execute_reply":"2024-03-19T03:05:30.794478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra información concisa del DataFrames de Panda\ndata_df_pd_02.info()","metadata":{"id":"sar8Us0auY6c","outputId":"eb7b0758-47cb-4bff-cafb-dc591f55fb93","execution":{"iopub.status.busy":"2024-03-19T03:05:37.566517Z","iopub.execute_input":"2024-03-19T03:05:37.567045Z","iopub.status.idle":"2024-03-19T03:05:37.595331Z","shell.execute_reply.started":"2024-03-19T03:05:37.567009Z","shell.execute_reply":"2024-03-19T03:05:37.593815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra la cantidad de nulos que tiene mi variables del DataFrames de Panda\ndata_df_pd_02.isna().sum()","metadata":{"id":"ns-6RJQZulkk","outputId":"09def92d-d3b2-4af1-ba6d-71fe7c5a7cca","execution":{"iopub.status.busy":"2024-03-19T03:06:28.839072Z","iopub.execute_input":"2024-03-19T03:06:28.839776Z","iopub.status.idle":"2024-03-19T03:06:34.369750Z","shell.execute_reply.started":"2024-03-19T03:06:28.839740Z","shell.execute_reply":"2024-03-19T03:06:34.368401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mostrando el tipo de la variable \"train_static_cb\"\nprint(type(train_static_cb))","metadata":{"outputId":"c4ed4d49-102a-4b5c-8688-b32720c5ee7e","id":"A1_zRaiKT9PE","execution":{"iopub.status.busy":"2024-03-19T03:06:53.262934Z","iopub.execute_input":"2024-03-19T03:06:53.263357Z","iopub.status.idle":"2024-03-19T03:06:53.271504Z","shell.execute_reply.started":"2024-03-19T03:06:53.263326Z","shell.execute_reply":"2024-03-19T03:06:53.269738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra las dimensiones del DataFrames (filas, columnas)\ntrain_static_cb.shape","metadata":{"id":"D90xmjSlUD5f","outputId":"d872e4fd-f4fb-435b-8f4c-79abfd1ecfae","execution":{"iopub.status.busy":"2024-03-19T03:06:56.927100Z","iopub.execute_input":"2024-03-19T03:06:56.927484Z","iopub.status.idle":"2024-03-19T03:06:56.936266Z","shell.execute_reply.started":"2024-03-19T03:06:56.927455Z","shell.execute_reply":"2024-03-19T03:06:56.934734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestras las columnas del DataFrames\ntrain_static_cb.columns","metadata":{"id":"CmJNEM7pUMUz","outputId":"de9067ab-afc2-4634-d76d-c6484cc3e844","execution":{"iopub.status.busy":"2024-03-19T03:06:59.359577Z","iopub.execute_input":"2024-03-19T03:06:59.360000Z","iopub.status.idle":"2024-03-19T03:06:59.369947Z","shell.execute_reply.started":"2024-03-19T03:06:59.359963Z","shell.execute_reply":"2024-03-19T03:06:59.368166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra las 10 primeras filas del DataFrames\ntrain_static_cb.head(10)","metadata":{"id":"wCHZgS21UX3E","outputId":"d8a7e00f-5237-41e7-9398-d0e30dd7883b","execution":{"iopub.status.busy":"2024-03-19T03:07:02.676748Z","iopub.execute_input":"2024-03-19T03:07:02.677197Z","iopub.status.idle":"2024-03-19T03:07:02.703115Z","shell.execute_reply.started":"2024-03-19T03:07:02.677165Z","shell.execute_reply":"2024-03-19T03:07:02.701515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra datos descriptivas de las columnas numéricas del DataFrame\ntrain_static_cb.describe()","metadata":{"outputId":"7faf9904-bbb2-49ff-c152-bb5dd352d93a","id":"lquYbzhdUlNC","execution":{"iopub.status.busy":"2024-03-19T03:07:06.074536Z","iopub.execute_input":"2024-03-19T03:07:06.074940Z","iopub.status.idle":"2024-03-19T03:07:06.771297Z","shell.execute_reply.started":"2024-03-19T03:07:06.074909Z","shell.execute_reply":"2024-03-19T03:07:06.769896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convierte el DataFrame train_basetable de Polars a un DataFrames de Pandas\ndata_df_pd_03 = train_static_cb.to_pandas()\n\n# Obteniendo la cantidad de columnas y almacenando en la variable \"serie\"\nserie = train_static_cb.columns\n\n# Generando el histograma y mostrando el grafico\ndata_df_pd_03[serie].hist(bins=20, figsize=(20,10))\nplt.show()","metadata":{"id":"L2wavVK9Uypk","outputId":"2f206adc-b871-484b-8ad7-9366c089e224","execution":{"iopub.status.busy":"2024-03-19T03:07:09.534617Z","iopub.execute_input":"2024-03-19T03:07:09.535329Z","iopub.status.idle":"2024-03-19T03:07:19.757116Z","shell.execute_reply.started":"2024-03-19T03:07:09.535295Z","shell.execute_reply":"2024-03-19T03:07:19.755943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra información concisa del DataFrames de Panda\ndata_df_pd_03.info()","metadata":{"outputId":"8547c6be-1394-4a02-de74-427f901743f3","id":"-dLltoV9VSUl","execution":{"iopub.status.busy":"2024-03-19T03:07:29.389252Z","iopub.execute_input":"2024-03-19T03:07:29.389653Z","iopub.status.idle":"2024-03-19T03:07:31.194733Z","shell.execute_reply.started":"2024-03-19T03:07:29.389623Z","shell.execute_reply":"2024-03-19T03:07:31.193161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra la cantidad de nulos que tiene mi variables del DataFrames de Panda\ndata_df_pd_03.isna().sum()","metadata":{"id":"RN3NiKKoVbcU","outputId":"7a6945eb-7383-4b41-af47-d0a43a0295b2","execution":{"iopub.status.busy":"2024-03-19T03:07:34.451189Z","iopub.execute_input":"2024-03-19T03:07:34.451599Z","iopub.status.idle":"2024-03-19T03:07:36.169996Z","shell.execute_reply.started":"2024-03-19T03:07:34.451570Z","shell.execute_reply":"2024-03-19T03:07:36.168191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra la cantidad de valores nulos por columna de manera grafica\n# Se puede identificar un patron de las variables nula\nmsno.matrix(data_df_pd_03)\nplt.show()","metadata":{"id":"U7doWqKQY_UR","outputId":"a7731162-081f-4bca-ab4b-ae8c810bbb35","execution":{"iopub.status.busy":"2024-03-19T03:07:39.404117Z","iopub.execute_input":"2024-03-19T03:07:39.404542Z","iopub.status.idle":"2024-03-19T03:08:00.410479Z","shell.execute_reply.started":"2024-03-19T03:07:39.404512Z","shell.execute_reply":"2024-03-19T03:08:00.409136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.heatmap(data_df_pd_03)\nplt.show()","metadata":{"id":"7P6zMXkBb9JU","outputId":"6f5b27a8-78ed-4b1f-f276-3bdd005e61e0","execution":{"iopub.status.busy":"2024-03-19T03:08:11.402066Z","iopub.execute_input":"2024-03-19T03:08:11.402721Z","iopub.status.idle":"2024-03-19T03:08:28.984207Z","shell.execute_reply.started":"2024-03-19T03:08:11.402689Z","shell.execute_reply":"2024-03-19T03:08:28.981759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.dendrogram(data_df_pd_03)\nplt.show()","metadata":{"id":"-R3lwFc1cVQ-","outputId":"625b539a-17e7-48b4-b49e-e51bc5ce66f4","execution":{"iopub.status.busy":"2024-03-19T03:08:40.906664Z","iopub.execute_input":"2024-03-19T03:08:40.908115Z","iopub.status.idle":"2024-03-19T03:08:46.301279Z","shell.execute_reply.started":"2024-03-19T03:08:40.908071Z","shell.execute_reply":"2024-03-19T03:08:46.299676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mostrando el tipo de la variable \"train_person_1\"\nprint(type(train_person_1))","metadata":{"id":"pyj86R3Ycfec","outputId":"8fa89abb-fb4d-4efb-fb79-3fc0b80ddee0","execution":{"iopub.status.busy":"2024-03-19T03:08:51.766568Z","iopub.execute_input":"2024-03-19T03:08:51.766992Z","iopub.status.idle":"2024-03-19T03:08:51.774581Z","shell.execute_reply.started":"2024-03-19T03:08:51.766962Z","shell.execute_reply":"2024-03-19T03:08:51.773059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra las dimensiones del DataFrames (filas, columnas)\ntrain_person_1.shape","metadata":{"id":"mFAQDdtIcjCq","outputId":"78b5a7ab-8c86-4cfa-f993-edba7202a1cf","execution":{"iopub.status.busy":"2024-03-19T03:08:54.613643Z","iopub.execute_input":"2024-03-19T03:08:54.614079Z","iopub.status.idle":"2024-03-19T03:08:54.623975Z","shell.execute_reply.started":"2024-03-19T03:08:54.614047Z","shell.execute_reply":"2024-03-19T03:08:54.622496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestras las columnas del DataFrames\ntrain_person_1.columns","metadata":{"id":"e-GszYilcmru","outputId":"b1752bde-cea7-4c63-fbbe-8dc9e1f4a306","execution":{"iopub.status.busy":"2024-03-19T03:08:57.026590Z","iopub.execute_input":"2024-03-19T03:08:57.026990Z","iopub.status.idle":"2024-03-19T03:08:57.038117Z","shell.execute_reply.started":"2024-03-19T03:08:57.026960Z","shell.execute_reply":"2024-03-19T03:08:57.036325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra las 20 primeras filas del DataFrames\ntrain_person_1.head(20)","metadata":{"id":"3fcCGDJjcpQk","outputId":"189414ed-6c43-4ac7-eee8-50e037ef9997","execution":{"iopub.status.busy":"2024-03-19T03:08:59.371288Z","iopub.execute_input":"2024-03-19T03:08:59.372029Z","iopub.status.idle":"2024-03-19T03:08:59.408220Z","shell.execute_reply.started":"2024-03-19T03:08:59.371968Z","shell.execute_reply":"2024-03-19T03:08:59.406209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra datos descriptivas de las columnas numéricas del DataFrame\ntrain_person_1.describe()","metadata":{"id":"UOFLST95c2z1","outputId":"5c2b7174-822c-4dda-ad6b-414c4911b2a7","execution":{"iopub.status.busy":"2024-03-19T03:09:02.682560Z","iopub.execute_input":"2024-03-19T03:09:02.683032Z","iopub.status.idle":"2024-03-19T03:09:03.800190Z","shell.execute_reply.started":"2024-03-19T03:09:02.682997Z","shell.execute_reply":"2024-03-19T03:09:03.799150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convierte el DataFrame train_basetable de Polars a un DataFrames de Pandas\ndata_df_pd_04 = train_person_1.to_pandas()\n\n# Obteniendo la cantidad de columnas y almacenando en la variable \"serie\"\nserie = train_person_1.columns\n\n# Generando el histograma y mostrando el grafico\ndata_df_pd_04[serie].hist(bins=20, figsize=(20,10))\nplt.show()","metadata":{"id":"qHSkp8g-c8of","outputId":"ea823485-96d5-47aa-8191-9c204068ffa8","execution":{"iopub.status.busy":"2024-03-19T03:09:08.023050Z","iopub.execute_input":"2024-03-19T03:09:08.024456Z","iopub.status.idle":"2024-03-19T03:09:16.779823Z","shell.execute_reply.started":"2024-03-19T03:09:08.024413Z","shell.execute_reply":"2024-03-19T03:09:16.778450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra información concisa del DataFrames de Panda\ndata_df_pd_04.info()","metadata":{"id":"7OOAqzmkdENZ","outputId":"631fb362-40b1-4d18-e122-6c96a20f9f7e","execution":{"iopub.status.busy":"2024-03-19T03:09:20.429809Z","iopub.execute_input":"2024-03-19T03:09:20.430245Z","iopub.status.idle":"2024-03-19T03:09:20.445976Z","shell.execute_reply.started":"2024-03-19T03:09:20.430213Z","shell.execute_reply":"2024-03-19T03:09:20.444618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra la cantidad de nulos que tiene mi variables del DataFrames de Panda\ndata_df_pd_04.isna().sum()","metadata":{"id":"gl6hPp4fdOUi","outputId":"cc123d70-e71b-48b8-c3b2-36f9106e308e","execution":{"iopub.status.busy":"2024-03-19T03:09:23.756086Z","iopub.execute_input":"2024-03-19T03:09:23.756498Z","iopub.status.idle":"2024-03-19T03:09:29.419397Z","shell.execute_reply.started":"2024-03-19T03:09:23.756467Z","shell.execute_reply":"2024-03-19T03:09:29.417982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra la cantidad de valores nulos por columna de manera grafica\n# Se puede identificar un patron de las variables nula\nmsno.matrix(data_df_pd_04)\nplt.show()","metadata":{"outputId":"b156e133-85ea-4fd4-f805-e9b368737f2f","id":"ZTIMOjgLdmpi","execution":{"iopub.status.busy":"2024-03-19T03:09:40.744862Z","iopub.execute_input":"2024-03-19T03:09:40.745498Z","iopub.status.idle":"2024-03-19T03:10:20.811169Z","shell.execute_reply.started":"2024-03-19T03:09:40.745447Z","shell.execute_reply":"2024-03-19T03:10:20.809545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.heatmap(data_df_pd_04)\nplt.show()","metadata":{"id":"oKctN1YogR3t","outputId":"456a02b8-b21c-49e1-d003-122a787ed7e3","execution":{"iopub.status.busy":"2024-03-19T03:10:26.570231Z","iopub.execute_input":"2024-03-19T03:10:26.570630Z","iopub.status.idle":"2024-03-19T03:10:45.729716Z","shell.execute_reply.started":"2024-03-19T03:10:26.570601Z","shell.execute_reply":"2024-03-19T03:10:45.728580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mostrando el tipo de la variable \"train_credit_bureau_b_2\"\nprint(type(train_credit_bureau_b_2))","metadata":{"id":"0j6raNZwhG6J","outputId":"73c50e0d-c800-4676-c4c4-725a18931819","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra las dimensiones del DataFrames (filas, columnas)\ntrain_credit_bureau_b_2.shape","metadata":{"id":"YIN1HhbxhOku","outputId":"56b6b99d-f4ee-478e-fe4d-185164341a30","execution":{"iopub.status.busy":"2024-03-19T03:10:54.107105Z","iopub.execute_input":"2024-03-19T03:10:54.107649Z","iopub.status.idle":"2024-03-19T03:10:54.116516Z","shell.execute_reply.started":"2024-03-19T03:10:54.107609Z","shell.execute_reply":"2024-03-19T03:10:54.114986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestras las columnas del DataFrames\ntrain_credit_bureau_b_2.columns","metadata":{"id":"__V9bY66hUTX","outputId":"4dfb70d0-80cc-4f69-91e6-0c675659ec06","execution":{"iopub.status.busy":"2024-03-19T03:10:56.099993Z","iopub.execute_input":"2024-03-19T03:10:56.100405Z","iopub.status.idle":"2024-03-19T03:10:56.110310Z","shell.execute_reply.started":"2024-03-19T03:10:56.100376Z","shell.execute_reply":"2024-03-19T03:10:56.108960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra las 5 primeras filas del DataFrames\ntrain_credit_bureau_b_2.head()","metadata":{"id":"TH5wo7v0hb7d","outputId":"47d34437-7ecb-4c3c-ea1c-d6f9cdc223c4","execution":{"iopub.status.busy":"2024-03-19T03:10:58.848306Z","iopub.execute_input":"2024-03-19T03:10:58.848709Z","iopub.status.idle":"2024-03-19T03:10:58.860008Z","shell.execute_reply.started":"2024-03-19T03:10:58.848679Z","shell.execute_reply":"2024-03-19T03:10:58.857931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra datos descriptivas de las columnas numéricas del DataFrame\ntrain_credit_bureau_b_2.describe()","metadata":{"id":"LuF9R2qdhgYx","outputId":"df1d765f-22da-4889-cc26-6fde03bc0921","execution":{"iopub.status.busy":"2024-03-19T03:11:01.687448Z","iopub.execute_input":"2024-03-19T03:11:01.688144Z","iopub.status.idle":"2024-03-19T03:11:01.812280Z","shell.execute_reply.started":"2024-03-19T03:11:01.688077Z","shell.execute_reply":"2024-03-19T03:11:01.810849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convierte el DataFrame train_basetable de Polars a un DataFrames de Pandas\ndata_df_pd_05 = train_credit_bureau_b_2.to_pandas()\n\n# Obteniendo la cantidad de columnas y almacenando en la variable \"serie\"\nserie = train_credit_bureau_b_2.columns\n\n# Generando el histograma y mostrando el grafico\ndata_df_pd_05[serie].hist(bins=20, figsize=(20,10))\nplt.show()","metadata":{"id":"rPdyb9DKhr2V","outputId":"22e8f44f-7838-4fff-d411-ff4aae0348b2","execution":{"iopub.status.busy":"2024-03-19T03:11:05.366154Z","iopub.execute_input":"2024-03-19T03:11:05.366597Z","iopub.status.idle":"2024-03-19T03:11:07.218540Z","shell.execute_reply.started":"2024-03-19T03:11:05.366563Z","shell.execute_reply":"2024-03-19T03:11:07.217393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra información concisa del DataFrames de Panda\ndata_df_pd_05.info()","metadata":{"id":"sSfpBIVFh1py","outputId":"20e5432a-3709-4d0f-b327-1e0abe732c8f","execution":{"iopub.status.busy":"2024-03-19T03:11:11.929120Z","iopub.execute_input":"2024-03-19T03:11:11.929566Z","iopub.status.idle":"2024-03-19T03:11:12.098999Z","shell.execute_reply.started":"2024-03-19T03:11:11.929532Z","shell.execute_reply":"2024-03-19T03:11:12.097409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra la cantidad de nulos que tiene mi variables del DataFrames de Panda\ndata_df_pd_05.isna().sum()","metadata":{"id":"QUKNeaTCh9b3","outputId":"e163296f-8870-4ce8-b2e0-cca0a0e1052e","execution":{"iopub.status.busy":"2024-03-19T03:11:15.193534Z","iopub.execute_input":"2024-03-19T03:11:15.194193Z","iopub.status.idle":"2024-03-19T03:11:15.357290Z","shell.execute_reply.started":"2024-03-19T03:11:15.194119Z","shell.execute_reply":"2024-03-19T03:11:15.355428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Muestra la cantidad de valores nulos por columna de manera grafica\n# Se puede identificar un patron de las variables nula\nmsno.matrix(data_df_pd_05)\nplt.show()","metadata":{"id":"ARj3LZrIiZpZ","outputId":"6f91b016-3f4d-4286-d5c8-881b86570235","execution":{"iopub.status.busy":"2024-03-19T03:11:17.067172Z","iopub.execute_input":"2024-03-19T03:11:17.067572Z","iopub.status.idle":"2024-03-19T03:11:20.687137Z","shell.execute_reply.started":"2024-03-19T03:11:17.067538Z","shell.execute_reply":"2024-03-19T03:11:20.685471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# obtiene las columnas del DataFrame\ncolumn = data_df_pd_05.columns\n\n# Crea un histograma por cada una de las columnas\ndata_df_pd_05[column].hist(bins=50, figsize=(20,15))\n\n# Muestra el histograma generado\nplt.show()","metadata":{"id":"aVPlhd7_itzD","outputId":"90e813e0-ae0a-4e86-f75f-e5bcd15561db","execution":{"iopub.status.busy":"2024-03-19T03:11:25.707783Z","iopub.execute_input":"2024-03-19T03:11:25.708249Z","iopub.status.idle":"2024-03-19T03:11:27.830294Z","shell.execute_reply.started":"2024-03-19T03:11:25.708215Z","shell.execute_reply":"2024-03-19T03:11:27.828584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Indico las columnas que quiero del DataFrame\ncolumn = [\"pmts_dpdvalue_108P\", \"pmts_pmtsoverdue_635A\"]\n\n# Crea un histograma por cada una de las columnas\ndata_df_pd_05[column].hist(bins=50, figsize=(20,15))\n\n# Muestra el histograma generado\nplt.show()","metadata":{"id":"t9tCFJiuiWYL","outputId":"49effeee-70da-4796-e276-2e11a4f37bee","execution":{"iopub.status.busy":"2024-03-19T03:11:33.238066Z","iopub.execute_input":"2024-03-19T03:11:33.238549Z","iopub.status.idle":"2024-03-19T03:11:34.191431Z","shell.execute_reply.started":"2024-03-19T03:11:33.238515Z","shell.execute_reply":"2024-03-19T03:11:34.189006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PASO 07 - ANALISIS EXPLORATORIO / TEST\n","metadata":{"id":"WfXDEktnmwl0"}},{"cell_type":"code","source":"test_static","metadata":{"id":"KwBGhLMLODPM","outputId":"de23c54c-cc84-403c-f34a-9ef04cfe4c1e","execution":{"iopub.status.busy":"2024-03-19T03:11:39.087785Z","iopub.execute_input":"2024-03-19T03:11:39.088295Z","iopub.status.idle":"2024-03-19T03:11:39.148636Z","shell.execute_reply.started":"2024-03-19T03:11:39.088264Z","shell.execute_reply":"2024-03-19T03:11:39.147262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable","metadata":{"id":"fo8f73ZEOo5U","outputId":"b32ea9bf-2952-492a-f748-646a1428ca9e","execution":{"iopub.status.busy":"2024-03-19T03:11:42.436922Z","iopub.execute_input":"2024-03-19T03:11:42.437369Z","iopub.status.idle":"2024-03-19T03:11:42.449168Z","shell.execute_reply.started":"2024-03-19T03:11:42.437336Z","shell.execute_reply":"2024-03-19T03:11:42.447493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_person_1","metadata":{"id":"vRQyrGGPwOYN","outputId":"4f868385-43ea-4b6c-a951-e4b78669e608","execution":{"iopub.status.busy":"2024-03-19T03:11:46.520304Z","iopub.execute_input":"2024-03-19T03:11:46.520706Z","iopub.status.idle":"2024-03-19T03:11:46.541351Z","shell.execute_reply.started":"2024-03-19T03:11:46.520678Z","shell.execute_reply":"2024-03-19T03:11:46.539590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_b_2","metadata":{"id":"eb68Xh0owSHg","outputId":"efd86094-f7a4-4f88-ee94-a18f3b4435df","execution":{"iopub.status.busy":"2024-03-19T03:11:49.213640Z","iopub.execute_input":"2024-03-19T03:11:49.214084Z","iopub.status.idle":"2024-03-19T03:11:49.227089Z","shell.execute_reply.started":"2024-03-19T03:11:49.214050Z","shell.execute_reply":"2024-03-19T03:11:49.225004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable","metadata":{"id":"veO8Y3UcwTvl","outputId":"f76326c0-8869-4288-b0d0-7032b960f000","execution":{"iopub.status.busy":"2024-03-19T03:11:52.457103Z","iopub.execute_input":"2024-03-19T03:11:52.457981Z","iopub.status.idle":"2024-03-19T03:11:52.473643Z","shell.execute_reply.started":"2024-03-19T03:11:52.457869Z","shell.execute_reply":"2024-03-19T03:11:52.472418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PASO 08 - FEATURE ENGINEERING","metadata":{"id":"1da-F-i8nrhz"}},{"cell_type":"code","source":"# Operación de agrupación y agregación en un DataFrame de Polars (train_person_1).\ntrain_person_1_feats_1 = train_person_1.group_by(\"case_id\").agg(                      # Agrupar todas las filas que tengan el mismo valor en la columna \"case_id\".\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),       # Calcula el valor máximo de la columna \"mainoccupationinc_384A\" para cada grupo y se le asigna un alias \"mainoccupationinc_384A_max\n\n# Verifica si algún valor en la columna \"incometype_1044T\" dentro de cada grupo es igual a \"SELFEMPLOYED\".\n# El resultado se almacena en la columna \"mainoccupationinc_384A_any_selfemployed\", indicando si en algún momento del grupo se registra \"SELFEMPLOYED\".\n      (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)","metadata":{"id":"A5SXPboIUsrk","execution":{"iopub.status.busy":"2024-03-19T03:11:55.962901Z","iopub.execute_input":"2024-03-19T03:11:55.963480Z","iopub.status.idle":"2024-03-19T03:11:56.515583Z","shell.execute_reply.started":"2024-03-19T03:11:55.963437Z","shell.execute_reply":"2024-03-19T03:11:56.514401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1_feats_1","metadata":{"id":"SBmxlhkAU8BM","outputId":"3fb9af30-b8a0-4e9d-ed64-2b7c823bebf5","execution":{"iopub.status.busy":"2024-03-19T03:12:02.985411Z","iopub.execute_input":"2024-03-19T03:12:02.985869Z","iopub.status.idle":"2024-03-19T03:12:02.997329Z","shell.execute_reply.started":"2024-03-19T03:12:02.985834Z","shell.execute_reply":"2024-03-19T03:12:02.995816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Selecciona ciertas columnas del DataFrame train_person_1, aplica un filtro para mantener solo las filas donde el valor\n# en la columna \"num_group1\" sea igual a 0, elimina la columna \"num_group1\" y finalmente renombra la columna \"housetype_905L\"\n# como \"person_housetype\" en el DataFrame resultante\n\ntrain_person_1_feats_2 = train_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]\n                         ).filter( pl.col(\"num_group1\") == 0\n                        ).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})","metadata":{"id":"ahGNRVocVMxD","execution":{"iopub.status.busy":"2024-03-19T03:12:07.498602Z","iopub.execute_input":"2024-03-19T03:12:07.499312Z","iopub.status.idle":"2024-03-19T03:12:07.547799Z","shell.execute_reply.started":"2024-03-19T03:12:07.499278Z","shell.execute_reply":"2024-03-19T03:12:07.546310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1_feats_2","metadata":{"id":"Z5eP95rmVWac","outputId":"5e577084-d392-4174-cf2b-8e3c15fc348b","execution":{"iopub.status.busy":"2024-03-19T03:12:11.250519Z","iopub.execute_input":"2024-03-19T03:12:11.250940Z","iopub.status.idle":"2024-03-19T03:12:11.262569Z","shell.execute_reply.started":"2024-03-19T03:12:11.250909Z","shell.execute_reply":"2024-03-19T03:12:11.260015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Agrupa todas la s filas que tengan el mismo valor de la columna \"case_id\" y  luego agregara otras columnas agrupadas de \"pmts_pmtsoverdue_635A\" y \"pmts_dpdvalue_108P\"\ntrain_credit_bureau_b_2_feats = train_credit_bureau_b_2.group_by(\"case_id\").agg(\n\n# Calcula el valor máximo de la columna para cada grupo de \"pmts_pmtsoverdue_635A\" y se le asigna un nuevo nombre \"pmts_pmtsoverdue_635A_max\"\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n\n# Calcula el valor máximo de la columna para cada grupo de \"pmts_dpdvalue_108P\" que sea mayor a 31 y se le asigna un nuevo nombre \"pmts_dpdvalue_108P_over31\"\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)","metadata":{"id":"4NuqWF8eVdIT","execution":{"iopub.status.busy":"2024-03-19T03:12:16.897304Z","iopub.execute_input":"2024-03-19T03:12:16.898782Z","iopub.status.idle":"2024-03-19T03:12:16.939207Z","shell.execute_reply.started":"2024-03-19T03:12:16.898720Z","shell.execute_reply":"2024-03-19T03:12:16.937868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_b_2_feats","metadata":{"id":"6esJzDhxVi1r","outputId":"729f694e-a3df-407d-b7bc-6c01a79e60e3","execution":{"iopub.status.busy":"2024-03-19T03:12:20.156022Z","iopub.execute_input":"2024-03-19T03:12:20.156711Z","iopub.status.idle":"2024-03-19T03:12:20.170943Z","shell.execute_reply.started":"2024-03-19T03:12:20.156676Z","shell.execute_reply":"2024-03-19T03:12:20.169241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Selecciona todas las columnas del DataFrame \"train_static\" cuyo último carácter es \"A\" o \"M\" y almacena los nombres de estas columnas en la lista\n\n# Inicializa la lista vacia llamada \"selected_static_cols\"\nselected_static_cols = []\nfor col in train_static.columns:\n\n  # Verifica si el ultimo carcter del nombre de la columna termina en \"A\" o \"M\"\n    if col[-1] in (\"A\", \"M\"):\n\n      # Si la condición se cumple, inserta el nombre de la columna en lista \"selected_static_cols\"\n        selected_static_cols.append(col)\n\n# Mostramos al final todas las columnas que insertamos\nprint(selected_static_cols)","metadata":{"id":"vGXoOKXDWBET","outputId":"a3f8c830-1053-4a5b-a37e-afebe14239b0","execution":{"iopub.status.busy":"2024-03-19T03:12:24.016198Z","iopub.execute_input":"2024-03-19T03:12:24.016697Z","iopub.status.idle":"2024-03-19T03:12:24.026576Z","shell.execute_reply.started":"2024-03-19T03:12:24.016663Z","shell.execute_reply":"2024-03-19T03:12:24.024395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Selecciona todas las columnas del DataFrame \"train_static_cb\" cuyo último carácter es \"A\" o \"M\" y almacena los nombres de estas columnas en la lista\n\n# Inicializa la lista vacia llamada \"selected_static_cb_cols\"\nselected_static_cb_cols = []\nfor col in train_static_cb.columns:\n\n  # Verifica si el ultimo carcter del nombre de la columna termina en \"A\" o \"M\"\n    if col[-1] in (\"A\", \"M\"):\n\n      # Si la condición se cumple, inserta el nombre de la columna en lista \"selected_static_cb_cols\"\n        selected_static_cb_cols.append(col)\n\n# Mostramos al final todas las columnas que insertamos\nprint(selected_static_cb_cols)","metadata":{"id":"eXHs-yxHWI0j","outputId":"1614baaa-640c-4361-f1c7-99244728b6ac","execution":{"iopub.status.busy":"2024-03-19T03:12:28.130840Z","iopub.execute_input":"2024-03-19T03:12:28.131394Z","iopub.status.idle":"2024-03-19T03:12:28.142558Z","shell.execute_reply.started":"2024-03-19T03:12:28.131352Z","shell.execute_reply":"2024-03-19T03:12:28.141042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Unir todas la tablas\n# Combinar datos de múltiples DataFrames en un solo DataFrame llamado data, utilizando la columna \"case_id\" como clave de unión.\n# La unión izquierda garantiza que todas las filas del DataFrame train_basetable se conserven en el DataFrame resultante, y las filas correspondientes\n# de los otros DataFrames se agregan si hay una coincidencia en la columna \"case_id\".\n\ndata_united = train_basetable.join(\n    train_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\").join(\n    train_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\").join(\n    train_person_1_feats_1, how=\"left\", on=\"case_id\").join(\n    train_person_1_feats_2, how=\"left\", on=\"case_id\").join(\n    train_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n)\nprint(type(data_united))\ndata = replace_null_with_moda(data_united)\n#data = data_united","metadata":{"papermill":{"duration":1.664029,"end_time":"2024-02-07T21:28:22.750906","exception":false,"start_time":"2024-02-07T21:28:21.086877","status":"completed"},"tags":[],"id":"eed7be3e","outputId":"519487c0-2f18-426a-ae9e-fab4fe54ad00","execution":{"iopub.status.busy":"2024-03-19T03:12:32.068058Z","iopub.execute_input":"2024-03-19T03:12:32.068702Z","iopub.status.idle":"2024-03-19T03:12:33.756022Z","shell.execute_reply.started":"2024-03-19T03:12:32.068671Z","shell.execute_reply":"2024-03-19T03:12:33.754859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analizando la variables numericas podemos observar que existen valores atipico en gran mendida en la variable annuitynextmonth_57A,pmts_pmtsoverdue_635A_max\ndata_united.describe()","metadata":{"id":"fXUiARC7V2Pa","outputId":"da96057d-68f7-40d5-f497-977187e55cab","execution":{"iopub.status.busy":"2024-03-19T03:14:11.539261Z","iopub.execute_input":"2024-03-19T03:14:11.539669Z","iopub.status.idle":"2024-03-19T03:14:13.118179Z","shell.execute_reply.started":"2024-03-19T03:14:11.539641Z","shell.execute_reply":"2024-03-19T03:14:13.116336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convertir el DataFrame de Polars a un DataFrame de Pandas\ndata_pd = data.to_pandas()\n\n# Graficar los datos\nplt.boxplot(data_pd[\"annuity_780A\"], vert = True)\nplt.show()","metadata":{"id":"dE8_OZvZcfv6","outputId":"ecddb195-74b1-466c-b104-853af920f58f","execution":{"iopub.status.busy":"2024-03-19T03:14:20.991157Z","iopub.execute_input":"2024-03-19T03:14:20.991578Z","iopub.status.idle":"2024-03-19T03:14:22.899346Z","shell.execute_reply.started":"2024-03-19T03:14:20.991546Z","shell.execute_reply":"2024-03-19T03:14:22.897531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Revisando la cantidad de nulos de todas las variables\ndata_pd.isna().sum()","metadata":{"id":"4iE2rG3ux7VI","outputId":"896f0298-999b-441c-b316-9b3be9515b60","execution":{"iopub.status.busy":"2024-03-19T03:14:25.996444Z","iopub.execute_input":"2024-03-19T03:14:25.997070Z","iopub.status.idle":"2024-03-19T03:14:28.505604Z","shell.execute_reply.started":"2024-03-19T03:14:25.997034Z","shell.execute_reply":"2024-03-19T03:14:28.504165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Revisar si hay relación de las varibles nulos, No se encontro relaciones en las variables de campos nulos\nmsno.matrix(data_pd)","metadata":{"id":"dqaxuQ8rVDzf","execution":{"iopub.status.busy":"2024-03-19T03:14:31.105387Z","iopub.execute_input":"2024-03-19T03:14:31.105799Z","iopub.status.idle":"2024-03-19T03:15:03.467292Z","shell.execute_reply.started":"2024-03-19T03:14:31.105767Z","shell.execute_reply":"2024-03-19T03:15:03.466299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PASO 09 - ELIMINACION DE NULOS","metadata":{"id":"Gaygxot8p9K0"}},{"cell_type":"markdown","source":"\n\n*   Revisando las columnas que tienen demasiado nulos\n\n","metadata":{"id":"9vRPduUSwuvc"}},{"cell_type":"code","source":"data.describe()","metadata":{"id":"2rseULVeuY7M","outputId":"29296ea8-a036-4554-8e6a-e6557fc1d783"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{"id":"LGUDRAEcxW27"}},{"cell_type":"code","source":"# En Pandas\n# data[[\"lastotherinc_902A\", \"lastotherlnsexpense_631A\"]]\ndata.select([pl.col([\"lastotherinc_902A\", \"lastotherlnsexpense_631A\"])])\n","metadata":{"id":"DwIaoOTbqsr0","outputId":"b5fe1b63-8a52-4b60-e408-2146b3891d2a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.select([pl.col([\"lastotherinc_902A\", \"lastotherlnsexpense_631A\"])]).describe()","metadata":{"id":"FxJ8qDsqs9Ob","outputId":"dfb4a81d-e524-4d55-c5f6-07b4290050e2"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*   Eliminando las 2 columnas con demasiados nulos (Previo Backup)","metadata":{"id":"Gstao5VTynCd"}},{"cell_type":"code","source":"# X_train_bak = X_train\n# X_train = X_train.drop(columns = [\"lastotherinc_902A\", \"lastotherlnsexpense_631A\"])\n# data_bak = data.clone()\n# data = data.drop(['lastotherinc_902A', 'lastotherlnsexpense_631A'])","metadata":{"id":"y-GwFvIPytDr"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape","metadata":{"id":"Yy9PK22L2TST","outputId":"808c0311-05a3-4427-f615-94e9e071e0f0"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PASO 10 - CONTRUYENDO DATA FINAL DE TRAIN / TEST (SUBMISSION) / VALIDACION","metadata":{"id":"xm6dRerGwB-z"}},{"cell_type":"code","source":"# Operación de agrupación y agregación en un DataFrame de Polars (train_person_1).\n# Agrupar todas las filas que tengan el mismo valor en la columna \"case_id\".\ntest_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n\n# Calcula el valor máximo de la columna \"mainoccupationinc_384A\" para cada grupo y se le asigna un alias \"mainoccupationinc_384A_max\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n\n# Verifica si algún valor en la columna \"incometype_1044T\" dentro de cada grupo es igual a \"SELFEMPLOYED\".\n# El resultado se almacena en la columna \"mainoccupationinc_384A_any_selfemployed\", indicando si en algún momento del grupo se registra \"SELFEMPLOYED\".\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\n# Selecciona las columnas \"case_id\", \"num_group1\" y \"housetype_905L\" del DataFrame \"test_person_1\" y crea un nuevo DataFrame llamado \"test_person_1_feats_2\"\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n\n# Filtra el DataFrame \"test_person_1_feats_2\" para mantener solo las filas donde el valor de la columna \"num_group1\" sea igual a 0\n    pl.col(\"num_group1\") == 0\n# Elimina la columna \"num_group1\" del DataFrame resultante. Esto se hace con el método .drop() que recibe el nombre de la columna que se eliminará\n# Cambia el nombre de la columna \"housetype_905L\" a \"person_housetype\" en el DataFrame resultante\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\n# Agrupa todas la s filas que tengan el mismo valor de la columna \"case_id\" y  luego agregara otras columnas agrupadas de \"pmts_pmtsoverdue_635A\" y \"pmts_dpdvalue_108P\"\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n\n# Calcula el valor máximo de la columna para cada grupo de \"pmts_pmtsoverdue_635A\" y se le asigna un nuevo nombre \"pmts_pmtsoverdue_635A_max\"\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n\n# Calcula el valor máximo de la columna para cada grupo de \"pmts_dpdvalue_108P\" que sea mayor a 31 y se le asigna un nuevo nombre \"pmts_dpdvalue_108P_over31\"\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\n# Une múltiples DataFrames al DataFrame \"test_basetable\" utilizando la columna \"case_id\" como clave de unión, lo que resulta en un nuevo DataFrame llamado \"data_submission\".\n# Este nuevo DataFrame tendrá todas las columnas de test_basetable, junto con las columnas de los DataFrames unidos que coincidan con la columna \"case_id\". El tipo de unión utilizado (how=\"left\") asegura que se mantengan todas las filas de test_basetable\ndata_submission = test_basetable.join(\n    test_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n)","metadata":{"papermill":{"duration":0.025248,"end_time":"2024-02-07T21:28:22.783159","exception":false,"start_time":"2024-02-07T21:28:22.757911","status":"completed"},"tags":[],"id":"3cc4a29b","execution":{"iopub.status.busy":"2024-03-19T03:15:35.298117Z","iopub.execute_input":"2024-03-19T03:15:35.298905Z","iopub.status.idle":"2024-03-19T03:15:35.319947Z","shell.execute_reply.started":"2024-03-19T03:15:35.298843Z","shell.execute_reply":"2024-03-19T03:15:35.318568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extrae todos los identificadores únicos del DataFrame data, los desordena y los almacena en case_ids. Esto es para asegurar que los datos de entrenamiento, validación y prueba sean seleccionados de manera aleatoria\ncase_ids = data[\"case_id\"].unique().shuffle(seed=1)\n\n# Divide la data de entrenamiento que será un 60% para el entrenamiento y un 40% para el test y la validación\ncase_ids_train, case_ids_test = train_test_split(case_ids, train_size=0.6, random_state=1)\n\n# Divide la data de entrenamiento que será un 20% para el test y la validación respectivamente\ncase_ids_valid, case_ids_test = train_test_split(case_ids_test, train_size=0.5, random_state=1)\n\n# Inicializamos la lista vacia de nombre \"cols_pred\"\ncols_pred = []\nfor col in data.columns:\n\n# Verifica si el último carácter de la columna es una letra mayúscula y si todos los caracteres excepto el último son letras minúsculas\n    if col[-1].isupper() and col[:-1].islower():\n\n# Si se cumple la condición, se agrega el nombre de la columna a la lista \"cols_pred\"\n        cols_pred.append(col)\n# Imprime las columnas agregadas a la lista\nprint(cols_pred)\n\n# La función from_polars_to_pandas devuelve una tupla que contiene los tres DataFrames de Pandas mencionados anteriormente\n# Cada uno de estos DataFrames contiene datos relacionados con los casos presentes en el DataFrame de Polars case_ids\ndef from_polars_to_pandas(case_ids: pl.DataFrame) -> pl.DataFrame:\n    return (\n# Filtra el DataFrame data para mantener solo las filas donde el valor de la columna \"case_id\" está presente en el DataFrame \"case_ids\"\n        data.filter(pl.col(\"case_id\").is_in(case_ids))\n\n# Selecciona las columnas \"case_id\", \"WEEK_NUM\" y \"target\" de las filas filtradas y convierte este subconjunto de datos a un DataFrame de Pandas\n         [[\"case_id\", \"WEEK_NUM\", \"target\"]].to_pandas(),\n\n# Similar a la línea anterior, pero selecciona solo las columnas contenidas en la lista \"cols_pred\"\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[cols_pred].to_pandas(),\n\n# Similar a la línea anterior, solo selecciona la columna \"target\" de las filas filtradas\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[\"target\"].to_pandas()\n    )\n\n# Convierte la data en una tupla con los 3 DataFrame de Panda\nbase_train, X_train, y_train = from_polars_to_pandas(case_ids_train)\nbase_valid, X_valid, y_valid = from_polars_to_pandas(case_ids_valid)\nbase_test, X_test, y_test = from_polars_to_pandas(case_ids_test)\n\n# Transforma los tipos de datos de las columnas, la limpia de cadenas de texto y elimina los caracteres no deseados\nfor df in [X_train, X_valid, X_test]:\n    df = convert_strings(df)","metadata":{"papermill":{"duration":9.431294,"end_time":"2024-02-07T21:28:32.221469","exception":false,"start_time":"2024-02-07T21:28:22.790175","status":"completed"},"tags":[],"id":"7195e82a","outputId":"64c40282-2ffb-4d60-dd25-0676fd982b83","execution":{"iopub.status.busy":"2024-03-19T03:15:40.438355Z","iopub.execute_input":"2024-03-19T03:15:40.439018Z","iopub.status.idle":"2024-03-19T03:15:49.368667Z","shell.execute_reply.started":"2024-03-19T03:15:40.438984Z","shell.execute_reply":"2024-03-19T03:15:49.367534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_train.shape","metadata":{"id":"UNQRGrbAg2yz","outputId":"5ce38927-9e3d-4539-9fc7-31e36ff90532","execution":{"iopub.status.busy":"2024-03-19T03:15:55.497272Z","iopub.execute_input":"2024-03-19T03:15:55.497732Z","iopub.status.idle":"2024-03-19T03:15:55.506133Z","shell.execute_reply.started":"2024-03-19T03:15:55.497699Z","shell.execute_reply":"2024-03-19T03:15:55.504455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_train.info()","metadata":{"id":"wXg8R8uVgui8","outputId":"7d3d494e-9725-4644-9892-aef36c09f8cd","execution":{"iopub.status.busy":"2024-03-19T03:15:57.285036Z","iopub.execute_input":"2024-03-19T03:15:57.285524Z","iopub.status.idle":"2024-03-19T03:15:57.304537Z","shell.execute_reply.started":"2024-03-19T03:15:57.285487Z","shell.execute_reply":"2024-03-19T03:15:57.303538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_train","metadata":{"id":"xSYiAomdg-g7","outputId":"25c1ae12-d180-4362-83be-bc74f4e17196"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"id":"wOh5SlqoyceV","outputId":"7bf44c9f-c54b-4f4d-97a2-d062c2777c4f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(X_train))","metadata":{"id":"4BKsW6z3Yfl8","outputId":"a5455b20-641d-4482-f81e-fd4610956ed3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.head(200)","metadata":{"id":"CH-31oqQdyBM","outputId":"6c9cd957-426b-44f7-b6eb-e4ce5174e151"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"id":"3CpyNBkY1Ry4","outputId":"7aef0a8d-24ee-47eb-a5e9-2fa6f11ecb49"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"id":"mw6XrbTAhKbr","outputId":"e7ef87af-f924-49ac-810a-d08f30a79843"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.info()","metadata":{"id":"rBXu1hrWhWOL","outputId":"254700af-732e-4df6-fe2a-f0a2e32e0a60"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid.shape","metadata":{"id":"MJuGx-WbhdeL","outputId":"ef8f32a7-0bc2-4cb7-aaa8-61f7f83067be"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train: {X_train.shape}\")\nprint(f\"Valid: {X_valid.shape}\")\nprint(f\"Test: {X_test.shape}\")","metadata":{"execution":{"iopub.execute_input":"2024-02-07T21:28:32.238895Z","iopub.status.busy":"2024-02-07T21:28:32.238439Z","iopub.status.idle":"2024-02-07T21:28:32.246480Z","shell.execute_reply":"2024-02-07T21:28:32.244638Z"},"papermill":{"duration":0.019901,"end_time":"2024-02-07T21:28:32.249064","exception":false,"start_time":"2024-02-07T21:28:32.229163","status":"completed"},"tags":[],"id":"6a275858","outputId":"343c3bc6-31c2-4141-f216-09b8c48e6aaf"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PASO 11 - ENTRENAMIENTO USANDO K-FOLD CROSS-VALIDATION CON LIGHTGBM y XGBOOST","metadata":{"id":"pDCmZ31IoOSD"}},{"cell_type":"code","source":"# Crear una función para realizar la validación cruzada usando K-fold Cross-Validation \ndef k_fold_cross_validation():\n    # Definir el número de pliegues para la validación cruzada\n    n_splits = 2\n    random_state=2\n    \n    # Crear un objeto KFold con el número de pliegues definido\n    kfold = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)\n\n    # Inicializando la lista que almacenara los modelos entrenados y los puntajes de estabilidad en cada pliegue\n    models = []\n    \n    # Inicializando la lista que almacenara los puntajes de estabilidad en cada pliegue\n    stability_scores = []\n    \n    # Ciclo sobre los pliegues de la validación cruzada\n    for i, (train_index_kf, valid_index_kf) in enumerate(kfold.split(X_train)):\n\n        # Dividir el conjunto de la data en entrenamiento y validación para este pliegue\n        X_train_kf, X_valid_kf = X_train.iloc[train_index_kf], X_train.iloc[valid_index_kf]\n        y_train_kf, y_valid_kf = y_train.iloc[train_index_kf], y_train.iloc[valid_index_kf]\n        \n        # Devolver los conjuntos de datos divididos para este pliegue\n        return X_train_kf, X_valid_kf, y_train_kf, y_valid_kf","metadata":{"id":"FQsSIy9m5oYj","execution":{"iopub.status.busy":"2024-03-19T04:29:27.741955Z","iopub.execute_input":"2024-03-19T04:29:27.742400Z","iopub.status.idle":"2024-03-19T04:29:27.751194Z","shell.execute_reply.started":"2024-03-19T04:29:27.742366Z","shell.execute_reply":"2024-03-19T04:29:27.749741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Definir los parámetros del modelo LightGBM\nparams = {\n    \"boosting_type\": \"gbdt\",         # Tipo de boosting: Gradient Boosting Decision Tree (gbdt)\n    \"objective\": \"binary\",           # Objetivo de la tarea: clasificación binaria\n    \"metric\": \"auc\",                 # Métrica a optimizar: área bajo la curva (AUC)\n    \"max_depth\": 9,                  # Profundidad máxima de los árboles: 9\n    \"num_leaves\": 80,                # Número máximo de hojas por árbol: 80\n    \"learning_rate\": 0.04,           # Tasa de aprendizaje: 0.04\n    \"feature_fraction\": 0.7,         # Fracción de características a considerar en cada árbol: 0.7\n    \"bagging_fraction\": 0.9,         # Fracción de muestras (instancias) a considerar en cada árbol: 0.9\n    \"bagging_freq\": 5,               # Frecuencia de bagging: 5 (cada 5 iteraciones)\n    \"n_estimators\": 1000,            # Número de estimadores (árboles) a entrenar: 1000\n    \"verbose\": -1                    # Nivel de verbosidad: -1 (sin mensajes)\n}\n\n# Llama a la fucnion que divide los datos que contiene los conjuntos de datos de entrenamiento y validación para cada pliegue\nk_fold = k_fold_cross_validation()\n\n# Lista para almacenar los puntajes AUC en cada pliegue\nauc_scores = []\n\n# Crear el conjunto de datos de LightGBM para este pliegue\nlgb_train_k_fold = lgb.Dataset(k_fold[0], label=k_fold[2])\nlgb_valid_k_fold = lgb.Dataset(k_fold[1], label=k_fold[3], reference=lgb_train_k_fold)\n\n\n# Entrenar el modelo LightGBM para este pliegue\ngbm = lgb.train(\n    params,                                                     # Parámetros del modelo\n    lgb_train_k_fold,                                           # Conjunto de datos de entrenamiento\n    valid_sets=[lgb_train_k_fold, lgb_valid_k_fold],            # Conjunto de datos de entrenamiento y validación\n    callbacks=[lgb.log_evaluation(50), lgb.early_stopping(10)]  # Callbacks para registrar la evaluación y detener el entrenamiento prematuramente\n)\n\n# Evaluar el modelo en el conjunto de validación y almacena el AUC\n# Realiza predicciones en el conjunto de validación del pliegue actual utilizando el modelo LightGBM y especifica el número óptimo de iteraciones (árboles) a utilizar para hacer las predicciones\ny_pred_valid_k_fold = gbm.predict(k_fold[1], num_iteration=gbm.best_iteration)\n\n# Calcula el AUC utilizando las etiquetas verdaderas del conjunto de validación \nauc_k_fold = roc_auc_score(k_fold[3], y_pred_valid_k_fold)\n\n# Agrega el AUC calculado al final de la lista \"auc_scores\". Esto permite rastrear los puntajes AUC de todos los pliegues de la validación cruzada K-fold\nauc_scores.append(auc_k_fold)\n","metadata":{"id":"DqsO-F0jCjbI","outputId":"5d729c9a-fda9-4c75-d312-b7d1bacdd9e4","execution":{"iopub.status.busy":"2024-03-19T04:29:30.739230Z","iopub.execute_input":"2024-03-19T04:29:30.744748Z","iopub.status.idle":"2024-03-19T04:30:40.604130Z","shell.execute_reply.started":"2024-03-19T04:29:30.744692Z","shell.execute_reply":"2024-03-19T04:30:40.602664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Definir los parámetros del modelo XGBoost\nparams = {\n    \"device\":\"cuda\",                  # Indica el dispositivo de hardware que se utilizará para entrenar el modelo, en este caso es GPU\n    \"objective\":'binary:logistic',    # Indica que se realizará una clasificación binaria utilizando la función de pérdida de regresión logística\n    \"tree_method\":\"hist\",             # Indica que se utilizará un método basado en histogramas para construir los árboles\n    \"enable_categorical\":True,        # Indica que esta habilitado el manejo de características categóricas\n    \"eval_metric\":'auc',              # Indica que se utilizará el área bajo la curva ROC como métrica de evaluación\n    \"subsample\":1,                    # El valor de 1 indica que se utilizan todas las muestras para entrenar cada arbol\n    \"colsample_bytree\":1,             # El valor de 1 indica que se utilizan todas las características para entrenar cada arbol\n    \"min_child_weight\":1,             # Especifica el peso mínimo necesario para crear un nuevo nodo en el árbol. Un valor más alto puede ayudar a prevenir el sobreajuste\n    \"max_depth\":20,                   # Limita la profundidad máxima de los árboles. En este caso, se establece en 20\n    #gamma=0.7,\n    #reg_alpha=0.7,\n    \"n_estimators\":1200,              # Indica el número de árboles que se entrenarán. En este caso, se establece en 1200\n    \"random_state\":5,                #Semilla para la generación de números aleatorios. Esto asegura que los resultados sean reproducibles\n}\n\n# Llama a la fucnion que divide los datos que contiene los conjuntos de datos de entrenamiento y validación para cada pliegue\nk_fold = k_fold_cross_validation()\n\n# Lista para almacenar los puntajes AUC en cada pliegue\nauc_scores = []\nfitted_models = []\n\n# Crea un modelo de clasificación binaria utilizando la clase XGBClassifier de la biblioteca XGBoost\nxgb_model = xgb.XGBClassifier(**params)\n\n# Entrena el modelo en los datos de entrenamiento y validar su rendimiento en los datos de validación\nxgb_model.fit(\n    k_fold[0], k_fold[2],\n    eval_set=[(k_fold[1], k_fold[3])],\n    early_stopping_rounds=100,\n    verbose=True,\n)\n\n# El modelo entrenado se agrega a una lista fitted_models para su posterior uso\nfitted_models.append(xgb_model)\n\n# Se hace una predicción en el conjunto de validación utilizando el método \"predict_proba\", estas predicciones se utilizan para calcular el AUC\ny_pred_valid_k_fold = xgb_model.predict_proba(k_fold[1])[:, 1]\n\n# Se calcula comparando las predicciones del modelo con las etiquetas verdaderas utilizando la función \"roc_auc_score\"\nauc_k_fold = roc_auc_score(k_fold[3], y_pred_valid_k_fold)\n\n# Agrega el AUC calculado al final de la lista \"auc_scores\". Esto permite rastrear los puntajes AUC de todos los pliegues de la validación cruzada K-fold\nauc_scores.append(auc_k_fold)\n","metadata":{"id":"svHN7Z7EOIMu","outputId":"46f29b9c-56f4-48f1-8bb1-789973000cc2","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtrain = xgb.DMatrix(data=X_train, label=y_train, enable_categorical=True)\ndvalid = xgb.DMatrix(data=X_valid, label=y_valid, enable_categorical=True)\nparameters = {'objective': 'binary:logistic',\n              'learning_rate': 0.05,\n              'max_depth': 4,\n              'min_child_weight': 0.5,\n              'reg_alpha': 1,\n              'reg_lambda': 1,\n              'eval_metric': 'auc'}\n\n#watch_list  = [(dtrain,'train'),(dtest,'test')]\nwatch_list  = [(dtrain,'train'),(dvalid,'valid')]\n\nxgb_fit = xgb.train(params = parameters, dtrain = dtrain, evals = watch_list,\n                    num_boost_round = 500, early_stopping_rounds=50, verbose_eval=1)","metadata":{"id":"vKa4v8Ge6nTS","outputId":"171396b0-57c6-4aa1-a69a-05f2329d1b13","execution":{"iopub.status.busy":"2024-03-19T04:35:15.547527Z","iopub.execute_input":"2024-03-19T04:35:15.547989Z","iopub.status.idle":"2024-03-19T04:39:45.110816Z","shell.execute_reply.started":"2024-03-19T04:35:15.547954Z","shell.execute_reply":"2024-03-19T04:39:45.109370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PASO 12 - EVALUANDO LA DATA DE TEST Y HACIENDO LA SUBMISSION","metadata":{}},{"cell_type":"markdown","source":"## XGBOOST","metadata":{}},{"cell_type":"code","source":"for base, X in [(base_train, X_train), (base_valid, X_valid), (base_test, X_test)]:\n    datamatrix = xgb.DMatrix(data=X, enable_categorical=True)\n    y_pred = xgb_fit.predict(datamatrix)\n    base[\"score\"] = y_pred\n\nprint(f'The AUC score on the train set is: {roc_auc_score(base_train[\"target\"], base_train[\"score\"])}') \nprint(f'The AUC score on the valid set is: {roc_auc_score(base_valid[\"target\"], base_valid[\"score\"])}') \nprint(f'The AUC score on the test set is: {roc_auc_score(base_test[\"target\"], base_test[\"score\"])}')","metadata":{"execution":{"iopub.status.busy":"2024-03-19T04:41:08.396154Z","iopub.execute_input":"2024-03-19T04:41:08.396663Z","iopub.status.idle":"2024-03-19T04:41:45.413478Z","shell.execute_reply.started":"2024-03-19T04:41:08.396628Z","shell.execute_reply":"2024-03-19T04:41:45.411932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n\n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\nstability_score_train = gini_stability(base_train)\nstability_score_valid = gini_stability(base_valid)\nstability_score_test = gini_stability(base_test)\n\nprint(f'The stability score on the train set is: {stability_score_train}')\nprint(f'The stability score on the valid set is: {stability_score_valid}')\nprint(f'The stability score on the test set is: {stability_score_test}')","metadata":{"papermill":{"duration":1.131134,"end_time":"2024-02-07T21:30:16.624526","exception":false,"start_time":"2024-02-07T21:30:15.493392","status":"completed"},"tags":[],"id":"2e10914f","execution":{"iopub.status.busy":"2024-03-19T04:45:06.291792Z","iopub.execute_input":"2024-03-19T04:45:06.292275Z","iopub.status.idle":"2024-03-19T04:45:07.527506Z","shell.execute_reply.started":"2024-03-19T04:45:06.292242Z","shell.execute_reply":"2024-03-19T04:45:07.526008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_submission = data_submission[cols_pred].to_pandas()\nX_submission = convert_strings(X_submission)\ncategorical_cols = X_train.select_dtypes(include=['category']).columns\n\nfor col in categorical_cols:\n    train_categories = set(X_train[col].cat.categories)\n    submission_categories = set(X_submission[col].cat.categories)\n    new_categories = submission_categories - train_categories\n    X_submission.loc[X_submission[col].isin(new_categories), col] = \"Unknown\"\n    new_dtype = pd.CategoricalDtype(categories=train_categories, ordered=True)\n    X_train[col] = X_train[col].astype(new_dtype)\n    X_submission[col] = X_submission[col].astype(new_dtype)\n\ny_submission_pred = gbm.predict(X_submission, num_iteration=gbm.best_iteration)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T04:31:25.504967Z","iopub.execute_input":"2024-03-19T04:31:25.505738Z","iopub.status.idle":"2024-03-19T04:31:25.608477Z","shell.execute_reply.started":"2024-03-19T04:31:25.505694Z","shell.execute_reply":"2024-03-19T04:31:25.607111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": data_submission[\"case_id\"].to_numpy(),\n    \"score\": y_submission_pred\n}).set_index('case_id')\nsubmission.to_csv(\"./submission_01.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-03-19T04:46:21.413481Z","iopub.execute_input":"2024-03-19T04:46:21.413898Z","iopub.status.idle":"2024-03-19T04:46:21.427109Z","shell.execute_reply.started":"2024-03-19T04:46:21.413850Z","shell.execute_reply":"2024-03-19T04:46:21.425019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LIGHTGBM","metadata":{}},{"cell_type":"code","source":"# Evaluaciòn : Entrenamiento (base_train), validación (base_valid), y prueba (base_test).\n\nfor base, X in [(base_train, X_train), (base_valid, X_valid), (base_test, X_test)]:  # Iterar en cada conjunto de datos de entrenamiento, validación y prueba\n    y_pred = gbm.predict(X, num_iteration=gbm.best_iteration)                        # Realizar predicciones utilizando el modelo LightGBM\n    base[\"score\"] = y_pred                                                           # Agregar las predicciones como una nueva columna llamada \"score\" en el DataFrame base\n\nprint(f'The AUC score on the train set is: {roc_auc_score(base_train[\"target\"], base_train[\"score\"])}')  # Imprimir el AUC score para el conjunto de datos de entrenamiento\nprint(f'The AUC score on the valid set is: {roc_auc_score(base_valid[\"target\"], base_valid[\"score\"])}')  # Imprimir el AUC score para el conjunto de datos de validación\nprint(f'The AUC score on the test set is: {roc_auc_score(base_test[\"target\"], base_test[\"score\"])}')     # Imprimir el AUC score para el conjunto de datos de prueba","metadata":{"papermill":{"duration":22.617002,"end_time":"2024-02-07T21:30:15.484653","exception":false,"start_time":"2024-02-07T21:29:52.867651","status":"completed"},"tags":[],"id":"3e5c4fdb","outputId":"2e13a1f2-4018-4994-95af-63f5d5695fea","execution":{"iopub.status.busy":"2024-03-19T04:30:50.396559Z","iopub.execute_input":"2024-03-19T04:30:50.397215Z","iopub.status.idle":"2024-03-19T04:31:09.474559Z","shell.execute_reply.started":"2024-03-19T04:30:50.397024Z","shell.execute_reply":"2024-03-19T04:31:09.473148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n\n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\nstability_score_train = gini_stability(base_train)\nstability_score_valid = gini_stability(base_valid)\nstability_score_test = gini_stability(base_test)\n\nprint(f'The stability score on the train set is: {stability_score_train}')\nprint(f'The stability score on the valid set is: {stability_score_valid}')\nprint(f'The stability score on the test set is: {stability_score_test}')","metadata":{"execution":{"iopub.status.busy":"2024-03-19T05:00:17.181960Z","iopub.execute_input":"2024-03-19T05:00:17.182616Z","iopub.status.idle":"2024-03-19T05:00:18.458424Z","shell.execute_reply.started":"2024-03-19T05:00:17.182574Z","shell.execute_reply":"2024-03-19T05:00:18.456974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_submission = data_submission[cols_pred].to_pandas()\nX_submission = convert_strings(X_submission)\ncategorical_cols = X_train.select_dtypes(include=['category']).columns\n\nfor col in categorical_cols:\n    train_categories = set(X_train[col].cat.categories)\n    submission_categories = set(X_submission[col].cat.categories)\n    new_categories = submission_categories - train_categories\n    X_submission.loc[X_submission[col].isin(new_categories), col] = \"Unknown\"\n    new_dtype = pd.CategoricalDtype(categories=train_categories, ordered=True)\n    X_train[col] = X_train[col].astype(new_dtype)\n    X_submission[col] = X_submission[col].astype(new_dtype)\n\ndatamatrix = xgb.DMatrix(data=X_submission, enable_categorical=True)\ny_submission_pred = xgb_fit.predict(datamatrix)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T04:45:41.023047Z","iopub.execute_input":"2024-03-19T04:45:41.024331Z","iopub.status.idle":"2024-03-19T04:45:41.130970Z","shell.execute_reply.started":"2024-03-19T04:45:41.024275Z","shell.execute_reply":"2024-03-19T04:45:41.128925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": data_submission[\"case_id\"].to_numpy(),\n    \"score\": y_submission_pred\n}).set_index('case_id')\nsubmission.to_csv(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-03-19T04:31:36.066041Z","iopub.execute_input":"2024-03-19T04:31:36.066548Z","iopub.status.idle":"2024-03-19T04:31:36.075330Z","shell.execute_reply.started":"2024-03-19T04:31:36.066512Z","shell.execute_reply":"2024-03-19T04:31:36.074253Z"},"trusted":true},"execution_count":null,"outputs":[]}]}