{"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **STEP 1 - IMPORTAR LIBRERIAS**","metadata":{"id":"UqPEdBDLjtcd"}},{"cell_type":"code","source":"import polars as pl                # Importa la librería Polars\nimport numpy as np                 # Importa la librería numpy\nimport pandas as pd                # Importa la librería Pandas\nimport matplotlib.pyplot as plt    # Importa la librería matplotlib\nimport lightgbm as lgb             # Importa la librería LightGBM\n\n# Importa la función train_test_split del módulo model_selection de la librería scikit-learn.\n# Esta función se utiliza para dividir un conjunto de datos en conjuntos de entrenamiento y prueba.\nfrom sklearn.model_selection import train_test_split, cross_val_score   # cross_val_score Delete\n\n# Importa la función roc_auc_score del módulo metrics de la librería scikit-learn.\n# Esta función se utiliza para calcular el área bajo la curva ROC (AUC) para problemas de clasificación binaria.\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import KFold\n\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n","metadata":{"id":"URUIm3T2GfoI","execution":{"iopub.status.busy":"2024-03-19T04:00:41.370415Z","iopub.execute_input":"2024-03-19T04:00:41.370759Z","iopub.status.idle":"2024-03-19T04:00:43.453912Z","shell.execute_reply.started":"2024-03-19T04:00:41.370728Z","shell.execute_reply":"2024-03-19T04:00:43.452678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STEP 2 - CREAR FUNCIONES**","metadata":{"id":"vcRnv6SYkAV_"}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n# función set_table_dtypes que toma un DataFrame de Polars (pl.DataFrame) como entrada y devuelve un DataFrame de Polars también\n\n    for col in df.columns:\n        # la última letra del nombre de la columna determinará el tipo\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n\n    return df\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n# convertir las columnas de tipo 'object' o 'string' de un DataFrame de Pandas a tipo 'category', añadiendo la categoría \"Unknown\" si es necesario, y devuelve el DataFrame modificado.\n\n    for col in df.columns:\n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return df","metadata":{"id":"hxSZTy6JG-OG","execution":{"iopub.status.busy":"2024-03-19T04:00:43.456327Z","iopub.execute_input":"2024-03-19T04:00:43.456730Z","iopub.status.idle":"2024-03-19T04:00:43.464755Z","shell.execute_reply.started":"2024-03-19T04:00:43.456695Z","shell.execute_reply":"2024-03-19T04:00:43.463571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STEP 3 - CARGA DE ARCHIVOS CSV - TRAIN SET**","metadata":{"id":"8V2xglnIkPPl"}},{"cell_type":"code","source":"#carga un archivo CSV llamado \"train_base.csv\" ubicado en la ruta especificada dataPath\ntrain_basetable = pl.read_csv(dataPath + \"csv_files/train/train_base.csv\")\n\n# crear un DataFrame de Polars llamado train_static concatenando dos DataFrames previamente leídos desde archivos CSV.\n# .pipe(set_table_dtypes): Llama a la función set_table_dtypes definida anteriormente, que se aplica al DataFrame recién creado para ajustar los tipos de datos según ciertos criterios.\n\ntrain_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_1.csv\").pipe(set_table_dtypes),\n    ],\n\n# El parámetro how=\"vertical_relaxed\" indica que la concatenación se realizará verticalmente, es decir, uno encima del otro, y se permite una relajación de la alineación de columnas.\n    how=\"vertical_relaxed\",\n)\n\n#carga un archivo CSV llamado \"train_static_cb_0.csv\"\ntrain_static_cb = pl.read_csv(dataPath + \"csv_files/train/train_static_cb_0.csv\").pipe(set_table_dtypes)\n\n#carga un archivo CSV llamado \"train_person_1.csv\"\ntrain_person_1 = pl.read_csv(dataPath + \"csv_files/train/train_person_1.csv\").pipe(set_table_dtypes)\n\n#carga un archivo CSV llamado \"train_credit_bureau_b_2.csv\"\ntrain_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_b_2.csv\").pipe(set_table_dtypes)","metadata":{"id":"Cmwkv8v5HFJO","execution":{"iopub.status.busy":"2024-03-19T04:00:43.466041Z","iopub.execute_input":"2024-03-19T04:00:43.466483Z","iopub.status.idle":"2024-03-19T04:00:55.891733Z","shell.execute_reply.started":"2024-03-19T04:00:43.466451Z","shell.execute_reply":"2024-03-19T04:00:55.890992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STEP 4 - CARGA DE ARCHIVOS CSV - TEST SET**","metadata":{"id":"Hdd4X7N9k4Vs"}},{"cell_type":"code","source":"#carga un archivo CSV llamado \"test_base.csv\" ubicado en la ruta especificada dataPath\ntest_basetable = pl.read_csv(dataPath + \"csv_files/test/test_base.csv\")\n\n# crear un DataFrame de Polars llamado test_static concatenando tres DataFrames previamente leídos desde archivos CSV.\n# .pipe(set_table_dtypes): Llama a la función set_table_dtypes, que se aplica al DataFrame recién creado para ajustar los tipos de datos según ciertos criterios.\ntest_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes),\n    ],\n# El parámetro how=\"vertical_relaxed\" indica que la concatenación se realizará verticalmente, es decir, uno encima del otro, y se permite una relajación de la alineación de columnas.\n    how=\"vertical_relaxed\",\n)\n\n#carga un archivo CSV llamado \"test_static_cb_0.csv\"\ntest_static_cb = pl.read_csv(dataPath + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\n\n#carga un archivo CSV llamado \"test_person_1.csv\"\ntest_person_1 = pl.read_csv(dataPath + \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes)\n\n#carga un archivo CSV llamado \"test_credit_bureau_b_2.csv\"\ntest_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes)","metadata":{"id":"skrww3HYlg6N","execution":{"iopub.status.busy":"2024-03-19T04:00:55.892523Z","iopub.execute_input":"2024-03-19T04:00:55.892768Z","iopub.status.idle":"2024-03-19T04:00:55.984573Z","shell.execute_reply.started":"2024-03-19T04:00:55.892745Z","shell.execute_reply":"2024-03-19T04:00:55.983729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STEP 5 - ANALISIS EXPLORATORIO - TRAIN SET**\n","metadata":{"id":"K_7JIee6ltUt"}},{"cell_type":"code","source":"train_basetable.head()\n","metadata":{"id":"qbq4_7AUHQvv","outputId":"e9764461-acab-4505-b324-e3f412328631","execution":{"iopub.status.busy":"2024-03-19T04:00:55.986748Z","iopub.execute_input":"2024-03-19T04:00:55.987020Z","iopub.status.idle":"2024-03-19T04:00:56.002439Z","shell.execute_reply.started":"2024-03-19T04:00:55.986996Z","shell.execute_reply":"2024-03-19T04:00:56.001121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable.describe()","metadata":{"id":"e2U6UyoWe8Za","outputId":"2a34a812-437c-4362-d371-2bddf44a5851","execution":{"iopub.status.busy":"2024-03-19T04:00:56.003674Z","iopub.execute_input":"2024-03-19T04:00:56.003950Z","iopub.status.idle":"2024-03-19T04:00:56.106256Z","shell.execute_reply.started":"2024-03-19T04:00:56.003912Z","shell.execute_reply":"2024-03-19T04:00:56.105678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convierte el DataFrame train_basetable de Polars a un DataFrame de Pandas\ntrain_basetable_pd = train_basetable.to_pandas()\n\n# Crea un conjunto (set) de los valores únicos de la columna target\nset_target = set(train_basetable_pd['target'].to_list())\nprint(set_target)\n\n# Graficar los datos\nplt.figure(figsize=(10, 6))\ntrain_basetable_pd.hist(bins=20)\nplt.tight_layout()\n","metadata":{"id":"dJyd3h4sealy","outputId":"05c222db-1870-4ed1-b241-3a35b688b9ac","execution":{"iopub.status.busy":"2024-03-19T04:00:56.107232Z","iopub.execute_input":"2024-03-19T04:00:56.107590Z","iopub.status.idle":"2024-03-19T04:00:57.128861Z","shell.execute_reply.started":"2024-03-19T04:00:56.107566Z","shell.execute_reply":"2024-03-19T04:00:57.128020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tipo de objeto (train_basetable)\nprint(type(train_basetable))","metadata":{"id":"UfiOJvSuPISD","outputId":"6403f405-c349-4fd6-caa6-fedfab76f304","execution":{"iopub.status.busy":"2024-03-19T04:00:57.129927Z","iopub.execute_input":"2024-03-19T04:00:57.130218Z","iopub.status.idle":"2024-03-19T04:00:57.134720Z","shell.execute_reply.started":"2024-03-19T04:00:57.130191Z","shell.execute_reply":"2024-03-19T04:00:57.133768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Listar nombres de todas las columnas del DataFrame train_basetable.\ntrain_basetable.columns","metadata":{"id":"ttMhxepMP7gc","outputId":"481f88aa-5bb8-4304-87c4-85a658e9c3f1","execution":{"iopub.status.busy":"2024-03-19T04:00:57.136033Z","iopub.execute_input":"2024-03-19T04:00:57.136303Z","iopub.status.idle":"2024-03-19T04:00:57.150073Z","shell.execute_reply.started":"2024-03-19T04:00:57.136280Z","shell.execute_reply":"2024-03-19T04:00:57.149200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dimensiones del DataFrame train_basetable (filas, columnas)\ntrain_basetable.shape","metadata":{"id":"o9Fj2KFjQekU","outputId":"e5745478-97af-4b52-b357-79651f32922d","execution":{"iopub.status.busy":"2024-03-19T04:00:57.151547Z","iopub.execute_input":"2024-03-19T04:00:57.151809Z","iopub.status.idle":"2024-03-19T04:00:57.167284Z","shell.execute_reply.started":"2024-03-19T04:00:57.151784Z","shell.execute_reply":"2024-03-19T04:00:57.166327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static","metadata":{"id":"22NyjPt_t4tF","outputId":"9abfc388-0e7f-4148-d5dc-53e6043671e2","execution":{"iopub.status.busy":"2024-03-19T04:00:57.168449Z","iopub.execute_input":"2024-03-19T04:00:57.168767Z","iopub.status.idle":"2024-03-19T04:00:57.215617Z","shell.execute_reply.started":"2024-03-19T04:00:57.168735Z","shell.execute_reply":"2024-03-19T04:00:57.214741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convierte el DataFrame train_static de Polars a un DataFrame de Pandas\ntrain_static_pd = train_static.to_pandas()\n\n# Crea un conjunto (set) de los valores únicos de la columna target\n#set_target = set(train_basetable_pd['amtinstpaidbefduel24m_4187115A'].to_list())\n#print(set_target)\n\n# Graficar los datos\n#plt.figure(figsize=(10, 6))\n#train_static_pd.hist(bins=20)\n#plt.tight_layout()\n","metadata":{"id":"DKHVe-eNmbhl","execution":{"iopub.status.busy":"2024-03-19T04:00:57.216743Z","iopub.execute_input":"2024-03-19T04:00:57.217636Z","iopub.status.idle":"2024-03-19T04:00:58.872531Z","shell.execute_reply.started":"2024-03-19T04:00:57.217600Z","shell.execute_reply":"2024-03-19T04:00:58.871655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_cb","metadata":{"id":"KIN7t0vAvyp-","outputId":"d4c473cb-7b5c-449a-e484-0607a6244f53","execution":{"iopub.status.busy":"2024-03-19T04:00:58.873577Z","iopub.execute_input":"2024-03-19T04:00:58.873854Z","iopub.status.idle":"2024-03-19T04:00:58.902180Z","shell.execute_reply.started":"2024-03-19T04:00:58.873825Z","shell.execute_reply":"2024-03-19T04:00:58.901279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1","metadata":{"id":"AUMz9KmWwRte","outputId":"ad4d6641-a287-43b4-db9b-a9fdab22608f","execution":{"iopub.status.busy":"2024-03-19T04:00:58.906001Z","iopub.execute_input":"2024-03-19T04:00:58.906283Z","iopub.status.idle":"2024-03-19T04:00:58.926367Z","shell.execute_reply.started":"2024-03-19T04:00:58.906258Z","shell.execute_reply":"2024-03-19T04:00:58.925598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_b_2","metadata":{"id":"uU9AVhEYwd8F","outputId":"52f31f5d-944e-4d0e-a288-d061cbb27f4a","execution":{"iopub.status.busy":"2024-03-19T04:00:58.927693Z","iopub.execute_input":"2024-03-19T04:00:58.928080Z","iopub.status.idle":"2024-03-19T04:00:58.937064Z","shell.execute_reply.started":"2024-03-19T04:00:58.928037Z","shell.execute_reply":"2024-03-19T04:00:58.936089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STEP 6 - ANALISIS EXPLORATORIO - TEST SET**\n","metadata":{"id":"WfXDEktnmwl0"}},{"cell_type":"code","source":"test_static","metadata":{"id":"KwBGhLMLODPM","outputId":"418167f3-32b6-428b-dd40-96a9c2ced4c1","execution":{"iopub.status.busy":"2024-03-19T04:00:58.938222Z","iopub.execute_input":"2024-03-19T04:00:58.938451Z","iopub.status.idle":"2024-03-19T04:00:58.975269Z","shell.execute_reply.started":"2024-03-19T04:00:58.938429Z","shell.execute_reply":"2024-03-19T04:00:58.974265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_static_cb","metadata":{"id":"vExH-fxxtD2r","outputId":"e367a3da-836f-41ff-ec41-e00ea3708fa7","execution":{"iopub.status.busy":"2024-03-19T04:00:58.976469Z","iopub.execute_input":"2024-03-19T04:00:58.976733Z","iopub.status.idle":"2024-03-19T04:00:58.989903Z","shell.execute_reply.started":"2024-03-19T04:00:58.976707Z","shell.execute_reply":"2024-03-19T04:00:58.989302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_person_1","metadata":{"id":"gac-R-WetNnq","outputId":"4c3c5f4d-d639-4a8c-a73b-e1c6f95e2d9a","execution":{"iopub.status.busy":"2024-03-19T04:00:58.990957Z","iopub.execute_input":"2024-03-19T04:00:58.992482Z","iopub.status.idle":"2024-03-19T04:00:59.005685Z","shell.execute_reply.started":"2024-03-19T04:00:58.992448Z","shell.execute_reply":"2024-03-19T04:00:59.004782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_b_2","metadata":{"id":"Rk4oglGitTY4","outputId":"e33214e8-0d83-4499-eca4-205dc02abbe0","execution":{"iopub.status.busy":"2024-03-19T04:00:59.006717Z","iopub.execute_input":"2024-03-19T04:00:59.006979Z","iopub.status.idle":"2024-03-19T04:00:59.019479Z","shell.execute_reply.started":"2024-03-19T04:00:59.006955Z","shell.execute_reply":"2024-03-19T04:00:59.018643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable","metadata":{"id":"fo8f73ZEOo5U","outputId":"784abda5-618e-4a49-8105-3caea0420b3e","execution":{"iopub.status.busy":"2024-03-19T04:00:59.020626Z","iopub.execute_input":"2024-03-19T04:00:59.020947Z","iopub.status.idle":"2024-03-19T04:00:59.031440Z","shell.execute_reply.started":"2024-03-19T04:00:59.020895Z","shell.execute_reply":"2024-03-19T04:00:59.030235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STEP 7 - FEATURE ENGINEERING**","metadata":{"id":"1da-F-i8nrhz"}},{"cell_type":"code","source":"# Operación de agrupación y agregación en un DataFrame de Polars (train_person_1).\ntrain_person_1_feats_1 = train_person_1.group_by(\"case_id\").agg(                      # Agrupar todas las filas que tengan el mismo valor en la columna \"case_id\".\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),       # Calcula el valor máximo de la columna \"mainoccupationinc_384A\" para cada grupo y se le asigna un alias \"mainoccupationinc_384A_max\n\n# Verifica si algún valor en la columna \"incometype_1044T\" dentro de cada grupo es igual a \"SELFEMPLOYED\".\n# El resultado se almacena en la columna \"mainoccupationinc_384A_any_selfemployed\", indicando si en algún momento del grupo se registra \"SELFEMPLOYED\".\n      (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)","metadata":{"id":"A5SXPboIUsrk","execution":{"iopub.status.busy":"2024-03-19T04:00:59.033267Z","iopub.execute_input":"2024-03-19T04:00:59.034178Z","iopub.status.idle":"2024-03-19T04:00:59.254776Z","shell.execute_reply.started":"2024-03-19T04:00:59.034125Z","shell.execute_reply":"2024-03-19T04:00:59.254105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1_feats_1","metadata":{"id":"SBmxlhkAU8BM","outputId":"51b5cae6-f416-4a85-d4dd-2cdfa4ad573e","execution":{"iopub.status.busy":"2024-03-19T04:00:59.255496Z","iopub.execute_input":"2024-03-19T04:00:59.255722Z","iopub.status.idle":"2024-03-19T04:00:59.262351Z","shell.execute_reply.started":"2024-03-19T04:00:59.255700Z","shell.execute_reply":"2024-03-19T04:00:59.261750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# num_group1 = 0 es la persona que solicitó el préstamo.\n\n# Selecciona las columnas \"case_id\", \"num_group1\" y \"housetype_905L\" del DataFrame train_person_1 y filtra las filas donde el valor en la columna \"num_group1\" es igual a 0.\n# .drop(\"num_group1\"): Elimina la columna \"num_group1\" del DataFrame resultante. Esto se hace después de haber filtrado las filas donde num_group1 es igual a 0, por lo que esta columna ya no es necesaria.\n# .rename({\"housetype_905L\": \"person_housetype\"}): Renombra la columna \"housetype_905L\" a \"person_housetype\".\n\ntrain_person_1_feats_2 = train_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\n","metadata":{"id":"ahGNRVocVMxD","execution":{"iopub.status.busy":"2024-03-19T04:00:59.263653Z","iopub.execute_input":"2024-03-19T04:00:59.264133Z","iopub.status.idle":"2024-03-19T04:00:59.321807Z","shell.execute_reply.started":"2024-03-19T04:00:59.264065Z","shell.execute_reply":"2024-03-19T04:00:59.321112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1_feats_2","metadata":{"id":"Z5eP95rmVWac","outputId":"57efc739-e101-4fe3-979b-9703dd3e1963","execution":{"iopub.status.busy":"2024-03-19T04:00:59.322557Z","iopub.execute_input":"2024-03-19T04:00:59.322795Z","iopub.status.idle":"2024-03-19T04:00:59.330660Z","shell.execute_reply.started":"2024-03-19T04:00:59.322772Z","shell.execute_reply":"2024-03-19T04:00:59.329190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Aquí tenemos num_goup1 y num_group2, por lo que necesitamos agregar nuevamente.\n# Agrupar todas las filas que tengan el mismo valor en la columna \"case_id\".\n\ntrain_credit_bureau_b_2_feats = train_credit_bureau_b_2.group_by(\"case_id\").agg(\n\n# Calcula el valor máximo de la columna \"pmts_pmtsoverdue_635A\" para cada grupo y se le asigna un alias \"pmts_pmtsoverdue_635A_max\"\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n\n# Verifica si algún valor en la columna \"pmts_dpdvalue_108P\" es mayor que 31. El resultado se almacena en la columna \"pmts_dpdvalue_108P_over31\"\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)","metadata":{"id":"4NuqWF8eVdIT","execution":{"iopub.status.busy":"2024-03-19T04:00:59.331915Z","iopub.execute_input":"2024-03-19T04:00:59.332684Z","iopub.status.idle":"2024-03-19T04:00:59.365991Z","shell.execute_reply.started":"2024-03-19T04:00:59.332651Z","shell.execute_reply":"2024-03-19T04:00:59.365217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_b_2_feats","metadata":{"id":"6esJzDhxVi1r","outputId":"445d50b3-e4f5-4970-a36d-d57478e1e3d2","execution":{"iopub.status.busy":"2024-03-19T04:00:59.370018Z","iopub.execute_input":"2024-03-19T04:00:59.370425Z","iopub.status.idle":"2024-03-19T04:00:59.379609Z","shell.execute_reply.started":"2024-03-19T04:00:59.370384Z","shell.execute_reply":"2024-03-19T04:00:59.378367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# selecciona solo las columnas del DataFrame train_static que terminan en \"A\" o \"M\".\n\nselected_static_cols = []                        # crea una lista vacía que se utilizará para almacenar los nombres de las columnas seleccionadas del DataFrame train_static\nfor col in train_static.columns:                 # Ciclo for que itera a través de todas las columnas del DataFrame train_static.\n    if col[-1] in (\"A\", \"M\"):                    # verifica si el último carácter del nombre de la columna, es igual a \"A\" o \"M\"\n        selected_static_cols.append(col)         # Si la condición anterior se cumple, se agrega el nombre de la columna a la lista selected_static_cols\nprint(selected_static_cols)                      # Se imprime la lista selected_static_cols, que contiene los nombres de las columnas seleccionadas.","metadata":{"id":"vGXoOKXDWBET","outputId":"fa32a32a-335f-46fd-ada5-4f546bf4722f","execution":{"iopub.status.busy":"2024-03-19T04:00:59.380801Z","iopub.execute_input":"2024-03-19T04:00:59.382815Z","iopub.status.idle":"2024-03-19T04:00:59.393911Z","shell.execute_reply.started":"2024-03-19T04:00:59.382784Z","shell.execute_reply":"2024-03-19T04:00:59.393174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_static_cb_cols = []                     # Crea una lista vacía para almacenar los nombres de las columnas seleccionadas del DataFrame train_static_cb\nfor col in train_static_cb.columns:              # Ciclo for que itera a través de todas las columnas del DataFrame train_static_cb\n    if col[-1] in (\"A\", \"M\"):                    # Verifica si el último carácter del nombre de la columna, es igual a \"A\" o \"M\"\n        selected_static_cb_cols.append(col)      # Si la condición anterior se cumple, se agrega el nombre de la columna a la lista selected_static_cb_cols\nprint(selected_static_cb_cols)                   # Se imprime la lista selected_static_cb_cols, que contiene los nombres de las columnas seleccionadas.","metadata":{"id":"eXHs-yxHWI0j","outputId":"848af7bf-eb29-4f65-bc0a-bc114684ce3d","execution":{"iopub.status.busy":"2024-03-19T04:00:59.394814Z","iopub.execute_input":"2024-03-19T04:00:59.395328Z","iopub.status.idle":"2024-03-19T04:00:59.410821Z","shell.execute_reply.started":"2024-03-19T04:00:59.395298Z","shell.execute_reply":"2024-03-19T04:00:59.409885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Unir todas la tablas\n# Combinar datos de múltiples DataFrames en un solo DataFrame llamado data, utilizando la columna \"case_id\" como clave de unión.\n# La unión izquierda garantiza que todas las filas del DataFrame train_basetable se conserven en el DataFrame resultante, y las filas correspondientes\n# de los otros DataFrames se agregan si hay una coincidencia en la columna \"case_id\".\n\ndata = (\n    train_basetable\n    .join(train_static.select([\"case_id\"] + selected_static_cols), how=\"left\", on=\"case_id\")\n    .join(train_static_cb.select([\"case_id\"] + selected_static_cb_cols), how=\"left\", on=\"case_id\")\n    .join(train_person_1_feats_1, how=\"left\", on=\"case_id\")\n    .join(train_person_1_feats_2, how=\"left\", on=\"case_id\")\n    .join(train_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\")\n)\n\n# Reemplazar los valores nulos por la moda en todas las columnas\nfor col in data.columns:\n    data = data.with_columns(\n        pl.col(col).fill_null(pl.lit(data[col].mode()))\n    )","metadata":{"papermill":{"duration":1.664029,"end_time":"2024-02-07T21:28:22.750906","exception":false,"start_time":"2024-02-07T21:28:21.086877","status":"completed"},"tags":[],"id":"eed7be3e","execution":{"iopub.status.busy":"2024-03-19T04:00:59.412107Z","iopub.execute_input":"2024-03-19T04:00:59.412647Z","iopub.status.idle":"2024-03-19T04:01:03.395040Z","shell.execute_reply.started":"2024-03-19T04:00:59.412619Z","shell.execute_reply":"2024-03-19T04:01:03.394337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.describe()","metadata":{"id":"gAo0OgsSuJKD","outputId":"a536fca5-e9ca-4398-a07c-2f41c14df5ab","execution":{"iopub.status.busy":"2024-03-19T04:01:03.396287Z","iopub.execute_input":"2024-03-19T04:01:03.396592Z","iopub.status.idle":"2024-03-19T04:01:05.112070Z","shell.execute_reply.started":"2024-03-19T04:01:03.396563Z","shell.execute_reply":"2024-03-19T04:01:05.111450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convertir el DataFrame de Polars a un DataFrame de Pandas\ndata_pd = data.to_pandas()\n\n# Graficar los datos\nplt.boxplot(data_pd[\"annuity_780A\"], vert = True)\nplt.show()","metadata":{"id":"dE8_OZvZcfv6","outputId":"24fd56ab-7922-4ced-e25f-a63ce316ac98","execution":{"iopub.status.busy":"2024-03-19T04:01:05.114018Z","iopub.execute_input":"2024-03-19T04:01:05.114352Z","iopub.status.idle":"2024-03-19T04:01:06.024582Z","shell.execute_reply.started":"2024-03-19T04:01:05.114322Z","shell.execute_reply":"2024-03-19T04:01:06.023193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.boxplot(data_pd[\"annuitynextmonth_57A\"], vert = True)\nplt.show()","metadata":{"id":"H8svdJan7yxV","outputId":"9b2393aa-b92f-41fd-9784-a5bbae2034ac","execution":{"iopub.status.busy":"2024-03-19T04:01:06.026265Z","iopub.execute_input":"2024-03-19T04:01:06.026604Z","iopub.status.idle":"2024-03-19T04:01:06.372878Z","shell.execute_reply.started":"2024-03-19T04:01:06.026572Z","shell.execute_reply":"2024-03-19T04:01:06.371808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.boxplot(data_pd[\"credamount_770A\"], vert = True)\nplt.show()","metadata":{"id":"mNQxk-FR80Ua","outputId":"5809feb3-f881-4339-f92f-7074c73429f3","execution":{"iopub.status.busy":"2024-03-19T04:01:06.374543Z","iopub.execute_input":"2024-03-19T04:01:06.374875Z","iopub.status.idle":"2024-03-19T04:01:06.657607Z","shell.execute_reply.started":"2024-03-19T04:01:06.374842Z","shell.execute_reply":"2024-03-19T04:01:06.656657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.describe()","metadata":{"id":"x2lAYY1kyGIN","outputId":"7651bf09-6fa7-45e2-995e-fa22a47b3783","execution":{"iopub.status.busy":"2024-03-19T04:01:06.658695Z","iopub.execute_input":"2024-03-19T04:01:06.658938Z","iopub.status.idle":"2024-03-19T04:01:08.397175Z","shell.execute_reply.started":"2024-03-19T04:01:06.658914Z","shell.execute_reply":"2024-03-19T04:01:08.395934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data)","metadata":{"id":"CHK-X_Vne5KT","outputId":"4d9ca537-528b-45d3-80c4-5e85cb8b27d9","execution":{"iopub.status.busy":"2024-03-19T04:01:08.398288Z","iopub.execute_input":"2024-03-19T04:01:08.398604Z","iopub.status.idle":"2024-03-19T04:01:08.404228Z","shell.execute_reply.started":"2024-03-19T04:01:08.398577Z","shell.execute_reply":"2024-03-19T04:01:08.402906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STEP 8 - EVALUACIÓN DE VALORES NULOS**","metadata":{"id":"Gaygxot8p9K0"}},{"cell_type":"code","source":"# Resumen estadístico del DataFrame data.\n# Calcular automáticamente estadísticas descriptivas para cada columna numérica en el DataFrame, como la cantidad de valores, la media, la desviación estándar, los valores mínimo y máximo, etc.\n\ndata.describe()","metadata":{"id":"2rseULVeuY7M","outputId":"09fafaa0-9965-4bbf-f505-fa1abcd6a85f","execution":{"iopub.status.busy":"2024-03-19T04:01:08.405336Z","iopub.execute_input":"2024-03-19T04:01:08.405726Z","iopub.status.idle":"2024-03-19T04:01:10.185027Z","shell.execute_reply.started":"2024-03-19T04:01:08.405692Z","shell.execute_reply":"2024-03-19T04:01:10.184382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.select([\"lastotherinc_902A\", \"lastotherlnsexpense_631A\"])","metadata":{"id":"DwIaoOTbqsr0","outputId":"489ab68a-b310-4618-b10d-7bf553030240","execution":{"iopub.status.busy":"2024-03-19T04:01:10.186011Z","iopub.execute_input":"2024-03-19T04:01:10.186806Z","iopub.status.idle":"2024-03-19T04:01:10.192916Z","shell.execute_reply.started":"2024-03-19T04:01:10.186777Z","shell.execute_reply":"2024-03-19T04:01:10.192305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.select([\"lastotherinc_902A\", \"lastotherlnsexpense_631A\"]).describe()","metadata":{"id":"FxJ8qDsqs9Ob","outputId":"59f7c6e2-cdb2-4a11-f113-82eacfeb43cb","execution":{"iopub.status.busy":"2024-03-19T04:01:10.193806Z","iopub.execute_input":"2024-03-19T04:01:10.194755Z","iopub.status.idle":"2024-03-19T04:01:10.229875Z","shell.execute_reply.started":"2024-03-19T04:01:10.194730Z","shell.execute_reply":"2024-03-19T04:01:10.229109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data_bak = data.clone()\n#data = data.drop(['lastotherinc_902A', 'lastotherlnsexpense_631A',])","metadata":{"id":"y-GwFvIPytDr","execution":{"iopub.status.busy":"2024-03-19T04:01:10.230887Z","iopub.execute_input":"2024-03-19T04:01:10.231118Z","iopub.status.idle":"2024-03-19T04:01:10.234644Z","shell.execute_reply.started":"2024-03-19T04:01:10.231095Z","shell.execute_reply":"2024-03-19T04:01:10.233897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape","metadata":{"id":"Yy9PK22L2TST","outputId":"5ca930ec-d898-4be9-d85d-fd83695e61c7","execution":{"iopub.status.busy":"2024-03-19T04:01:10.235624Z","iopub.execute_input":"2024-03-19T04:01:10.236009Z","iopub.status.idle":"2024-03-19T04:01:10.250021Z","shell.execute_reply.started":"2024-03-19T04:01:10.235986Z","shell.execute_reply":"2024-03-19T04:01:10.249188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # **STEP 9 - CONTRUYENDO DATA_SUBMISSION (FINAL) PARA TRAIN / TEST / VALID**","metadata":{"id":"xm6dRerGwB-z"}},{"cell_type":"code","source":"# Procesamiento de datos y creación del DataFrame final data_submission:\n# Agrupar los datos del DataFrame test_person_1 por la columna \"case_id\". calcular el valor máximo de la columna \"mainoccupationinc_384A\" para cada grupo de \"case_id\".\n# Verificar si el valor de la columna \"incometype_1044T\" es igual a \"SELFEMPLOYED\" para cada grupo. Los resultados se almacenan en un nuevo DataFrame llamado test_person_1_feats_1.\n\ntest_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\n# Seleccionar las columnas del DataFrame test_person_1 (\"case_id\", \"num_group1\", \"housetype_905L\") y filtrar las filas donde \"num_group1\" es igual a 0,\n# eliminar la columna \"num_group1\" y renombrar la columna \"housetype_905L\" como \"person_housetype\".\n\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\n# Agrupar los datos del DataFrame\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\n# Unir todos los DataFrames recién creados con el DataFrame test_basetable, test_static y test_static_cb utilizando las columnas \"case_id\" como claves de unión.\n# El resultado final es un DataFrame llamado data_submission, que contiene datos para su posterior uso, como la presentación de informes o la predicción del modelo\n\ndata_submission = test_basetable.join(\n    test_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n)\n\n#for col in data_submission.columns:\n #   data_submission = data_submission.with_columns(\n  #      pl.col(col).fill_null(pl.lit(data_submission[col].mode()))  # Obtener la moda y reemplazar los nulos\n   # )\n","metadata":{"papermill":{"duration":0.025248,"end_time":"2024-02-07T21:28:22.783159","exception":false,"start_time":"2024-02-07T21:28:22.757911","status":"completed"},"tags":[],"id":"3cc4a29b","execution":{"iopub.status.busy":"2024-03-19T04:01:10.257155Z","iopub.execute_input":"2024-03-19T04:01:10.257733Z","iopub.status.idle":"2024-03-19T04:01:10.269673Z","shell.execute_reply.started":"2024-03-19T04:01:10.257695Z","shell.execute_reply":"2024-03-19T04:01:10.268771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submission.describe()","metadata":{"id":"JrlZmxLmxyu-","outputId":"faa856df-cfc0-45e8-fbfb-a51f6ddaaedb","execution":{"iopub.status.busy":"2024-03-19T04:01:10.270927Z","iopub.execute_input":"2024-03-19T04:01:10.271859Z","iopub.status.idle":"2024-03-19T04:01:10.301728Z","shell.execute_reply.started":"2024-03-19T04:01:10.271809Z","shell.execute_reply":"2024-03-19T04:01:10.300995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Seleccionar los IDs únicos de casos y dividirlos en conjuntos de entrenamiento, validación y prueba\ncase_ids = data[\"case_id\"].unique().shuffle(seed=1)\ncase_ids_train, case_ids_test = train_test_split(case_ids, train_size=0.6, random_state=1)\ncase_ids_valid, case_ids_test = train_test_split(case_ids_test, train_size=0.5, random_state=1)\n\n# Identificar las columnas predictoras\ncols_pred = []\nfor col in data.columns:\n    if col[-1].isupper() and col[:-1].islower():\n        cols_pred.append(col)\n\n# Imprimir las columnas predictoras seleccionadas\nprint(cols_pred)\n\n# Función para convertir conjuntos de datos de Polars a Pandas\ndef from_polars_to_pandas(case_ids: pl.DataFrame) -> pl.DataFrame:\n    return (\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[[\"case_id\", \"WEEK_NUM\", \"target\"]].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[cols_pred].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[\"target\"].to_pandas()\n    )\n\n# Obtener conjuntos de datos de entrenamiento, validación y prueba en formato Pandas\nbase_train, X_train, y_train = from_polars_to_pandas(case_ids_train)\nbase_valid, X_valid, y_valid = from_polars_to_pandas(case_ids_valid)\nbase_test, X_test, y_test = from_polars_to_pandas(case_ids_test)\n\n# Convertir las columnas de tipo 'object' o 'string' a tipo 'category' en todos los conjuntos de datos\nfor df in [X_train, X_valid, X_test]:\n    df = convert_strings(df)","metadata":{"papermill":{"duration":9.431294,"end_time":"2024-02-07T21:28:32.221469","exception":false,"start_time":"2024-02-07T21:28:22.790175","status":"completed"},"tags":[],"id":"7195e82a","outputId":"a178ff81-c13a-469a-e2af-026b8d6d07a2","execution":{"iopub.status.busy":"2024-03-19T04:01:10.302752Z","iopub.execute_input":"2024-03-19T04:01:10.303331Z","iopub.status.idle":"2024-03-19T04:01:15.115775Z","shell.execute_reply.started":"2024-03-19T04:01:10.303304Z","shell.execute_reply":"2024-03-19T04:01:15.114042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_train.shape                     # Forma del conjunto de datos (filas, columnas)","metadata":{"id":"UNQRGrbAg2yz","outputId":"954955aa-3d67-4a4c-a6ea-f25dba3bd062","execution":{"iopub.status.busy":"2024-03-19T04:01:15.117381Z","iopub.execute_input":"2024-03-19T04:01:15.117751Z","iopub.status.idle":"2024-03-19T04:01:15.124178Z","shell.execute_reply.started":"2024-03-19T04:01:15.117716Z","shell.execute_reply":"2024-03-19T04:01:15.123200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_train.info()                    # Resumen del DataFrame, incluyendo el tipo de datos de cada columna y la cantidad de valores no nulos","metadata":{"id":"wXg8R8uVgui8","outputId":"fa7c3600-d258-48d3-8101-2cdc0c7f9a0e","execution":{"iopub.status.busy":"2024-03-19T04:01:15.125424Z","iopub.execute_input":"2024-03-19T04:01:15.125733Z","iopub.status.idle":"2024-03-19T04:01:15.151300Z","shell.execute_reply.started":"2024-03-19T04:01:15.125701Z","shell.execute_reply":"2024-03-19T04:01:15.150235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_train                           # Imprimir base_train","metadata":{"id":"xSYiAomdg-g7","outputId":"284cbec9-e8b0-4fdc-a835-445b77436b33","execution":{"iopub.status.busy":"2024-03-19T04:01:15.152436Z","iopub.execute_input":"2024-03-19T04:01:15.152710Z","iopub.status.idle":"2024-03-19T04:01:15.168499Z","shell.execute_reply.started":"2024-03-19T04:01:15.152683Z","shell.execute_reply":"2024-03-19T04:01:15.167467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape                     # Forma del conjunto de datos (filas, columnas)","metadata":{"id":"wOh5SlqoyceV","outputId":"fa13a2ee-9e8d-4b4d-f049-f7bcfcb78158","execution":{"iopub.status.busy":"2024-03-19T04:01:15.171197Z","iopub.execute_input":"2024-03-19T04:01:15.171966Z","iopub.status.idle":"2024-03-19T04:01:15.179481Z","shell.execute_reply.started":"2024-03-19T04:01:15.171929Z","shell.execute_reply":"2024-03-19T04:01:15.177969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(X_train))              # Imprimir el tipo de datos de la variable X_train","metadata":{"id":"4BKsW6z3Yfl8","outputId":"eee80787-bfa9-471c-e11d-169e1a6751d2","execution":{"iopub.status.busy":"2024-03-19T04:01:15.180610Z","iopub.execute_input":"2024-03-19T04:01:15.180919Z","iopub.status.idle":"2024-03-19T04:01:15.190330Z","shell.execute_reply.started":"2024-03-19T04:01:15.180891Z","shell.execute_reply":"2024-03-19T04:01:15.189278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"id":"CH-31oqQdyBM","outputId":"a3fe64a4-c951-462d-95aa-43b195303b85","execution":{"iopub.status.busy":"2024-03-19T04:01:15.191588Z","iopub.execute_input":"2024-03-19T04:01:15.192120Z","iopub.status.idle":"2024-03-19T04:01:15.323338Z","shell.execute_reply.started":"2024-03-19T04:01:15.192090Z","shell.execute_reply":"2024-03-19T04:01:15.321660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape                     # Forma del conjunto de datos (filas, columnas)","metadata":{"id":"mw6XrbTAhKbr","outputId":"e66d243e-ec41-49cc-c8de-e503ec15fb7d","execution":{"iopub.status.busy":"2024-03-19T04:01:15.324600Z","iopub.execute_input":"2024-03-19T04:01:15.324936Z","iopub.status.idle":"2024-03-19T04:01:15.332807Z","shell.execute_reply.started":"2024-03-19T04:01:15.324903Z","shell.execute_reply":"2024-03-19T04:01:15.331376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.info()                    # Resumen del DataFrame, incluyendo el tipo de datos de cada columna y la cantidad de valores no nulos","metadata":{"id":"rBXu1hrWhWOL","outputId":"41a79834-e0cc-4b4c-c622-f6905b4a218e","execution":{"iopub.status.busy":"2024-03-19T04:01:15.334647Z","iopub.execute_input":"2024-03-19T04:01:15.335063Z","iopub.status.idle":"2024-03-19T04:01:15.355716Z","shell.execute_reply.started":"2024-03-19T04:01:15.335020Z","shell.execute_reply":"2024-03-19T04:01:15.354828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid.shape                     # Forma del conjunto de datos (filas, columnas)","metadata":{"id":"MJuGx-WbhdeL","outputId":"c12d8ad8-5acc-48a2-e78a-02001fce297f","execution":{"iopub.status.busy":"2024-03-19T04:01:15.357014Z","iopub.execute_input":"2024-03-19T04:01:15.357473Z","iopub.status.idle":"2024-03-19T04:01:15.364692Z","shell.execute_reply.started":"2024-03-19T04:01:15.357447Z","shell.execute_reply":"2024-03-19T04:01:15.363272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Imprimir las dimensiones de los conjuntos de datos de entrenamiento, validación y prueba\n\nprint(f\"Train: {X_train.shape}\")   # Imprime las dimensiones del conjunto de entrenamiento\nprint(f\"Valid: {X_valid.shape}\")   # Imprime las dimensiones del conjunto de validación\nprint(f\"Test: {X_test.shape}\")     # Imprime las dimensiones del conjunto de prueba","metadata":{"papermill":{"duration":0.019901,"end_time":"2024-02-07T21:28:32.249064","exception":false,"start_time":"2024-02-07T21:28:32.229163","status":"completed"},"tags":[],"id":"6a275858","outputId":"4ab09ff1-1c1e-43d5-f8ad-965703f7017d","execution":{"iopub.status.busy":"2024-03-19T04:01:15.365914Z","iopub.execute_input":"2024-03-19T04:01:15.366239Z","iopub.status.idle":"2024-03-19T04:01:15.375187Z","shell.execute_reply.started":"2024-03-19T04:01:15.366209Z","shell.execute_reply":"2024-03-19T04:01:15.374195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STEP 10 - ENTRENAMIENTO 1 USANDO EL MODELO USANDO LIGHTGBM Y VALIDACIÓN CRUZADA**","metadata":{"id":"pDCmZ31IoOSD"}},{"cell_type":"code","source":"# Definir los parámetros del modelo LightGBM\nparams = {\n    \"boosting_type\": \"gbdt\",         # Tipo de boosting: Gradient Boosting Decision Tree (gbdt)\n    \"objective\": \"binary\",           # Objetivo de la tarea: clasificación binaria\n    \"metric\": \"auc\",                 # Métrica a optimizar: área bajo la curva (AUC)\n    \"max_depth\": 9,                  # Profundidad máxima de los árboles: 9\n    \"num_leaves\": 80,                # Número máximo de hojas por árbol: 80\n    \"learning_rate\": 0.04,           # Tasa de aprendizaje: 0.04\n    \"feature_fraction\": 0.7,         # Fracción de características a considerar en cada árbol: 0.7\n    \"bagging_fraction\": 0.9,         # Fracción de muestras (instancias) a considerar en cada árbol: 0.9\n    \"bagging_freq\": 5,               # Frecuencia de bagging: 5 (cada 5 iteraciones)\n    \"n_estimators\": 1000,            # Número de estimadores (árboles) a entrenar: 1000\n    \"verbose\": -1                    # Nivel de verbosidad: -1 (sin mensajes)\n}\n\n# Definir el número de pliegues para la validación cruzada\nn_splits = 2\nkf = KFold(n_splits=n_splits, shuffle=True, random_state=1)\n\n# Lista para almacenar los modelos entrenados\nmodels = []\n\n# Lista para almacenar los puntajes AUC en cada pliegue\nauc_scores = []\n\n# Lista para almacenar los puntajes de estabilidad en cada pliegue\nstability_scores = []\n\n# Ciclo sobre los pliegues de la validación cruzada\nfor i, (train_index, valid_index) in enumerate(kf.split(X_train)):\n    print(f\"Fold {i+1}/{n_splits}\")\n\n    # Dividir el conjunto de entrenamiento en entrenamiento y validación para este pliegue\n    X_train_fold, X_valid_fold = X_train.iloc[train_index], X_train.iloc[valid_index]\n    y_train_fold, y_valid_fold = y_train.iloc[train_index], y_train.iloc[valid_index]\n\n    # Crear el conjunto de datos de LightGBM para este pliegue\n    lgb_train_fold = lgb.Dataset(X_train_fold, label=y_train_fold)\n    lgb_valid_fold = lgb.Dataset(X_valid_fold, label=y_valid_fold, reference=lgb_train_fold)\n\n\n    # Entrenar el modelo LightGBM para este pliegue\n    gbm = lgb.train(\n        params,                          # Parámetros del modelo\n        lgb_train_fold,                  # Conjunto de datos de entrenamiento\n        valid_sets=[lgb_train_fold, lgb_valid_fold],  # Conjunto de datos de entrenamiento y validación\n        callbacks=[lgb.log_evaluation(50), lgb.early_stopping(10)]  # Callbacks para registrar la evaluación y detener el entrenamiento prematuramente\n    )\n\n     # Evaluar el modelo en el conjunto de validación y almacena el AUC\n    y_pred_valid_fold = gbm.predict(X_valid_fold, num_iteration=gbm.best_iteration)\n    auc_fold = roc_auc_score(y_valid_fold, y_pred_valid_fold)\n    auc_scores.append(auc_fold)\n\n# Definir la función para calcular el puntaje de estabilidad Gini\ndef gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n\n# Calcular el Gini en el tiempo para cada semana en el DataFrame base\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n\n# Ajustar una línea de regresión lineal a los valores de Gini en el tiempo\n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n\n# Calcular los residuos y su desviación estándar\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n\n# Calcular el promedio del Gini en el tiempo\n    avg_gini = np.mean(gini_in_time)\n\n# Calcular el puntaje de estabilidad Gini como la suma del promedio del Gini,\n# el peso de la tasa de caída (w_fallingrate) multiplicado por el mínimo de 0 y la pendiente (a),\n# y el peso de la desviación estándar de los residuos (w_resstd) multiplicado por la desviación estándar (res_std)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\n    # Almacenar el modelo entrenado\n    models.append(gbm)\n\n    # Calcular y almacenar el puntaje de estabilidad\n    stability_fold = gini_stability(base_valid, w_fallingrate=88.0, w_resstd=-0.5)\n    stability_scores.append(stability_fold)\n\n    # Calcular el promedio de los puntajes AUC y de estabilidad\n    avg_auc = np.mean(auc_scores)\n    avg_stability = np.mean(stability_scores)\n\n    # Imprimir los puntajes promedio\n    print(f\"AUC Promedio en Validación Cruzada: {avg_auc}\")\n    print(f\"Estabilidad Promedio en Validación Cruzada: {avg_stability}\")\n","metadata":{"id":"lrMPBYBfEFdl","outputId":"cbc292e1-74cc-4fbf-c9d1-760ebd2d3542","execution":{"iopub.status.busy":"2024-03-19T04:01:15.376750Z","iopub.execute_input":"2024-03-19T04:01:15.377462Z","iopub.status.idle":"2024-03-19T04:02:55.214633Z","shell.execute_reply.started":"2024-03-19T04:01:15.377422Z","shell.execute_reply":"2024-03-19T04:02:55.213594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STEP 10 - ENTRENAMIENTO 2 USANDO EL MODELO USANDO LIGHTGBM**\n","metadata":{"id":"ya84u-39WlPP"}},{"cell_type":"code","source":"# Crear un conjunto de datos de entrenamiento para LightGBM con características (X_train) y etiquetas (y_train)\nlgb_train = lgb.Dataset(X_train, label=y_train)\n\n# Crear un conjunto de datos de validación para LightGBM con características (X_valid) y etiquetas (y_valid),\n# haciendo referencia al conjunto de datos de entrenamiento lgb_train\nlgb_valid = lgb.Dataset(X_valid, label=y_valid, reference=lgb_train)\n\n# Definir los parámetros del modelo LightGBM\nparams = {\n    \"boosting_type\": \"gbdt\",    # Tipo de boosting: Gradient Boosting Decision Tree (gbdt)\n    \"objective\": \"binary\",      # Objetivo de la tarea: clasificación binaria\n    \"metric\": \"auc\",            # Métrica a optimizar: área bajo la curva (AUC)\n    \"max_depth\": 9,             # Profundidad máxima de los árboles: 9\n    \"num_leaves\": 80,           # Número máximo de hojas por árbol: 80\n    \"learning_rate\": 0.04,      # Tasa de aprendizaje: 0.04\n    \"feature_fraction\": 0.7,    # Fracción de características a considerar en cada árbol: 0.7\n    \"bagging_fraction\": 0.9,    # Fracción de muestras (instancias) a considerar en cada árbol: 0.9\n    \"bagging_freq\": 5,          # Frecuencia de bagging: 5 (cada 5 iteraciones)\n    \"n_estimators\": 1000,       # Número de estimadores (árboles) a entrenar: 1000\n    \"verbose\": -1               # Nivel de verbosidad: -1\n}\n\n# Entrenar el modelo LightGBM con los parámetros definidos\ngbm = lgb.train(\n    params,                     # Parámetros del modelo\n    lgb_train,                  # Conjunto de datos de entrenamiento\n    valid_sets=lgb_valid,       # Conjunto de datos de validación\n\n# Callbacks para registrar la evaluación y detener el entrenamiento prematuramente\n    callbacks=[lgb.log_evaluation(50), lgb.early_stopping(10)]\n)\n","metadata":{"papermill":{"duration":80.572251,"end_time":"2024-02-07T21:29:52.842351","exception":false,"start_time":"2024-02-07T21:28:32.270100","status":"completed"},"tags":[],"id":"fb1d2bc7","outputId":"39b589c7-ec93-4862-d1da-bdad99c639e2","execution":{"iopub.status.busy":"2024-03-19T04:02:55.215608Z","iopub.execute_input":"2024-03-19T04:02:55.216061Z","iopub.status.idle":"2024-03-19T04:04:00.854638Z","shell.execute_reply.started":"2024-03-19T04:02:55.216035Z","shell.execute_reply":"2024-03-19T04:04:00.853306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STEP 11 - EVALUACIÓN DEL MODELO LIGHTGMB**","metadata":{"id":"NkLE6aFdXSWQ"}},{"cell_type":"code","source":"# Evaluaciòn : Entrenamiento (base_train), validación (base_valid), y prueba (base_test).\n\nfor base, X in [(base_train, X_train), (base_valid, X_valid), (base_test, X_test)]:  # Iterar en cada conjunto de datos de entrenamiento, validación y prueba\n    y_pred = gbm.predict(X, num_iteration=gbm.best_iteration)                        # Realizar predicciones utilizando el modelo LightGBM\n    base[\"score\"] = y_pred                                                           # Agregar las predicciones como una nueva columna llamada \"score\" en el DataFrame base\n\nprint(f'The AUC score on the train set is: {roc_auc_score(base_train[\"target\"], base_train[\"score\"])}')  # Imprimir el AUC score para el conjunto de datos de entrenamiento\nprint(f'The AUC score on the valid set is: {roc_auc_score(base_valid[\"target\"], base_valid[\"score\"])}')  # Imprimir el AUC score para el conjunto de datos de validación\nprint(f'The AUC score on the test set is: {roc_auc_score(base_test[\"target\"], base_test[\"score\"])}')     # Imprimir el AUC score para el conjunto de datos de prueba","metadata":{"papermill":{"duration":22.617002,"end_time":"2024-02-07T21:30:15.484653","exception":false,"start_time":"2024-02-07T21:29:52.867651","status":"completed"},"tags":[],"id":"3e5c4fdb","outputId":"c53012b8-2010-457b-c095-1a8c9dc6fd9c","execution":{"iopub.status.busy":"2024-03-19T04:04:00.855706Z","iopub.execute_input":"2024-03-19T04:04:00.855983Z","iopub.status.idle":"2024-03-19T04:04:20.259342Z","shell.execute_reply.started":"2024-03-19T04:04:00.855958Z","shell.execute_reply":"2024-03-19T04:04:20.258069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Definir la función para calcular el puntaje de estabilidad Gini\ndef gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n\n# Calcular el Gini en el tiempo para cada semana en el DataFrame base\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n\n# Ajustar una línea de regresión lineal a los valores de Gini en el tiempo\n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n\n# Calcular los residuos y su desviación estándar\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n\n# Calcular el promedio del Gini en el tiempo\n    avg_gini = np.mean(gini_in_time)\n\n# Calcular el puntaje de estabilidad Gini como la suma del promedio del Gini,\n# el peso de la tasa de caída (w_fallingrate) multiplicado por el mínimo de 0 y la pendiente (a),\n# y el peso de la desviación estándar de los residuos (w_resstd) multiplicado por la desviación estándar (res_std)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\n\n# Calcular el puntaje de estabilidad Gini para el conjunto de entrenamiento, validación y prueba\nstability_score_train = gini_stability(base_train)\nstability_score_valid = gini_stability(base_valid)\nstability_score_test = gini_stability(base_test)\n\n# Imprimir los puntajes de estabilidad Gini para cada conjunto de datos\nprint(f'The stability score on the train set is: {stability_score_train}')\nprint(f'The stability score on the valid set is: {stability_score_valid}')\nprint(f'The stability score on the test set is: {stability_score_test}')","metadata":{"papermill":{"duration":1.131134,"end_time":"2024-02-07T21:30:16.624526","exception":false,"start_time":"2024-02-07T21:30:15.493392","status":"completed"},"tags":[],"id":"2e10914f","outputId":"f03261ab-2201-43f8-ca24-fa9fb8be2ada","execution":{"iopub.status.busy":"2024-03-19T04:04:20.260555Z","iopub.execute_input":"2024-03-19T04:04:20.260903Z","iopub.status.idle":"2024-03-19T04:04:21.059266Z","shell.execute_reply.started":"2024-03-19T04:04:20.260859Z","shell.execute_reply":"2024-03-19T04:04:21.058209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STEP 12 - SUBMISSION**","metadata":{"id":"WxZk_fZakAGJ"}},{"cell_type":"code","source":"X_submission = data_submission[cols_pred].to_pandas()\nX_submission = convert_strings(X_submission)\ncategorical_cols = X_train.select_dtypes(include=['category']).columns\n\nfor col in categorical_cols:\n    train_categories = set(X_train[col].cat.categories)\n    submission_categories = set(X_submission[col].cat.categories)\n    new_categories = submission_categories - train_categories\n    X_submission.loc[X_submission[col].isin(new_categories), col] = \"Unknown\"\n    new_dtype = pd.CategoricalDtype(categories=train_categories, ordered=True)\n    X_train[col] = X_train[col].astype(new_dtype)\n    X_submission[col] = X_submission[col].astype(new_dtype)\n\ny_submission_pred = gbm.predict(X_submission, num_iteration=gbm.best_iteration)","metadata":{"id":"HBQnx0hAj8Bh","execution":{"iopub.status.busy":"2024-03-19T04:04:48.619977Z","iopub.execute_input":"2024-03-19T04:04:48.620336Z","iopub.status.idle":"2024-03-19T04:04:48.688923Z","shell.execute_reply.started":"2024-03-19T04:04:48.620307Z","shell.execute_reply":"2024-03-19T04:04:48.687784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": data_submission[\"case_id\"].to_numpy(),\n    \"score\": y_submission_pred\n}).set_index('case_id')\nsubmission.to_csv(\"./submission.csv\")","metadata":{"id":"PtGCceqzj_Ph","execution":{"iopub.status.busy":"2024-03-19T04:04:51.363605Z","iopub.execute_input":"2024-03-19T04:04:51.363938Z","iopub.status.idle":"2024-03-19T04:04:51.372335Z","shell.execute_reply.started":"2024-03-19T04:04:51.363911Z","shell.execute_reply":"2024-03-19T04:04:51.370481Z"},"trusted":true},"execution_count":null,"outputs":[]}]}