{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":35332,"databundleVersionId":3723648,"sourceType":"competition"},{"sourceId":3739819,"sourceType":"datasetVersion","datasetId":2231132}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Inicio de trabajo final\n# Implementación","metadata":{}},{"cell_type":"markdown","source":"## 1. EDA y Optimización de memoria","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport dask.dataframe as dd\nimport matplotlib.pyplot as plt\nimport gc\nimport os\nimport time\nprint(\"Librerias importadas\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T11:31:02.495789Z","iopub.execute_input":"2024-11-27T11:31:02.497532Z","iopub.status.idle":"2024-11-27T11:31:02.512848Z","shell.execute_reply.started":"2024-11-27T11:31:02.497410Z","shell.execute_reply":"2024-11-27T11:31:02.509962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!conda install dask distributed -c conda-forge -y","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from dask.distributed import LocalCluster, Client\n\nnworkers = 4 # num workers\nnthreads = 2 # hilos por worker\n\ncluster = LocalCluster(n_workers=nworkers, threads_per_worker=nthreads)\nclient = Client(cluster)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Iniciar la medición del tiempo de ejecución\nstart_time = time.perf_counter()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T10:22:50.794170Z","iopub.execute_input":"2024-11-27T10:22:50.794554Z","iopub.status.idle":"2024-11-27T10:22:50.799403Z","shell.execute_reply.started":"2024-11-27T10:22:50.794524Z","shell.execute_reply":"2024-11-27T10:22:50.798163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\")\nsns.countplot(x=labels[\"target\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T10:22:52.826267Z","iopub.execute_input":"2024-11-27T10:22:52.827145Z","iopub.status.idle":"2024-11-27T10:22:54.143647Z","shell.execute_reply.started":"2024-11-27T10:22:52.827107Z","shell.execute_reply":"2024-11-27T10:22:54.142494Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1ra Oportunidad de Optimización","metadata":{}},{"cell_type":"code","source":"n_repeticiones = 5\n\n# Lista para almacenar los tiempos de ejecución secuenciales\ntiempos_secuenciales = []\n\n# Lista para almacenar los tiempos de ejecución paralelos\ntiempos_paralelos = []\n\nfor _ in range(n_repeticiones):\n    # --- Código secuencial ---\n    BN1_start_time = time.perf_counter()\n    customer_date_df = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\",usecols=[x for x in range(2)])\n    BN1_end_time = time.perf_counter()\n    BN1_elapsed_time = BN1_end_time - BN1_start_time\n    tiempos_secuenciales.append(BN1_elapsed_time)\n\n\n\n    # --- Código paralelo ---\n    PA1_start_time = time.perf_counter()\n    customer_date_df = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=[x for x in range(2)])\n    customer_date_df = customer_date_df.compute()\n    PA1_end_time = time.perf_counter()\n    PA1_elapsed_time = PA1_end_time - PA1_start_time\n    tiempos_paralelos.append(PA1_elapsed_time)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T10:22:56.832124Z","iopub.execute_input":"2024-11-27T10:22:56.832535Z","iopub.status.idle":"2024-11-27T10:24:38.337908Z","shell.execute_reply.started":"2024-11-27T10:22:56.832503Z","shell.execute_reply":"2024-11-27T10:24:38.336499Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calcular el tiempo promedio para ambos casos\ntiempo_secuencial_promedio = sum(tiempos_secuenciales) / n_repeticiones\ntiempo_paralelo_promedio = sum(tiempos_paralelos) / n_repeticiones\n\nnum_cores = os.cpu_count()\n\nspeedup = tiempo_secuencial_promedio / tiempo_paralelo_promedio\nefficiency = speedup / num_cores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T10:28:31.985335Z","iopub.execute_input":"2024-11-27T10:28:31.985789Z","iopub.status.idle":"2024-11-27T10:30:08.263577Z","shell.execute_reply.started":"2024-11-27T10:28:31.985758Z","shell.execute_reply":"2024-11-27T10:30:08.262104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Tiempo secuencial promedio: {tiempo_secuencial_promedio:.4f} segundos\")\nprint(f\"Tiempo paralelo promedio: {tiempo_paralelo_promedio:.4f} segundos\")\nprint(f\"Speedup: {speedup:.4f}\")\nprint(f\"Eficiencia: {efficiency:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## ANALISIS DE ESCALABILIDAD\nworkers_threads_start_time = time.perf_counter()\n# blocksize controla el tamaño de cada partición del DataFrame. Es importante elegir un tamaño adecuado para un rendimiento óptimo.\nddf = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=[x for x in range(2)], blocksize=\"100MB\")\nddf = ddf.compute()\nworkers_threads_end_time = time.perf_counter()\nelapsed_WT = workers_threads_end_time - workers_threads_start_time\nprint(f\"Tiempo con {nworkers} workers y {nthreads} hilos: {elapsed_WT:.4f} segundos\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customer_date_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T10:24:54.682112Z","iopub.execute_input":"2024-11-27T10:24:54.683528Z","iopub.status.idle":"2024-11-27T10:24:54.697336Z","shell.execute_reply.started":"2024-11-27T10:24:54.683467Z","shell.execute_reply":"2024-11-27T10:24:54.696303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customer_date_df.info(memory_usage='deep')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T10:24:57.146699Z","iopub.execute_input":"2024-11-27T10:24:57.147845Z","iopub.status.idle":"2024-11-27T10:24:58.327975Z","shell.execute_reply.started":"2024-11-27T10:24:57.147801Z","shell.execute_reply":"2024-11-27T10:24:58.326491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This show the number of unique values for each column in the dataframe\ncustomer_date_df.nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T10:25:00.704029Z","iopub.execute_input":"2024-11-27T10:25:00.704425Z","iopub.status.idle":"2024-11-27T10:25:01.877255Z","shell.execute_reply.started":"2024-11-27T10:25:00.704394Z","shell.execute_reply":"2024-11-27T10:25:01.876128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert customer_ID to int\nunique_ids = customer_date_df['customer_ID'].unique()\nid_mapping = {_id: i for i, _id in enumerate(unique_ids)}\ncustomer_date_df['customer_ID'] = customer_date_df['customer_ID'].map(id_mapping).astype(pd.Int32Dtype())\ncustomer_date_df.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T10:25:03.668351Z","iopub.execute_input":"2024-11-27T10:25:03.669286Z","iopub.status.idle":"2024-11-27T10:25:05.367636Z","shell.execute_reply.started":"2024-11-27T10:25:03.669245Z","shell.execute_reply":"2024-11-27T10:25:05.366408Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2da Oportunidad de Optimización","metadata":{}},{"cell_type":"code","source":"n_repeticiones = 5\n\n# Lista para almacenar los tiempos de ejecución secuenciales\ntiempos_secuenciales_optimizacion_2 = []\n\n# Lista para almacenar los tiempos de ejecución paralelos\ntiempos_paralelos_optimizacion_2 = []\n\nfor _ in range(n_repeticiones):\n    # --- Código secuencial ---\n    BN2_start_time = time.perf_counter()\n    customer_date_df = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\",usecols=[x for x in range(2)])\n    customer_date_df[['year', 'month', 'day']] = customer_date_df['S_2'].str.split('-', expand=True)\n    customer_date_df = customer_date_df.drop(\"S_2\", axis=1)\n    BN2_end_time = time.perf_counter()\n    BN2_elapsed_time = BN2_end_time - BN2_start_time\n    tiempos_secuenciales_optimizacion_2.append(BN2_elapsed_time)\n\n\n    # --- Código paralelo ---\n    PA2_start_time = time.perf_counter()\n    customer_date_parallel_df = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=[x for x in range(2)])\n    customer_date_parallel_df[\"year\"] = customer_date_parallel_df[\"S_2\"].str[:4]\n    customer_date_parallel_df[\"month\"] = customer_date_parallel_df[\"S_2\"].str[5:7]\n    customer_date_parallel_df[\"day\"] = customer_date_parallel_df[\"S_2\"].str[8:10]\n    customer_date_parallel_df = customer_date_parallel_df.drop(columns=[\"S_2\"])\n    customer_date_parallel_df = customer_date_parallel_df.persist() # Persistir los datos para evitar recalculado de operaciones\n    customer_date_parallel_df = customer_date_parallel_df.compute() # Convierte solo al final\n    PA2_end_time = time.perf_counter()\n    PA2_elapsed_time = PA2_end_time - PA2_start_time\n    tiempos_paralelos_optimizacion_2.append(PA2_elapsed_time)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T10:51:37.625631Z","iopub.execute_input":"2024-11-27T10:51:37.626676Z","iopub.status.idle":"2024-11-27T11:29:45.297911Z","shell.execute_reply.started":"2024-11-27T10:51:37.626632Z","shell.execute_reply":"2024-11-27T11:29:45.296727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calcular el tiempo promedio para ambos casos\ntiempo_secuencial_promedio = sum(tiempos_secuenciales_optimizacion_2) / n_repeticiones\ntiempo_paralelo_promedio = sum(tiempos_paralelos_optimizacion_2) / n_repeticiones\n\nnum_cores = os.cpu_count()\n\nspeedup = tiempo_secuencial_promedio / tiempo_paralelo_promedio\nefficiency = speedup / num_cores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T11:31:23.456255Z","iopub.execute_input":"2024-11-27T11:31:23.456769Z","iopub.status.idle":"2024-11-27T11:31:23.463671Z","shell.execute_reply.started":"2024-11-27T11:31:23.456724Z","shell.execute_reply":"2024-11-27T11:31:23.462254Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Tiempo secuencial promedio: {tiempo_secuencial_promedio:.4f} segundos\")\nprint(f\"Tiempo paralelo promedio: {tiempo_paralelo_promedio:.4f} segundos\")\nprint(f\"Speedup: {speedup:.4f}\")\nprint(f\"Eficiencia: {efficiency:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T11:31:27.446148Z","iopub.execute_input":"2024-11-27T11:31:27.446583Z","iopub.status.idle":"2024-11-27T11:31:27.452973Z","shell.execute_reply.started":"2024-11-27T11:31:27.446550Z","shell.execute_reply":"2024-11-27T11:31:27.451607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## ANALISIS DE ESCALABILIDAD\nworkers_threads_start_time = time.perf_counter()\n# blocksize controla el tamaño de cada partición del DataFrame. Es importante elegir un tamaño adecuado para un rendimiento óptimo.\ncustomer_date_parallel_df = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=[x for x in range(2)])\ncustomer_date_parallel_df[\"year\"] = customer_date_parallel_df[\"S_2\"].str[:4]\ncustomer_date_parallel_df[\"month\"] = customer_date_parallel_df[\"S_2\"].str[5:7]\ncustomer_date_parallel_df[\"day\"] = customer_date_parallel_df[\"S_2\"].str[8:10]\ncustomer_date_parallel_df = customer_date_parallel_df.drop(columns=[\"S_2\"])\ncustomer_date_parallel_df = customer_date_parallel_df.persist()\ncustomer_date_parallel_df = customer_date_parallel_df.compute()\nworkers_threads_end_time = time.perf_counter()\nelapsed_WT = workers_threads_end_time - workers_threads_start_time\nprint(f\"Tiempo con {nworkers} workers y {nthreads} hilos: {elapsed_WT:.4f} segundos\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert the new columns to integers\ncustomer_date_df[\"year\"] = customer_date_df[\"year\"].astype(pd.Int32Dtype())\ncustomer_date_df[\"year\"] -= 2000\nfor col in [\"year\", \"month\", \"day\"]:\n    customer_date_df[col] = customer_date_df[col].astype(pd.Int8Dtype())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customer_date_df.info(memory_usage='deep')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_null_values(df):\n    null_values = df.isnull().mean() * 100\n    null_values = null_values.sort_values(ascending=False)\n    null_values = null_values[null_values > 0]\n    \n    if not null_values.empty:\n        sns.barplot(y=null_values.index, x=null_values.values)\n    else:\n        print(\"No null values\")\n\nplot_null_values(customer_date_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del customer_date_df\ndel unique_ids\ndel id_mapping\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3ra Oportunidad de Optimización","metadata":{}},{"cell_type":"code","source":"n_repeticiones = 5\n\n# Lista para almacenar los tiempos de ejecución secuenciales\ntiempos_secuenciales_optimizacion_3 = []\n\n# Lista para almacenar los tiempos de ejecución paralelos\ntiempos_paralelos_optimizacion_3 = []\n\nfor _ in range(n_repeticiones):\n    # --- Código secuencial ---\n    BN3_start_time = time.perf_counter()\n    categorical_columns = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n    cat_df = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=categorical_columns[:2])\n    BN3_end_time = time.perf_counter()\n    BN3_elapsed_time = BN3_end_time - BN3_start_time\n    tiempos_secuenciales_optimizacion_3.append(BN3_elapsed_time)\n\n\n    # --- Código paralelo ---\n    PA3_start_time = time.perf_counter()\n    categorical_columns = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n    cat_df = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=categorical_columns[:2])\n    cat_df = cat_df.compute() # Para obtener el resultado final como un DataFrame de Pandas.\n    PA3_end_time = time.perf_counter()\n    PA3_elapsed_time = PA3_end_time - PA3_start_time\n    tiempos_paralelos_optimizacion_3.append(PA3_elapsed_time)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T11:32:25.356040Z","iopub.execute_input":"2024-11-27T11:32:25.356488Z","iopub.status.idle":"2024-11-27T12:03:51.984173Z","shell.execute_reply.started":"2024-11-27T11:32:25.356424Z","shell.execute_reply":"2024-11-27T12:03:51.982629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calcular el tiempo promedio para ambos casos\ntiempo_secuencial_promedio = sum(tiempos_secuenciales_optimizacion_3) / n_repeticiones\ntiempo_paralelo_promedio = sum(tiempos_paralelos_optimizacion_3) / n_repeticiones\n\nnum_cores = os.cpu_count()\n\nspeedup = tiempo_secuencial_promedio / tiempo_paralelo_promedio\nefficiency = speedup / num_cores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T12:07:31.981831Z","iopub.execute_input":"2024-11-27T12:07:31.982261Z","iopub.status.idle":"2024-11-27T12:07:31.988799Z","shell.execute_reply.started":"2024-11-27T12:07:31.982229Z","shell.execute_reply":"2024-11-27T12:07:31.987382Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Tiempo secuencial promedio: {tiempo_secuencial_promedio:.4f} segundos\")\nprint(f\"Tiempo paralelo promedio: {tiempo_paralelo_promedio:.4f} segundos\")\nprint(f\"Speedup: {speedup:.4f}\")\nprint(f\"Eficiencia: {efficiency:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-27T12:07:34.143475Z","iopub.execute_input":"2024-11-27T12:07:34.144425Z","iopub.status.idle":"2024-11-27T12:07:34.151102Z","shell.execute_reply.started":"2024-11-27T12:07:34.144385Z","shell.execute_reply":"2024-11-27T12:07:34.149652Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## ANALISIS DE ESCALABILIDAD\nworkers_threads_start_time = time.perf_counter()\n# blocksize controla el tamaño de cada partición del DataFrame. Es importante elegir un tamaño adecuado para un rendimiento óptimo.\ncategorical_columns = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\ncat_df = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=categorical_columns[:2])\ncat_df = cat_df.compute()\nworkers_threads_end_time = time.perf_counter()\nelapsed_WT = workers_threads_end_time - workers_threads_start_time\nprint(f\"Tiempo con {nworkers} workers y {nthreads} hilos: {elapsed_WT:.4f} segundos\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_df.info(memory_usage='deep')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We can map each value of the categorical features to a sequential integer\nfor col in cat_df.columns:\n    unique = cat_df[col].unique()\n    mapping = {val: i for i, val in enumerate(unique)}\n    cat_df[col] = cat_df[col].map(mapping).astype(pd.Int8Dtype())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_df.info(memory_usage='deep')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_null_values(cat_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_df.nunique()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.countplot(data=cat_df, y=\"B_38\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del categorical_columns\ndel cat_df\ndel unique\ndel mapping\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4ta Oportunidad de Optimización","metadata":{}},{"cell_type":"code","source":"n_repeticiones = 5\n\n# Lista para almacenar los tiempos de ejecución secuenciales\ntiempos_secuenciales_optimizacion_4 = []\n\n# Lista para almacenar los tiempos de ejecución paralelos\ntiempos_paralelos_optimizacion_4 = []\n\nfor _ in range(n_repeticiones):\n    # --- Código secuencial ---\n    BN4_start_time = time.perf_counter()\n    load_cols = [\"customer_ID\", \"D_130\"]\n    num_df = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=load_cols)\n    BN4_end_time = time.perf_counter()\n    BN4_elapsed_time = BN4_end_time - BN4_start_time\n    tiempos_secuenciales_optimizacion_4.append(BN4_elapsed_time)\n\n\n    # --- Código paralelo ---\n    PA4_start_time = time.perf_counter()\n    num_df = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=load_cols)\n    num_df = num_df.compute()\n    PA4_end_time = time.perf_counter()\n    PA4_elapsed_time = PA4_end_time - PA4_start_time\n    tiempos_paralelos_optimizacion_4.append(PA4_elapsed_time)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calcular el tiempo promedio para ambos casos\ntiempo_secuencial_promedio = sum(tiempos_secuenciales_optimizacion_4) / n_repeticiones\ntiempo_paralelo_promedio = sum(tiempos_paralelos_optimizacion_4) / n_repeticiones\n\nnum_cores = os.cpu_count()\n\nspeedup = tiempo_secuencial_promedio / tiempo_paralelo_promedio\nefficiency = speedup / num_cores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Tiempo secuencial promedio: {tiempo_secuencial_promedio:.4f} segundos\")\nprint(f\"Tiempo paralelo promedio: {tiempo_paralelo_promedio:.4f} segundos\")\nprint(f\"Speedup: {speedup:.4f}\")\nprint(f\"Eficiencia: {efficiency:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## ANALISIS DE ESCALABILIDAD\nworkers_threads_start_time = time.perf_counter()\n# blocksize controla el tamaño de cada partición del DataFrame. Es importante elegir un tamaño adecuado para un rendimiento óptimo.\nnum_df = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=load_cols)\nnum_df = num_df.compute()\nworkers_threads_end_time = time.perf_counter()\nelapsed_WT = workers_threads_end_time - workers_threads_start_time\nprint(f\"Tiempo con {nworkers} workers y {nthreads} hilos: {elapsed_WT:.4f} segundos\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_df.info(memory_usage='deep')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_null_values(num_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.histplot(data=num_df.sample(100000), x=\"D_130\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.histplot(data=num_df[num_df[\"D_130\"] < 0.1], x=\"D_130\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.histplot(data=num_df[num_df[\"D_130\"] > 1], x=\"D_130\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_df[\"D_130\"] = (np.floor(num_df[\"D_130\"]+1e-6)).fillna(-1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.histplot(data=num_df.sample(100000), x=\"D_130\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_df[\"D_130\"].value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"merge_df = pd.merge(num_df, labels, on=\"customer_ID\")\nmerge_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"merge_df.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_eda(data, feature, target):\n    fig, ax = plt.subplots(1,3, figsize=(15, 6))\n    sns.histplot(data=data, x=feature, ax=ax[0])\n    sns.histplot(data=data, x=feature, hue=target, ax=ax[1])\n    sns.regplot(data=data, x=feature, y=target, ax=ax[2])\n    \nplot_eda(merge_df.sample(100000), \"D_130\", \"target\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5ta Oportunidad de Optimización","metadata":{}},{"cell_type":"code","source":"n_repeticiones = 5\n\n# Lista para almacenar los tiempos de ejecución secuenciales\ntiempos_secuenciales_optimizacion_5 = []\n\n# Lista para almacenar los tiempos de ejecución paralelos\ntiempos_paralelos_optimizacion_5 = []\n\nfor _ in range(n_repeticiones):\n    # --- Código secuencial ---\n    BN5_start_time = time.perf_counter()\n    load_cols = [\"customer_ID\", \"P_2\"]\n    customer_date_df = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\",usecols=[x for x in range(2)])\n    BN5_end_time = time.perf_counter()\n    BN5_elapsed_time = BN5_end_time - BN5_start_time\n    tiempos_secuenciales_optimizacion_5.append(BN5_elapsed_time)\n\n\n\n    # --- Código paralelo ---\n    PA5_start_time = time.perf_counter()\n    load_cols = [\"customer_ID\", \"P_2\"]\n    num_df = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=load_cols)\n    num_df = num_df.compute()\n    PA5_end_time = time.perf_counter()\n    PA5_elapsed_time = PA5_end_time - PA5_start_time\n    tiempos_paralelos_optimizacion_5.append(PA5_elapsed_time)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calcular el tiempo promedio para ambos casos\ntiempo_secuencial_promedio = sum(tiempos_secuenciales_optimizacion_5) / n_repeticiones\ntiempo_paralelo_promedio = sum(tiempos_paralelos_optimizacion_5) / n_repeticiones\n\nnum_cores = os.cpu_count()\n\nspeedup = tiempo_secuencial_promedio / tiempo_paralelo_promedio\nefficiency = speedup / num_cores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Tiempo secuencial promedio: {tiempo_secuencial_promedio:.4f} segundos\")\nprint(f\"Tiempo paralelo promedio: {tiempo_paralelo_promedio:.4f} segundos\")\nprint(f\"Speedup: {speedup:.4f}\")\nprint(f\"Eficiencia: {efficiency:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## ANALISIS DE ESCALABILIDAD\nworkers_threads_start_time = time.perf_counter()\n# blocksize controla el tamaño de cada partición del DataFrame. Es importante elegir un tamaño adecuado para un rendimiento óptimo.\nload_cols = [\"customer_ID\", \"P_2\"]\nnum_df = dd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=load_cols)\nnum_df = num_df.compute()\nworkers_threads_end_time = time.perf_counter()\nelapsed_WT = workers_threads_end_time - workers_threads_start_time\nprint(f\"Tiempo con {nworkers} workers y {nthreads} hilos: {elapsed_WT:.4f} segundos\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"merge_df = pd.merge(num_df, labels, on=\"customer_ID\")\nmerge_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_eda(merge_df.sample(100000), \"P_2\", \"target\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del load_cols\ndel num_df\ndel merge_df\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## 2. Preprocesamiento & ingeniería de características","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_parquet(\"/kaggle/input/amex-data-integer-dtypes-parquet-format/train.parquet\")\ndf_train = df_train.set_index(\"customer_ID\")\ndf_train.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = pd.read_parquet(\"/kaggle/input/amex-data-integer-dtypes-parquet-format/test.parquet\")\ndf_test = df_test.set_index(\"customer_ID\")\ndf_test.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\", index_col=\"customer_ID\")\nlabels.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.merge(df_train, labels, left_index=True, right_index=True)\ndf_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_proportion = df_train[\"target\"].sum() / df_train.shape[0]\nclass_proportion","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 10k rows subsample\nsample_size = 10000\n\n# For our training dataset we are going to maintain the same positive/negative sample ratio!\nsample_df_positive = df_train[df_train[\"target\"] == 1].sample(int(sample_size * class_proportion))\nsample_df_negative = df_train[df_train[\"target\"] == 0].sample(sample_size - sample_df_positive.shape[0])\n\n# Concat and then shuffle\ndf_train = pd.concat([sample_df_positive, sample_df_negative])\ndf_train = df_train.sample(frac=1, random_state=42)\n\ndf_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = df_test.sample(10000)\ndf_test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\ndate_col = \"S_2\"\nid_col = \"customer_ID\"\ntarget_col = \"target\"\nnumerical_features = [x for x in df_train.columns if x not in categorical_features and x != date_col and x != id_col and x != target_col]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unique_categories = {}\nfor col in categorical_features:\n    unique_categories[col] = set(df_train[col].unique()).union(set(df_test[col].unique()))\n\n# Combine unique categories and create a mapping to integers\ncategory_to_int = {col: {category: i for i, category in enumerate(categories)} for col, categories in unique_categories.items()}\ncategory_to_int","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def one_hot_encoding(df, categorical_features, category_to_int):\n    df[categorical_features] = df[categorical_features].apply(lambda col: col.map(category_to_int[col.name]))\n    df = pd.get_dummies(df, columns=categorical_features)\n    \n    ohe_cols = []\n    \n    for col in categorical_features:\n        for k,v in category_to_int[col].items():\n            ohe_cols.append(f\"{col}_{v}\")\n            if f\"{col}_{v}\" not in df.columns:\n                df[f\"{col}_{v}\"] = 0\n    \n    return df, ohe_cols\n\ndf_train, _ = one_hot_encoding(df_train, categorical_features, category_to_int)\ndf_test, ohe_cols = one_hot_encoding(df_test, categorical_features, category_to_int)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aggregated_cat_df_train = df_train.groupby('customer_ID')[ohe_cols].agg(['sum'])\naggregated_cat_df_train.columns = aggregated_cat_df_train.columns.map('_'.join)\naggregated_cat_df_train = aggregated_cat_df_train.reset_index()\naggregated_cat_df_train.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aggregated_cat_df_test = df_test.groupby('customer_ID')[ohe_cols].agg(['sum'])\naggregated_cat_df_test.columns = aggregated_cat_df_test.columns.map('_'.join)\naggregated_cat_df_test = aggregated_cat_df_test.reset_index()\naggregated_cat_df_test.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aggregated_num_df_train = df_train.groupby('customer_ID')[numerical_features].agg(['sum', 'mean'])\naggregated_num_df_train.columns = aggregated_num_df_train.columns.map('_'.join)\naggregated_num_df_train = aggregated_num_df_train.reset_index()\naggregated_num_df_train.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aggregated_num_df_test = df_test.groupby('customer_ID')[numerical_features].agg(['sum', 'mean'])\naggregated_num_df_test.columns = aggregated_num_df_test.columns.map('_'.join)\naggregated_num_df_test = aggregated_num_df_test.reset_index()\naggregated_num_df_test.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"agg_numerical_features = []\nfor f in numerical_features:\n    for a in ['sum', 'mean']:\n        agg_numerical_features.append(f\"{f}_{a}\")\n\ndef feature_transformation(df):\n        \n    # Feature transformation\n    new_columns = []\n\n    import warnings\n\n    for feature in agg_numerical_features:\n\n        with warnings.catch_warnings():\n            warnings.filterwarnings('ignore', category=RuntimeWarning)\n            log_feature = np.log1p(df[feature])\n            square_feature = df[feature] ** 2\n\n        log_feature.name = f'{feature}_log'\n        square_feature.name = f'{feature}_square'\n\n        # Append the series to the list\n        new_columns.extend([log_feature, square_feature])\n    \n    df = pd.concat([df, *new_columns], axis=1)\n    return df\n\ntransformed_num_df_train = feature_transformation(aggregated_num_df_train)\ntransformed_num_df_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transformed_num_df_test = feature_transformation(aggregated_num_df_test)\ntransformed_num_df_test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.merge(aggregated_cat_df_train, transformed_num_df_train, left_on=\"customer_ID\", right_on=\"customer_ID\")\ndf_train = df_train.set_index(\"customer_ID\")\ndf_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = pd.merge(aggregated_cat_df_test, transformed_num_df_test, left_on=\"customer_ID\", right_on=\"customer_ID\")\ndf_test = df_test.set_index(\"customer_ID\")\ndf_test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del labels\ndel class_proportion\ndel numerical_features\ndel target_col\ndel id_col\ndel date_col\ndel categorical_features\ndel sample_size\ndel sample_df_positive\ndel sample_df_negative\ndel unique_categories\ndel category_to_int\ndel ohe_cols\ndel aggregated_cat_df_train\ndel aggregated_cat_df_test\ndel aggregated_num_df_train\ndel aggregated_num_df_test\ndel agg_numerical_features\ndel transformed_num_df_train\ndel transformed_num_df_test\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Selección de características","metadata":{}},{"cell_type":"code","source":"labels = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\", index_col=\"customer_ID\")\ndf_train = pd.merge(df_train, labels, left_index=True, right_index=True)\ndf_train = df_train.fillna(-1)\ndf_train = df_train.replace([np.inf, -np.inf], -1)\ndf_train.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6ta Oportunidad de Optimización","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_selection import RFE\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\n\n# \ntarget_column = 'target'\nfeatures = [col for col in df_train.columns if col != target_column]\n\n# We want to standardize features before passing them to the model\nscaler = StandardScaler()\n\nn_repeticiones = 3\n\n# Lista para almacenar los tiempos de ejecución secuenciales\ntiempos_secuenciales_6 = []\n\n# Lista para almacenar los tiempos de ejecución paralelos\ntiempos_paralelos_6 = []\n\nfor _ in range(n_repeticiones):\n    # --- Código secuencial ---\n    BN6_start_time = time.perf_counter()\n    rfe = RFE(estimator=RandomForestClassifier(), # RFE will use RandomForestClassifier as a base model to determine features importance\n                n_features_to_select=225, # number of features we want to keep\n                step=50) # how many features to remove at each step\n    \n    rfe.fit(scaler.fit_transform(df_train[features]), \n        df_train[target_column])\n\n    selected_features = list(np.array(features)[rfe.support_]) # Get the selected features\n    \n    BN6_end_time = time.perf_counter()\n    BN6_elapsed_time = BN6_end_time - BN6_start_time\n    tiempos_secuenciales_6.append(BN6_elapsed_time)\n\n\n\n    # --- Código paralelo ---\n    PA6_start_time = time.perf_counter()\n    rfe = RFE(estimator=RandomForestClassifier(n_jobs=-1), # RFE will use RandomForestClassifier as a base model to determine features importance\n                n_features_to_select=225, # number of features we want to keep\n                step=50) # how many features to remove at each step\n\n    rfe.fit(scaler.fit_transform(df_train[features]), \n        df_train[target_column])\n    \n    selected_features = list(np.array(features)[rfe.support_]) # Get the selected features\n    \n    PA6_end_time = time.perf_counter()\n    PA6_elapsed_time = PA6_end_time - PA6_start_time\n    tiempos_paralelos_6.append(PA6_elapsed_time)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calcular el tiempo promedio para ambos casos\ntiempo_secuencial_promedio = sum(tiempos_secuenciales_6) / n_repeticiones\ntiempo_paralelo_promedio = sum(tiempos_paralelos_6) / n_repeticiones\n\nnum_cores = os.cpu_count()\n\nspeedup = tiempo_secuencial_promedio / tiempo_paralelo_promedio\nefficiency = speedup / num_cores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Tiempo secuencial promedio: {tiempo_secuencial_promedio:.4f} segundos\")\nprint(f\"Tiempo paralelo promedio: {tiempo_paralelo_promedio:.4f} segundos\")\nprint(f\"Speedup: {speedup:.4f}\")\nprint(f\"Eficiencia: {efficiency:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## ANALISIS DE ESCALABILIDAD\nworkers_threads_start_time = time.perf_counter()\nrfe = RFE(estimator=RandomForestClassifier(n_jobs=-1),\n                n_features_to_select=225, \n                step=50) \n\nrfe.fit(scaler.fit_transform(df_train[features]), \n        df_train[target_column])\n    \nselected_features = list(np.array(features)[rfe.support_])\nworkers_threads_end_time = time.perf_counter()\nelapsed_WT = workers_threads_end_time - workers_threads_start_time\nprint(f\"Tiempo con {nworkers} workers y {nthreads} hilos: {elapsed_WT:.4f} segundos\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = df_train[selected_features]\ndf_test = df_test[selected_features]\ndf_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.pipeline import Pipeline\n\npipeline = Pipeline([\n    ('scaler', StandardScaler()),\n    ('pca', PCA())  # PCA without specifying n_components\n])\n\npipeline.fit(df_train)\n\n# Plot cumulative explained variance\npca = pipeline.named_steps['pca']\nexplained_variance_ratio = pca.explained_variance_ratio_\ncumulative_explained_variance = explained_variance_ratio.cumsum()\n\nplt.plot(cumulative_explained_variance)\nplt.xlabel('Number of Principal Components')\nplt.ylabel('Cumulative Explained Variance')\nplt.axhline(y=0.95, color='r', linestyle='--', label='95% Explained Variance')\nplt.axhline(y=0.99, color='g', linestyle='--', label='99% Explained Variance')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select the desired number of components based on your analysis\ndesired_num_components = 100\n\n# Fit PCA with the desired number of components\npipeline = Pipeline([\n    ('scaler', StandardScaler()),\n    ('pca', PCA(n_components=desired_num_components))\n])\n\n# Fit the pipeline on the training data\npipeline.fit(df_train)\n\ntrain_pca = pipeline.transform(df_train)\ndf_train = pd.DataFrame(train_pca, columns=[f'PC{i+1}' for i in range(desired_num_components)], index=df_train.index)\n\ndf_test = df_test.fillna(-1)\ndf_test = df_test.replace([np.inf, -np.inf], -1)\ntest_pca = pipeline.transform(df_test)\ndf_test = pd.DataFrame(test_pca, columns=[f'PC{i+1}' for i in range(desired_num_components)], index=df_test.index)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del scaler\ndel rfe\ndel selected_features\ndel pipeline\ndel pca\ndel explained_variance_ratio\ndel cumulative_explained_variance\ndel desired_num_components\ndel train_pca\ndel test_pca\ndel target_column\ndel features\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Modelamiento","metadata":{}},{"cell_type":"markdown","source":"### Baseline","metadata":{}},{"cell_type":"markdown","source":"## 7ma Oportunidad de Optimización","metadata":{}},{"cell_type":"code","source":"## SECUENCIAL\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## PARALELA\ndef amex_metric_dask(y_true: dd.DataFrame, y_pred: dd.DataFrame) -> float:\n    \"\"\"\n    Calcula la métrica AMEX utilizando Dask para optimizar los cálculos paralelizados.\n    \"\"\"\n\n    def top_four_percent_captured_dask(y_true: dd.DataFrame, y_pred: dd.DataFrame) -> float:\n        \"\"\"\n        Calcula la métrica de top cuatro por ciento capturado usando Dask.\n        \"\"\"\n        df = dd.concat([y_true, y_pred], axis=1).rename(columns={0: 'target', 1: 'prediction'})\n        df = df.sort_values('prediction', ascending=False)\n        df['weight'] = df['target'].apply(lambda x: 20 if x == 0 else 1, meta=('weight', 'int'))\n        four_pct_cutoff = (0.04 * df['weight'].sum()).compute()  # Se calcula el límite del 4% acumulado.\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df[df['weight_cumsum'] <= four_pct_cutoff]\n        captured = (df_cutoff['target'] == 1).sum().compute()\n        total_positives = (df['target'] == 1).sum().compute()\n        return captured / total_positives\n\n    def weighted_gini_dask(y_true: dd.DataFrame, y_pred: dd.DataFrame) -> float:\n        \"\"\"\n        Calcula el Gini ponderado utilizando Dask.\n        \"\"\"\n        df = dd.concat([y_true, y_pred], axis=1).rename(columns={0: 'target', 1: 'prediction'})\n        df = df.sort_values('prediction', ascending=False)\n        df['weight'] = df['target'].apply(lambda x: 20 if x == 0 else 1, meta=('weight', 'int'))\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum().compute()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum().compute()\n\n    def normalized_weighted_gini_dask(y_true: dd.DataFrame, y_pred: dd.DataFrame) -> float:\n        \"\"\"\n        Calcula el Gini ponderado normalizado utilizando Dask.\n        \"\"\"\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini_dask(y_true, y_pred) / weighted_gini_dask(y_true, y_true_pred)\n\n    # Calcular las métricas utilizando Dask\n    g = normalized_weighted_gini_dask(y_true, y_pred)\n    d = top_four_percent_captured_dask(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cv_amex_scores(df, model_1, model_2, features, target_column):\n    \n    kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n    scores = []\n    X = df[features]\n    y = df[target_column]\n\n    i = 0\n    \n    # Perform k-fold cross-validation\n    for train_idx, test_idx in kf.split(X, y):\n        \n        print(f\"Run {i+1}-fold CV...\")\n\n        X_train = X.iloc[train_idx]\n        X_test = X.iloc[test_idx]\n        y_train = y.iloc[train_idx]\n        y_test = y.iloc[test_idx]\n\n        # Train the models\n        model_1.fit(X_train, y_train)\n        model_2.fit(X_train, y_train)\n\n        # Predict probabilities for the test set\n        y_prob = 0.5 * (\n            model_1.predict_proba(X_test)[:, 1] + \n            model_2.predict_proba(X_test)[:, 1])\n        \n        pred_df = pd.DataFrame(y_prob, columns=['prediction'], index=y_test.index)\n        true_df = pd.DataFrame(y_test, columns=['target'])\n\n        result_df = pd.merge(pred_df, true_df, left_index=True, right_index=True)\n        plot_roc_auc(result_df)\n        \n        model_score = amex_metric(true_df, pred_df)\n        scores.append(model_score)\n        \n        print(f\"Completed {i+1}-fold CV. Score = {model_score}\")\n        i += 1\n\n    return scores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cv_amex_scores_using_parallel(df, model_1, model_2, features, target_column):\n    \n    kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n    scores = []\n    X = df[features]\n    y = df[target_column]\n\n    i = 0\n    \n    # Perform k-fold cross-validation\n    for train_idx, test_idx in kf.split(X, y):\n        \n        print(f\"Run {i+1}-fold CV...\")\n\n        X_train = X.iloc[train_idx]\n        X_test = X.iloc[test_idx]\n        y_train = y.iloc[train_idx]\n        y_test = y.iloc[test_idx]\n\n        # Train the models\n        model_1.fit(X_train, y_train)\n        model_2.fit(X_train, y_train)\n\n        # Predict probabilities for the test set\n        y_prob = 0.5 * (\n            model_1.predict_proba(X_test)[:, 1] + \n            model_2.predict_proba(X_test)[:, 1])\n        \n        pred_df = pd.DataFrame(y_prob, columns=['prediction'], index=y_test.index)\n        true_df = pd.DataFrame(y_test, columns=['target'])\n\n        result_df = pd.merge(pred_df, true_df, left_index=True, right_index=True)\n        plot_roc_auc(result_df)\n        \n        model_score = amex_metric_dask(true_df, pred_df)\n        scores.append(model_score)\n        \n        print(f\"Completed {i+1}-fold CV. Score = {model_score}\")\n        i += 1\n\n    return scores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pipeline = Pipeline([\n    ('scaler', StandardScaler()),\n    ('model', RandomForestClassifier())\n])\n\nlabels = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\", index_col=\"customer_ID\")\ndf_train = pd.merge(df_train, labels, left_index=True, right_index=True)\n\ntarget_column = 'target'\nfeatures = [col for col in df_train.columns if col != target_column]\n\nn_repeticiones = 3\n\n# Lista para almacenar los tiempos de ejecución secuenciales\ntiempos_secuenciales_7 = []\n\n# Lista para almacenar los tiempos de ejecución paralelos\ntiempos_paralelos_7 = []\n\nfor _ in range(n_repeticiones):\n    # --- Código secuencial ---\n    BN7_start_time = time.perf_counter()\n    scores = cv_amex_scores(df_train, pipeline, features, target_column)\n    BN7_end_time = time.perf_counter()\n    BN7_elapsed_time = BN7_end_time - BN7_start_time\n    tiempos_secuenciales_7.append(BN7_elapsed_time)\n\n\n\n    # --- Código paralelo ---\n    PA7_start_time = time.perf_counter()\n    scores = cv_amex_scores_using_parallel(df_train, pipeline, features, target_column)\n    PA7_end_time = time.perf_counter()\n    PA7_elapsed_time = PA7_end_time - PA7_start_time\n    tiempos_paralelos_7.append(PA7_elapsed_time)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calcular el tiempo promedio para ambos casos\ntiempo_secuencial_promedio = sum(tiempos_secuenciales_7) / n_repeticiones\ntiempo_paralelo_promedio = sum(tiempos_paralelos_7) / n_repeticiones\n\nnum_cores = os.cpu_count()\n\nspeedup = tiempo_secuencial_promedio / tiempo_paralelo_promedio\nefficiency = speedup / num_cores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Tiempo secuencial promedio: {tiempo_secuencial_promedio:.4f} segundos\")\nprint(f\"Tiempo paralelo promedio: {tiempo_paralelo_promedio:.4f} segundos\")\nprint(f\"Speedup: {speedup:.4f}\")\nprint(f\"Eficiencia: {efficiency:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## ANALISIS DE ESCALABILIDAD\nworkers_threads_start_time = time.perf_counter()\n# blocksize controla el tamaño de cada partición del DataFrame. Es importante elegir un tamaño adecuado para un rendimiento óptimo.\nscores = cv_amex_scores_using_parallel(df_train, pipeline, features, target_column)\nworkers_threads_end_time = time.perf_counter()\nelapsed_WT = workers_threads_end_time - workers_threads_start_time\nprint(f\"Tiempo con {nworkers} workers y {nthreads} hilos: {elapsed_WT:.4f} segundos\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Here's a method we will use to plot the ROC curve.\n\ndef plot_roc_auc(df):\n\n    from sklearn.metrics import roc_curve, auc\n\n    # Extract predicted probabilities and true labels\n    pred_prob = df['prediction'].values\n    true_labels = df['target'].values\n\n    # Compute ROC curve\n    fpr, tpr, _ = roc_curve(true_labels, pred_prob)\n    roc_auc = auc(fpr, tpr)\n\n    # Plot ROC curve\n    plt.figure()\n    plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = {:.2f})'.format(roc_auc))\n    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n    plt.axvline(x=0.04, color='red', linestyle='--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic')\n    plt.legend(loc=\"lower right\")\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Method to perform CV\n\nfrom sklearn.model_selection import StratifiedKFold\n\ndef cv_amex_scores(df, model, features, target_column):\n    \n    kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n    scores = []\n    X = df[features]\n    y = df[target_column]\n\n    i = 0\n    \n    # Perform k-fold cross-validation\n    for train_idx, test_idx in kf.split(X, y):\n        \n        print(f\"Run {i+1}-fold CV...\")\n\n        X_train = X.iloc[train_idx]\n        X_test = X.iloc[test_idx]\n        y_train = y.iloc[train_idx]\n        y_test = y.iloc[test_idx]\n\n        # Train the model\n        model.fit(X_train, y_train)\n\n        # Predict probabilities for the test set\n        y_prob = model.predict_proba(X_test)[:, 1]\n        pred_df = pd.DataFrame(y_prob, columns=['prediction'], index=y_test.index)\n        true_df = pd.DataFrame(y_test, columns=['target'])\n\n        result_df = pd.merge(pred_df, true_df, left_index=True, right_index=True)\n        plot_roc_auc(result_df)\n        \n        model_score = amex_metric(true_df, pred_df)\n        scores.append(model_score)\n        \n        print(f\"Completed {i+1}-fold CV. Score = {model_score}\")\n        i += 1\n\n    return scores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Baseline score\nnp.mean(scores)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='weak_train'></a>\n### Weak Learners - Training","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\nfrom catboost import CatBoostClassifier, Pool\nimport xgboost as xgb\nfrom sklearn.model_selection import train_test_split\n\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(df_train[features])\n\nX_train_split, X_val_split, y_train_split, y_val_split = train_test_split(\n    X_train_scaled, df_train[target_column], test_size=0.2, random_state=42\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Light GBM","metadata":{}},{"cell_type":"code","source":"# Train LightGBM\nlgb_params = {\n    'objective': 'binary',\n    'boosting_type': 'gbdt',\n    'metric': 'auc',\n    'num_iterations': 30\n}\n\nlgb_train = lgb.Dataset(X_train_split, y_train_split)\nlgb_val = lgb.Dataset(X_val_split, y_val_split, reference=lgb_train)\n\nevals={}\nlgb_model = lgb.train(params=lgb_params, \n                train_set = lgb_train, \n                valid_sets=[lgb_train, lgb_val],\n                callbacks = [lgb.record_evaluation(evals)])\n\nlgb.plot_metric(evals)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### CatBoost","metadata":{}},{"cell_type":"code","source":"# Train CatBoost\ncatboost_params = {\n    'loss_function': 'Logloss',\n    'eval_metric': 'AUC',\n    'iterations': 50\n}\ncatboost_model = CatBoostClassifier(**catboost_params)\ncatboost_model.fit(X_train_split, \n                   y_train_split, \n                   eval_set=(X_val_split, y_val_split), \n                   early_stopping_rounds=10)\n\n# Build pool datasets and get AUC scores over iterations\ntrain_pool = Pool(data=X_train_split, label=y_train_split)\nval_pool = Pool(data=X_val_split, label=y_val_split)\nauc_train_values = catboost_model.eval_metrics(train_pool, metrics=['AUC'])['AUC']\nauc_val_values = catboost_model.eval_metrics(val_pool, metrics=['AUC'])['AUC']\n\n# Get the iteration numbers\niterations = np.arange(len(auc_train_values))\n\n# Plot AUC vs iterations\nplt.figure(figsize=(10, 6))\nplt.plot(iterations, auc_train_values, label='Train AUC')\nplt.plot(iterations, auc_val_values, label='Validation AUC')\nplt.xlabel('Iterations')\nplt.ylabel('AUC')\nplt.legend()\nplt.grid()\nplt.title('AUC vs Iterations for CatBoost')\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### XGBoost","metadata":{}},{"cell_type":"code","source":"# Train XGBoost\nxgb_params = {\n    'objective': 'binary:logistic',\n    'eval_metric': 'auc',\n    'n_estimators': 20\n}\n\nxgb_model = xgb.XGBClassifier(**xgb_params)\nxgb_model.fit(X_train_split, \n          y_train_split, \n          eval_set=[(X_train_split, y_train_split), (X_val_split, y_val_split)])\n\ntrain_history = xgb_model.evals_result()\niterations = len(train_history[\"validation_0\"][\"auc\"])\n\nplt.plot(range(1,iterations+1), train_history[\"validation_0\"][\"auc\"], label=\"training\")\nplt.plot(range(1,iterations+1), train_history[\"validation_1\"][\"auc\"], label=\"validation\")\nplt.xlabel('Iterations')\nplt.ylabel('AUC')\nplt.legend()\nplt.grid()\nplt.title('AUC vs Iterations for XGBoost')\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Weak Learners - Predictions","metadata":{}},{"cell_type":"code","source":"X_test_scaled = scaler.fit_transform(df_test[features])\nX_test_scaled.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_pred_train = lgb_model.predict(X_train_scaled)\nlgb_pred_test = lgb_model.predict(X_test_scaled)\ncatboost_pred_train = catboost_model.predict_proba(X_train_scaled)[:, 1]\ncatboost_pred_test = catboost_model.predict_proba(X_test_scaled)[:, 1]\nxgb_pred_train = xgb_model.predict_proba(X_train_scaled)\nxgb_pred_test = xgb_model.predict_proba(X_test_scaled)\n\npredictions_df_train = pd.DataFrame({\n    'lightgbm_pred': lgb_pred_train,\n    'catboost_pred': catboost_pred_train,\n    'xgboost_pred': [x[1] for x in xgb_pred_train],\n    'target': df_train[target_column]\n})\npredictions_df_test = pd.DataFrame({\n    'lightgbm_pred': lgb_pred_test,\n    'catboost_pred': catboost_pred_test,\n    'xgboost_pred': [x[1] for x in xgb_pred_test]\n})","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# LGB SCORE (on training set)\npred_df = pd.DataFrame(lgb_pred_train, columns=['prediction'], index=df_train.index)\ntrue_df = pd.DataFrame(df_train[target_column], columns=['target'])\n\namex_metric(true_df, pred_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CATBOOST SCORE (on training set)\npred_df = pd.DataFrame(catboost_pred_train, columns=['prediction'], index=df_train.index)\ntrue_df = pd.DataFrame(df_train[target_column], columns=['target'])\n\namex_metric(true_df, pred_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# XGBOOST SCORE (on training set)\npred_df = pd.DataFrame(predictions_df_train[\"xgboost_pred\"].values, columns=['prediction'], index=df_train.index)\ntrue_df = pd.DataFrame(df_train[target_column], columns=['target'])\n\namex_metric(true_df, pred_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Strong Learners - Training","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression\n\npred_features = ['lightgbm_pred', 'catboost_pred', 'xgboost_pred']\ntarget_col = \"target\"\n\n# Train RandomForestClassifier\nrf_clf = RandomForestClassifier()\nrf_clf.fit(predictions_df_train[pred_features], predictions_df_train[target_col])\n\n# Train LogisticRegression\nlr_clf = LogisticRegression()\nlr_clf.fit(predictions_df_train[pred_features], predictions_df_train[target_col])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scores = cv_amex_scores(predictions_df_train, RandomForestClassifier(), LogisticRegression(), pred_features, target_col)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Final score\nnp.mean(scores)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Strong Learners - Prediction","metadata":{}},{"cell_type":"code","source":"# Predict probabilities using both models\nrf_pred_proba = rf_clf.predict_proba(predictions_df_test[pred_features])[:, 1]\nlr_pred_proba = lr_clf.predict_proba(predictions_df_test[pred_features])[:, 1]\n\n# Average the predicted probabilities (50% weight each)\nensemble_pred_proba = 0.5 * (rf_pred_proba + lr_pred_proba)\n\npred_df = pd.DataFrame(ensemble_pred_proba, columns=['prediction'], index=df_test.index)\npred_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_df.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"end_time = time.perf_counter()\nelapsed_time = end_time - start_time\n\nprint(f\"Tiempo de ejecución: {elapsed_time} segundos\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}