{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Inicio de trabajo final\n# Análisis de rendimiento inicial","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## 1. EDA y Optimización de memoria","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport gc\nimport time\nprint(\"Librerias importadas\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Iniciar la medición del tiempo de ejecución\nstart_time = time.perf_counter()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\")\nsns.countplot(x=labels[\"target\"])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customer_date_df = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=[x for x in range(2)])\ncustomer_date_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customer_date_df.info(memory_usage='deep')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This show the number of unique values for each column in the dataframe\ncustomer_date_df.nunique()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert customer_ID to int\nunique_ids = customer_date_df['customer_ID'].unique()\nid_mapping = {_id: i for i, _id in enumerate(unique_ids)}\ncustomer_date_df['customer_ID'] = customer_date_df['customer_ID'].map(id_mapping).astype(pd.Int32Dtype())\ncustomer_date_df.tail()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Split the 'date' column into day, month, and year columns\ncustomer_date_df[['year', 'month', 'day']] = customer_date_df['S_2'].str.split('-', expand=True)\ncustomer_date_df = customer_date_df.drop(\"S_2\", axis=1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert the new columns to integers\ncustomer_date_df[\"year\"] = customer_date_df[\"year\"].astype(pd.Int32Dtype())\ncustomer_date_df[\"year\"] -= 2000\nfor col in [\"year\", \"month\", \"day\"]:\n    customer_date_df[col] = customer_date_df[col].astype(pd.Int8Dtype())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customer_date_df.info(memory_usage='deep')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_null_values(df):\n    null_values = df.isnull().mean() * 100\n    null_values = null_values.sort_values(ascending=False)\n    null_values = null_values[null_values > 0]\n    \n    if not null_values.empty:\n        sns.barplot(y=null_values.index, x=null_values.values)\n    else:\n        print(\"No null values\")\n\nplot_null_values(customer_date_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del customer_date_df\ndel unique_ids\ndel id_mapping\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_columns = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n\n# let's just load a couple\ncat_df = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=categorical_columns[:2])\ncat_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_df.info(memory_usage='deep')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We can map each value of the categorical features to a sequential integer\nfor col in cat_df.columns:\n    unique = cat_df[col].unique()\n    mapping = {val: i for i, val in enumerate(unique)}\n    cat_df[col] = cat_df[col].map(mapping).astype(pd.Int8Dtype())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_df.info(memory_usage='deep')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_null_values(cat_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_df.nunique()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.countplot(data=cat_df, y=\"B_38\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del categorical_columns\ndel cat_df\ndel unique\ndel mapping\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Let's load the customer_ID so that we can join with labels and a single numerical feature\nload_cols = [\"customer_ID\", \"D_130\"]\n\nnum_df = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=load_cols)\nnum_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_df.info(memory_usage='deep')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_null_values(num_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.histplot(data=num_df.sample(100000), x=\"D_130\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.histplot(data=num_df[num_df[\"D_130\"] < 0.1], x=\"D_130\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.histplot(data=num_df[num_df[\"D_130\"] > 1], x=\"D_130\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_df[\"D_130\"] = (np.floor(num_df[\"D_130\"]+1e-6)).fillna(-1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.histplot(data=num_df.sample(100000), x=\"D_130\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_df[\"D_130\"].value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"merge_df = pd.merge(num_df, labels, on=\"customer_ID\")\nmerge_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"merge_df.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_eda(data, feature, target):\n    fig, ax = plt.subplots(1,3, figsize=(15, 6))\n    sns.histplot(data=data, x=feature, ax=ax[0])\n    sns.histplot(data=data, x=feature, hue=target, ax=ax[1])\n    sns.regplot(data=data, x=feature, y=target, ax=ax[2])\n    \nplot_eda(merge_df.sample(100000), \"D_130\", \"target\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"load_cols = [\"customer_ID\", \"P_2\"]\n\nnum_df = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=load_cols)\nnum_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"merge_df = pd.merge(num_df, labels, on=\"customer_ID\")\nmerge_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_eda(merge_df.sample(100000), \"P_2\", \"target\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del load_cols\ndel num_df\ndel merge_df\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## 2. Preprocesamiento & ingeniería de características","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_parquet(\"/kaggle/input/amex-data-integer-dtypes-parquet-format/train.parquet\")\ndf_train = df_train.set_index(\"customer_ID\")\ndf_train.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = pd.read_parquet(\"/kaggle/input/amex-data-integer-dtypes-parquet-format/test.parquet\")\ndf_test = df_test.set_index(\"customer_ID\")\ndf_test.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\", index_col=\"customer_ID\")\nlabels.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.merge(df_train, labels, left_index=True, right_index=True)\ndf_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_proportion = df_train[\"target\"].sum() / df_train.shape[0]\nclass_proportion","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 10k rows subsample\nsample_size = 10000\n\n# For our training dataset we are going to maintain the same positive/negative sample ratio!\nsample_df_positive = df_train[df_train[\"target\"] == 1].sample(int(sample_size * class_proportion))\nsample_df_negative = df_train[df_train[\"target\"] == 0].sample(sample_size - sample_df_positive.shape[0])\n\n# Concat and then shuffle\ndf_train = pd.concat([sample_df_positive, sample_df_negative])\ndf_train = df_train.sample(frac=1, random_state=42)\n\ndf_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = df_test.sample(10000)\ndf_test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\ndate_col = \"S_2\"\nid_col = \"customer_ID\"\ntarget_col = \"target\"\nnumerical_features = [x for x in df_train.columns if x not in categorical_features and x != date_col and x != id_col and x != target_col]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unique_categories = {}\nfor col in categorical_features:\n    unique_categories[col] = set(df_train[col].unique()).union(set(df_test[col].unique()))\n\n# Combine unique categories and create a mapping to integers\ncategory_to_int = {col: {category: i for i, category in enumerate(categories)} for col, categories in unique_categories.items()}\ncategory_to_int","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def one_hot_encoding(df, categorical_features, category_to_int):\n    df[categorical_features] = df[categorical_features].apply(lambda col: col.map(category_to_int[col.name]))\n    df = pd.get_dummies(df, columns=categorical_features)\n    \n    ohe_cols = []\n    \n    for col in categorical_features:\n        for k,v in category_to_int[col].items():\n            ohe_cols.append(f\"{col}_{v}\")\n            if f\"{col}_{v}\" not in df.columns:\n                df[f\"{col}_{v}\"] = 0\n    \n    return df, ohe_cols\n\ndf_train, _ = one_hot_encoding(df_train, categorical_features, category_to_int)\ndf_test, ohe_cols = one_hot_encoding(df_test, categorical_features, category_to_int)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aggregated_cat_df_train = df_train.groupby('customer_ID')[ohe_cols].agg(['sum'])\naggregated_cat_df_train.columns = aggregated_cat_df_train.columns.map('_'.join)\naggregated_cat_df_train = aggregated_cat_df_train.reset_index()\naggregated_cat_df_train.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aggregated_cat_df_test = df_test.groupby('customer_ID')[ohe_cols].agg(['sum'])\naggregated_cat_df_test.columns = aggregated_cat_df_test.columns.map('_'.join)\naggregated_cat_df_test = aggregated_cat_df_test.reset_index()\naggregated_cat_df_test.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aggregated_num_df_train = df_train.groupby('customer_ID')[numerical_features].agg(['sum', 'mean'])\naggregated_num_df_train.columns = aggregated_num_df_train.columns.map('_'.join)\naggregated_num_df_train = aggregated_num_df_train.reset_index()\naggregated_num_df_train.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aggregated_num_df_test = df_test.groupby('customer_ID')[numerical_features].agg(['sum', 'mean'])\naggregated_num_df_test.columns = aggregated_num_df_test.columns.map('_'.join)\naggregated_num_df_test = aggregated_num_df_test.reset_index()\naggregated_num_df_test.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"agg_numerical_features = []\nfor f in numerical_features:\n    for a in ['sum', 'mean']:\n        agg_numerical_features.append(f\"{f}_{a}\")\n\ndef feature_transformation(df):\n        \n    # Feature transformation\n    new_columns = []\n\n    import warnings\n\n    for feature in agg_numerical_features:\n\n        with warnings.catch_warnings():\n            warnings.filterwarnings('ignore', category=RuntimeWarning)\n            log_feature = np.log1p(df[feature])\n            square_feature = df[feature] ** 2\n\n        log_feature.name = f'{feature}_log'\n        square_feature.name = f'{feature}_square'\n\n        # Append the series to the list\n        new_columns.extend([log_feature, square_feature])\n    \n    df = pd.concat([df, *new_columns], axis=1)\n    return df\n\ntransformed_num_df_train = feature_transformation(aggregated_num_df_train)\ntransformed_num_df_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transformed_num_df_test = feature_transformation(aggregated_num_df_test)\ntransformed_num_df_test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.merge(aggregated_cat_df_train, transformed_num_df_train, left_on=\"customer_ID\", right_on=\"customer_ID\")\ndf_train = df_train.set_index(\"customer_ID\")\ndf_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = pd.merge(aggregated_cat_df_test, transformed_num_df_test, left_on=\"customer_ID\", right_on=\"customer_ID\")\ndf_test = df_test.set_index(\"customer_ID\")\ndf_test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del labels\ndel class_proportion\ndel numerical_features\ndel target_col\ndel id_col\ndel date_col\ndel categorical_features\ndel sample_size\ndel sample_df_positive\ndel sample_df_negative\ndel unique_categories\ndel category_to_int\ndel ohe_cols\ndel aggregated_cat_df_train\ndel aggregated_cat_df_test\ndel aggregated_num_df_train\ndel aggregated_num_df_test\ndel agg_numerical_features\ndel transformed_num_df_train\ndel transformed_num_df_test\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Selección de características","metadata":{}},{"cell_type":"code","source":"labels = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\", index_col=\"customer_ID\")\ndf_train = pd.merge(df_train, labels, left_index=True, right_index=True)\ndf_train = df_train.fillna(-1)\ndf_train = df_train.replace([np.inf, -np.inf], -1)\ndf_train.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.feature_selection import RFE\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\n\ntarget_column = 'target'\nfeatures = [col for col in df_train.columns if col != target_column]\n\n# We want to standardize features before passing them to the model\nscaler = StandardScaler()\n\n# RFE will use RandomForestClassifier as a base model to determine features importance\nrfe = RFE(estimator=RandomForestClassifier(), \n          n_features_to_select=225, # number of features we want to keep\n          step=50) # how many features to remove at each step\n\nrfe.fit(scaler.fit_transform(df_train[features]), \n        df_train[target_column])\n\n# Get the selected features\nselected_features = list(np.array(features)[rfe.support_])\n\n# Print the selected features\nprint(f\"Selected Features: {len(selected_features)}\")\nprint(selected_features)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = df_train[selected_features]\ndf_test = df_test[selected_features]\ndf_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.pipeline import Pipeline\n\npipeline = Pipeline([\n    ('scaler', StandardScaler()),\n    ('pca', PCA())  # PCA without specifying n_components\n])\n\npipeline.fit(df_train)\n\n# Plot cumulative explained variance\npca = pipeline.named_steps['pca']\nexplained_variance_ratio = pca.explained_variance_ratio_\ncumulative_explained_variance = explained_variance_ratio.cumsum()\n\nplt.plot(cumulative_explained_variance)\nplt.xlabel('Number of Principal Components')\nplt.ylabel('Cumulative Explained Variance')\nplt.axhline(y=0.95, color='r', linestyle='--', label='95% Explained Variance')\nplt.axhline(y=0.99, color='g', linestyle='--', label='99% Explained Variance')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select the desired number of components based on your analysis\ndesired_num_components = 100\n\n# Fit PCA with the desired number of components\npipeline = Pipeline([\n    ('scaler', StandardScaler()),\n    ('pca', PCA(n_components=desired_num_components))\n])\n\n# Fit the pipeline on the training data\npipeline.fit(df_train)\n\ntrain_pca = pipeline.transform(df_train)\ndf_train = pd.DataFrame(train_pca, columns=[f'PC{i+1}' for i in range(desired_num_components)], index=df_train.index)\n\ndf_test = df_test.fillna(-1)\ndf_test = df_test.replace([np.inf, -np.inf], -1)\ntest_pca = pipeline.transform(df_test)\ndf_test = pd.DataFrame(test_pca, columns=[f'PC{i+1}' for i in range(desired_num_components)], index=df_test.index)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del scaler\ndel rfe\ndel selected_features\ndel pipeline\ndel pca\ndel explained_variance_ratio\ndel cumulative_explained_variance\ndel desired_num_components\ndel train_pca\ndel test_pca\ndel target_column\ndel features\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Modelamiento\n### Baseline","metadata":{}},{"cell_type":"code","source":"def amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Here's a method we will use to plot the ROC curve.\n\ndef plot_roc_auc(df):\n\n    from sklearn.metrics import roc_curve, auc\n\n    # Extract predicted probabilities and true labels\n    pred_prob = df['prediction'].values\n    true_labels = df['target'].values\n\n    # Compute ROC curve\n    fpr, tpr, _ = roc_curve(true_labels, pred_prob)\n    roc_auc = auc(fpr, tpr)\n\n    # Plot ROC curve\n    plt.figure()\n    plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = {:.2f})'.format(roc_auc))\n    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n    plt.axvline(x=0.04, color='red', linestyle='--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic')\n    plt.legend(loc=\"lower right\")\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Method to perform CV\n\nfrom sklearn.model_selection import StratifiedKFold\n\ndef cv_amex_scores(df, model, features, target_column):\n    \n    kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n    scores = []\n    X = df[features]\n    y = df[target_column]\n\n    i = 0\n    \n    # Perform k-fold cross-validation\n    for train_idx, test_idx in kf.split(X, y):\n        \n        print(f\"Run {i+1}-fold CV...\")\n\n        X_train = X.iloc[train_idx]\n        X_test = X.iloc[test_idx]\n        y_train = y.iloc[train_idx]\n        y_test = y.iloc[test_idx]\n\n        # Train the model\n        model.fit(X_train, y_train)\n\n        # Predict probabilities for the test set\n        y_prob = model.predict_proba(X_test)[:, 1]\n        pred_df = pd.DataFrame(y_prob, columns=['prediction'], index=y_test.index)\n        true_df = pd.DataFrame(y_test, columns=['target'])\n\n        result_df = pd.merge(pred_df, true_df, left_index=True, right_index=True)\n        plot_roc_auc(result_df)\n        \n        model_score = amex_metric(true_df, pred_df)\n        scores.append(model_score)\n        \n        print(f\"Completed {i+1}-fold CV. Score = {model_score}\")\n        i += 1\n\n    return scores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pipeline = Pipeline([\n    ('scaler', StandardScaler()),\n    ('model', RandomForestClassifier())\n])\n\nlabels = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\", index_col=\"customer_ID\")\ndf_train = pd.merge(df_train, labels, left_index=True, right_index=True)\n\ntarget_column = 'target'\nfeatures = [col for col in df_train.columns if col != target_column]\n\nscores = cv_amex_scores(df_train, pipeline, features, target_column)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Baseline score\nnp.mean(scores)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='weak_train'></a>\n### Weak Learners - Training","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\nfrom catboost import CatBoostClassifier, Pool\nimport xgboost as xgb\nfrom sklearn.model_selection import train_test_split\n\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(df_train[features])\n\nX_train_split, X_val_split, y_train_split, y_val_split = train_test_split(\n    X_train_scaled, df_train[target_column], test_size=0.2, random_state=42\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Light GBM","metadata":{}},{"cell_type":"code","source":"# Train LightGBM\nlgb_params = {\n    'objective': 'binary',\n    'boosting_type': 'gbdt',\n    'metric': 'auc',\n    'num_iterations': 30\n}\n\nlgb_train = lgb.Dataset(X_train_split, y_train_split)\nlgb_val = lgb.Dataset(X_val_split, y_val_split, reference=lgb_train)\n\nevals={}\nlgb_model = lgb.train(params=lgb_params, \n                train_set = lgb_train, \n                valid_sets=[lgb_train, lgb_val],\n                callbacks = [lgb.record_evaluation(evals)])\n\nlgb.plot_metric(evals)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### CatBoost","metadata":{}},{"cell_type":"code","source":"# Train CatBoost\ncatboost_params = {\n    'loss_function': 'Logloss',\n    'eval_metric': 'AUC',\n    'iterations': 50\n}\ncatboost_model = CatBoostClassifier(**catboost_params)\ncatboost_model.fit(X_train_split, \n                   y_train_split, \n                   eval_set=(X_val_split, y_val_split), \n                   early_stopping_rounds=10)\n\n# Build pool datasets and get AUC scores over iterations\ntrain_pool = Pool(data=X_train_split, label=y_train_split)\nval_pool = Pool(data=X_val_split, label=y_val_split)\nauc_train_values = catboost_model.eval_metrics(train_pool, metrics=['AUC'])['AUC']\nauc_val_values = catboost_model.eval_metrics(val_pool, metrics=['AUC'])['AUC']\n\n# Get the iteration numbers\niterations = np.arange(len(auc_train_values))\n\n# Plot AUC vs iterations\nplt.figure(figsize=(10, 6))\nplt.plot(iterations, auc_train_values, label='Train AUC')\nplt.plot(iterations, auc_val_values, label='Validation AUC')\nplt.xlabel('Iterations')\nplt.ylabel('AUC')\nplt.legend()\nplt.grid()\nplt.title('AUC vs Iterations for CatBoost')\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### XGBoost","metadata":{}},{"cell_type":"code","source":"# Train XGBoost\nxgb_params = {\n    'objective': 'binary:logistic',\n    'eval_metric': 'auc',\n    'n_estimators': 20\n}\n\nxgb_model = xgb.XGBClassifier(**xgb_params)\nxgb_model.fit(X_train_split, \n          y_train_split, \n          eval_set=[(X_train_split, y_train_split), (X_val_split, y_val_split)])\n\ntrain_history = xgb_model.evals_result()\niterations = len(train_history[\"validation_0\"][\"auc\"])\n\nplt.plot(range(1,iterations+1), train_history[\"validation_0\"][\"auc\"], label=\"training\")\nplt.plot(range(1,iterations+1), train_history[\"validation_1\"][\"auc\"], label=\"validation\")\nplt.xlabel('Iterations')\nplt.ylabel('AUC')\nplt.legend()\nplt.grid()\nplt.title('AUC vs Iterations for XGBoost')\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Weak Learners - Predictions","metadata":{}},{"cell_type":"code","source":"X_test_scaled = scaler.fit_transform(df_test[features])\nX_test_scaled.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_pred_train = lgb_model.predict(X_train_scaled)\nlgb_pred_test = lgb_model.predict(X_test_scaled)\ncatboost_pred_train = catboost_model.predict_proba(X_train_scaled)[:, 1]\ncatboost_pred_test = catboost_model.predict_proba(X_test_scaled)[:, 1]\nxgb_pred_train = xgb_model.predict_proba(X_train_scaled)\nxgb_pred_test = xgb_model.predict_proba(X_test_scaled)\n\npredictions_df_train = pd.DataFrame({\n    'lightgbm_pred': lgb_pred_train,\n    'catboost_pred': catboost_pred_train,\n    'xgboost_pred': [x[1] for x in xgb_pred_train],\n    'target': df_train[target_column]\n})\npredictions_df_test = pd.DataFrame({\n    'lightgbm_pred': lgb_pred_test,\n    'catboost_pred': catboost_pred_test,\n    'xgboost_pred': [x[1] for x in xgb_pred_test]\n})","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# LGB SCORE (on training set)\npred_df = pd.DataFrame(lgb_pred_train, columns=['prediction'], index=df_train.index)\ntrue_df = pd.DataFrame(df_train[target_column], columns=['target'])\n\namex_metric(true_df, pred_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CATBOOST SCORE (on training set)\npred_df = pd.DataFrame(catboost_pred_train, columns=['prediction'], index=df_train.index)\ntrue_df = pd.DataFrame(df_train[target_column], columns=['target'])\n\namex_metric(true_df, pred_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# XGBOOST SCORE (on training set)\npred_df = pd.DataFrame(predictions_df_train[\"xgboost_pred\"].values, columns=['prediction'], index=df_train.index)\ntrue_df = pd.DataFrame(df_train[target_column], columns=['target'])\n\namex_metric(true_df, pred_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Strong Learners - Training","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression\n\npred_features = ['lightgbm_pred', 'catboost_pred', 'xgboost_pred']\ntarget_col = \"target\"\n\n# Train RandomForestClassifier\nrf_clf = RandomForestClassifier()\nrf_clf.fit(predictions_df_train[pred_features], predictions_df_train[target_col])\n\n# Train LogisticRegression\nlr_clf = LogisticRegression()\nlr_clf.fit(predictions_df_train[pred_features], predictions_df_train[target_col])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cv_amex_scores(df, model_1, model_2, features, target_column):\n    \n    kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n    scores = []\n    X = df[features]\n    y = df[target_column]\n\n    i = 0\n    \n    # Perform k-fold cross-validation\n    for train_idx, test_idx in kf.split(X, y):\n        \n        print(f\"Run {i+1}-fold CV...\")\n\n        X_train = X.iloc[train_idx]\n        X_test = X.iloc[test_idx]\n        y_train = y.iloc[train_idx]\n        y_test = y.iloc[test_idx]\n\n        # Train the models\n        model_1.fit(X_train, y_train)\n        model_2.fit(X_train, y_train)\n\n        # Predict probabilities for the test set\n        y_prob = 0.5 * (\n            model_1.predict_proba(X_test)[:, 1] + \n            model_2.predict_proba(X_test)[:, 1])\n        \n        pred_df = pd.DataFrame(y_prob, columns=['prediction'], index=y_test.index)\n        true_df = pd.DataFrame(y_test, columns=['target'])\n\n        result_df = pd.merge(pred_df, true_df, left_index=True, right_index=True)\n        plot_roc_auc(result_df)\n        \n        model_score = amex_metric(true_df, pred_df)\n        scores.append(model_score)\n        \n        print(f\"Completed {i+1}-fold CV. Score = {model_score}\")\n        i += 1\n\n    return scores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scores = cv_amex_scores(predictions_df_train, RandomForestClassifier(), LogisticRegression(), pred_features, target_col)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Final score\nnp.mean(scores)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Strong Learners - Prediction","metadata":{}},{"cell_type":"code","source":"# Predict probabilities using both models\nrf_pred_proba = rf_clf.predict_proba(predictions_df_test[pred_features])[:, 1]\nlr_pred_proba = lr_clf.predict_proba(predictions_df_test[pred_features])[:, 1]\n\n# Average the predicted probabilities (50% weight each)\nensemble_pred_proba = 0.5 * (rf_pred_proba + lr_pred_proba)\n\npred_df = pd.DataFrame(ensemble_pred_proba, columns=['prediction'], index=df_test.index)\npred_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_df.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"end_time = time.perf_counter()\nelapsed_time = end_time - start_time\n\nprint(f\"Tiempo de ejecución: {elapsed_time} segundos\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}