{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-09T04:20:19.136421Z","iopub.execute_input":"2022-10-09T04:20:19.136915Z","iopub.status.idle":"2022-10-09T04:20:19.152212Z","shell.execute_reply.started":"2022-10-09T04:20:19.136816Z","shell.execute_reply":"2022-10-09T04:20:19.151063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.impute import SimpleImputer\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:19.153680Z","iopub.execute_input":"2022-10-09T04:20:19.154033Z","iopub.status.idle":"2022-10-09T04:20:20.121093Z","shell.execute_reply.started":"2022-10-09T04:20:19.154003Z","shell.execute_reply":"2022-10-09T04:20:20.119871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Leer los dataframes de entrada (temporalmente limitado a 100K ya que con más registros se queda trabado el notebook)\nX = pd.read_csv('../input/amex-default-prediction/train_data.csv', nrows=100000)\ny = pd.read_csv('../input/amex-default-prediction/train_labels.csv', nrows=100000)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:20.122528Z","iopub.execute_input":"2022-10-09T04:20:20.122879Z","iopub.status.idle":"2022-10-09T04:20:27.685991Z","shell.execute_reply.started":"2022-10-09T04:20:20.122847Z","shell.execute_reply":"2022-10-09T04:20:27.684715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:27.690020Z","iopub.execute_input":"2022-10-09T04:20:27.691016Z","iopub.status.idle":"2022-10-09T04:20:27.730861Z","shell.execute_reply.started":"2022-10-09T04:20:27.690974Z","shell.execute_reply":"2022-10-09T04:20:27.729709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:27.732292Z","iopub.execute_input":"2022-10-09T04:20:27.732679Z","iopub.status.idle":"2022-10-09T04:20:27.743819Z","shell.execute_reply.started":"2022-10-09T04:20:27.732647Z","shell.execute_reply":"2022-10-09T04:20:27.742510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cantidad Eventos vs No Evento\n## Evento - Default\n## No Evento - No Default\ny.target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:27.745134Z","iopub.execute_input":"2022-10-09T04:20:27.745541Z","iopub.status.idle":"2022-10-09T04:20:27.763290Z","shell.execute_reply.started":"2022-10-09T04:20:27.745506Z","shell.execute_reply":"2022-10-09T04:20:27.761972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Numero de usuarios distintos (No lo significa que existen registro repetidos por usuario)\nlen(X.customer_ID.unique())","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:27.765394Z","iopub.execute_input":"2022-10-09T04:20:27.765969Z","iopub.status.idle":"2022-10-09T04:20:27.794880Z","shell.execute_reply.started":"2022-10-09T04:20:27.765922Z","shell.execute_reply":"2022-10-09T04:20:27.793547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tipos de dato\nX.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:27.797128Z","iopub.execute_input":"2022-10-09T04:20:27.797631Z","iopub.status.idle":"2022-10-09T04:20:27.806568Z","shell.execute_reply.started":"2022-10-09T04:20:27.797586Z","shell.execute_reply":"2022-10-09T04:20:27.805284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows', None)\nX.describe().transpose()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:27.808137Z","iopub.execute_input":"2022-10-09T04:20:27.808737Z","iopub.status.idle":"2022-10-09T04:20:29.142102Z","shell.execute_reply.started":"2022-10-09T04:20:27.808690Z","shell.execute_reply":"2022-10-09T04:20:29.140970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dado que solo existen 2 variables categoricas, de momento se dejaran fuera y el esfuerzo principal se hará sobre las variables numéricas\nX = X.drop(['D_63', 'D_64'], axis= 1).set_index(['customer_ID','S_2'])\ny = y.set_index('customer_ID')","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:29.143685Z","iopub.execute_input":"2022-10-09T04:20:29.144117Z","iopub.status.idle":"2022-10-09T04:20:29.292509Z","shell.execute_reply.started":"2022-10-09T04:20:29.144076Z","shell.execute_reply":"2022-10-09T04:20:29.291535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tratamiento de Valores Vacios","metadata":{"execution":{"iopub.status.busy":"2022-10-09T02:15:05.234720Z","iopub.execute_input":"2022-10-09T02:15:05.235221Z","iopub.status.idle":"2022-10-09T02:15:05.241101Z","shell.execute_reply.started":"2022-10-09T02:15:05.235183Z","shell.execute_reply":"2022-10-09T02:15:05.239619Z"}}},{"cell_type":"code","source":"# Metodo que se utilizara para validar la calidad de las tecnicas de manero de valores vacios\ndef score_dataset(X, y, model=LinearRegression()): \n    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.7,  test_size=0.3, random_state=0)\n    model.fit(X_train, y_train)\n    preds = model.predict(X_test)\n    return mean_absolute_error(y_test, preds)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:29.293824Z","iopub.execute_input":"2022-10-09T04:20:29.294171Z","iopub.status.idle":"2022-10-09T04:20:29.300190Z","shell.execute_reply.started":"2022-10-09T04:20:29.294141Z","shell.execute_reply":"2022-10-09T04:20:29.299358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Eliminar columnas con valores vacios","metadata":{}},{"cell_type":"code","source":"X_dropcolumns = X.dropna(axis=1)\n\nprint(\"Total de columnas: \",len(X.columns))\nprint(\"Total de columnas no vacias: \", len(X_dropcolumns.columns))\nprint(\"MAE:\", score_dataset(X_dropcolumns,y))","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:29.301569Z","iopub.execute_input":"2022-10-09T04:20:29.302054Z","iopub.status.idle":"2022-10-09T04:20:29.855011Z","shell.execute_reply.started":"2022-10-09T04:20:29.302023Z","shell.execute_reply":"2022-10-09T04:20:29.853538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imputar con media","metadata":{"execution":{"iopub.status.busy":"2022-10-09T02:44:42.717790Z","iopub.execute_input":"2022-10-09T02:44:42.718238Z","iopub.status.idle":"2022-10-09T02:44:42.723611Z","shell.execute_reply.started":"2022-10-09T02:44:42.718203Z","shell.execute_reply":"2022-10-09T02:44:42.722543Z"}}},{"cell_type":"code","source":"media_imputer_no_ind = SimpleImputer(strategy='mean', add_indicator = False)\nX_meanimputer_no_ind = pd.DataFrame(media_imputer_no_ind.fit_transform(X), columns=X.columns)\nprint(\"IMPUTACION CON MEDIA SIN INDICADORES DE IMPUTACION:\")\nprint(\"Número de columnas: \", len(X_meanimputer_no_ind.columns))\nprint(\"MAE:\", score_dataset(X_meanimputer_no_ind,y))","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:29.862309Z","iopub.execute_input":"2022-10-09T04:20:29.869763Z","iopub.status.idle":"2022-10-09T04:20:31.478200Z","shell.execute_reply.started":"2022-10-09T04:20:29.869684Z","shell.execute_reply":"2022-10-09T04:20:31.476539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imputar con mediana","metadata":{"execution":{"iopub.status.busy":"2022-10-09T03:05:52.795081Z","iopub.execute_input":"2022-10-09T03:05:52.795575Z"}}},{"cell_type":"code","source":"mediana_imputer_no_ind = SimpleImputer(strategy='median', add_indicator = False)\nX_imp_median_no_ind = pd.DataFrame(mediana_imputer_no_ind.fit_transform(X), columns=X.columns)\nprint(\"IMPUTACION CON MEDIANA SIN INDICADORES DE IMPUTACION:\")\nprint(\"Número de columnas: \", len(X_imp_median_no_ind.columns))\nprint(\"MAE:\", score_dataset(X_imp_median_no_ind,y))","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:31.486921Z","iopub.execute_input":"2022-10-09T04:20:31.491411Z","iopub.status.idle":"2022-10-09T04:20:36.192892Z","shell.execute_reply.started":"2022-10-09T04:20:31.491327Z","shell.execute_reply":"2022-10-09T04:20:36.191186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Resultado: Se utilizará el resultado de la imputación con media para continuar con el análisis","metadata":{"execution":{"iopub.status.busy":"2022-10-09T03:15:23.309859Z","iopub.execute_input":"2022-10-09T03:15:23.310367Z","iopub.status.idle":"2022-10-09T03:15:23.316717Z"}}},{"cell_type":"code","source":"X_meanimputer_no_ind.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:36.200581Z","iopub.execute_input":"2022-10-09T04:20:36.204758Z","iopub.status.idle":"2022-10-09T04:20:36.296186Z","shell.execute_reply.started":"2022-10-09T04:20:36.204677Z","shell.execute_reply":"2022-10-09T04:20:36.294507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X_meanimputer_no_ind","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:36.300143Z","iopub.execute_input":"2022-10-09T04:20:36.300662Z","iopub.status.idle":"2022-10-09T04:20:36.305815Z","shell.execute_reply.started":"2022-10-09T04:20:36.300617Z","shell.execute_reply":"2022-10-09T04:20:36.304860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mutual Information","metadata":{"execution":{"iopub.status.busy":"2022-10-09T02:00:18.298694Z","iopub.execute_input":"2022-10-09T02:00:18.299133Z","iopub.status.idle":"2022-10-09T02:00:18.306192Z","shell.execute_reply.started":"2022-10-09T02:00:18.299098Z","shell.execute_reply":"2022-10-09T02:00:18.304599Z"}}},{"cell_type":"code","source":"from sklearn.feature_selection import mutual_info_regression\n\ndef make_mi_scores(X, y):\n    X = X.copy()\n    mi_scores = mutual_info_regression(X, y, random_state=0)\n    mi_scores = pd.Series(mi_scores, name=\"MI Scores\", index=X.columns)\n    mi_scores = mi_scores.sort_values(ascending=False)\n    return mi_scores\n\n\ndef plot_mi_scores(scores):\n    scores = scores.sort_values(ascending=True)\n    width = np.arange(len(scores))\n    ticks = list(scores.index)\n    plt.barh(width, scores)\n    plt.yticks(width, ticks)\n    plt.title(\"Mutual Information Scores\")","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:36.307339Z","iopub.execute_input":"2022-10-09T04:20:36.307692Z","iopub.status.idle":"2022-10-09T04:20:36.330498Z","shell.execute_reply.started":"2022-10-09T04:20:36.307661Z","shell.execute_reply":"2022-10-09T04:20:36.329635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cálculo de mutual information\nmi_scores = make_mi_scores(X, y.target)\n\n# Graficar resultado\nplt.figure(dpi=100, figsize=(15, 35))\nplot_mi_scores(mi_scores)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:20:36.332338Z","iopub.execute_input":"2022-10-09T04:20:36.332725Z","iopub.status.idle":"2022-10-09T04:23:41.096919Z","shell.execute_reply.started":"2022-10-09T04:20:36.332691Z","shell.execute_reply":"2022-10-09T04:23:41.095787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Modelo base (regresión lineal usando todo el dataset) - MAE:\",score_dataset(X, y.target))\n\n# Se contruye un modelo más sencillo utilizando únicamente las features más importantes en cuanto a mutual information\nX_top_mutualinfo = X[list(mi_scores.head(10).index)]\nmodel = RandomForestRegressor()\nprint(\"Random Forest (utilizando top 10 con mayor mutual information) - MAE:\",score_dataset(X_top_mutualinfo, y.target, model=model))","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:24:26.454808Z","iopub.execute_input":"2022-10-09T04:24:26.457565Z","iopub.status.idle":"2022-10-09T04:25:54.972707Z","shell.execute_reply.started":"2022-10-09T04:24:26.457455Z","shell.execute_reply":"2022-10-09T04:25:54.971512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creando un dataset más sencillo (con únicamente 10 de las features con mayor mutual information) y usando un modelo un poco mejor en cuanto a capacidad de aprender comportamientos complejos se observa una ligera mejora. De haber intentado construir el modelo de RandomForestRegressor() el notebook en Kaggle se hubiera quedado trabado ya que consume mucho más memoria y procesamiento que una regresión Lineal.\n\nAdicionalmente, el dataset tiene 5 tipos de variables:\n\nD_* = Delinquency variables\n\nS_* = Spend variables\n\nP_* = Payment variables\n\nB_* = Balance variables\n\nR_* = Risk variables\n\nLa complejidad adicional de este análisis es que todas las variables son anónimas y nada más podemos saber de manera general el tipo de la variable.\n\nSin embargo, llama la atención que de las 10 variables con mayor mutual information, 8 son de la **categoría D_* = Delinquency variables**. Esto nos indica que será necesario poner más atención a estas variables ya que son las que parecen ser más prometedoras.","metadata":{}},{"cell_type":"markdown","source":"# PCA","metadata":{"execution":{"iopub.status.busy":"2022-10-09T03:55:21.692263Z","iopub.execute_input":"2022-10-09T03:55:21.692740Z","iopub.status.idle":"2022-10-09T03:55:21.697636Z","shell.execute_reply.started":"2022-10-09T03:55:21.692704Z","shell.execute_reply":"2022-10-09T03:55:21.696692Z"}}},{"cell_type":"markdown","source":"Dada la cantidad tan grande de features con las que cuenta este dataset, se evaluará a continuación la posibilidad de reducir la dimensionalidad y simultáneamente construir nuevas features mediante la utilización de la técnica de PCA:","metadata":{}},{"cell_type":"code","source":"# Para obtener mejores resultados en PCA es necesario estandarizar\nscaler = StandardScaler()\nX_standarizado = pd.DataFrame(scaler.fit_transform(X), columns=X.columns)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:25:55.265801Z","iopub.execute_input":"2022-10-09T04:25:55.266278Z","iopub.status.idle":"2022-10-09T04:25:55.549316Z","shell.execute_reply.started":"2022-10-09T04:25:55.266240Z","shell.execute_reply":"2022-10-09T04:25:55.547935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Aplicar PCA\npca = PCA()\nX_pca = pca.fit_transform(X)\ncomponent_names = [f\"PC{i+1}\" for i in range(X_pca.shape[1])]\nX_pca = pd.DataFrame(X_pca, columns=component_names)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:25:55.552637Z","iopub.execute_input":"2022-10-09T04:25:55.553150Z","iopub.status.idle":"2022-10-09T04:25:57.671198Z","shell.execute_reply.started":"2022-10-09T04:25:55.553097Z","shell.execute_reply":"2022-10-09T04:25:57.669892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_variance(pca, width=8, dpi=100):\n    # Create figure\n    fig, axs = plt.subplots(1, 2)\n    n = pca.n_components_\n    grid = np.arange(1, n + 1)\n    # Explained variance\n    evr = pca.explained_variance_ratio_\n    axs[0].bar(grid, evr)\n    axs[0].set(\n        xlabel=\"Component\", title=\"% Explained Variance\", ylim=(0.0, 1.0)\n    )\n    # Cumulative Variance\n    cv = np.cumsum(evr)\n    axs[1].plot(np.r_[0, grid], np.r_[0, cv], \"o-\")\n    axs[1].set(\n        xlabel=\"Component\", title=\"% Cumulative Variance\", ylim=(0.0, 1.0)\n    )\n    # Set up figure\n    fig.set(figwidth=15, dpi=100)\n    return axs\n\n# Graficar la varianza acumulado\nplot_variance(pca)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:25:57.672853Z","iopub.execute_input":"2022-10-09T04:25:57.673260Z","iopub.status.idle":"2022-10-09T04:25:58.611761Z","shell.execute_reply.started":"2022-10-09T04:25:57.673220Z","shell.execute_reply":"2022-10-09T04:25:58.610352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Valores de la varianza acumulada para los primeros 20 componentes\npd.DataFrame(np.cumsum(pca.explained_variance_ratio_), columns=['Varianza Acumulada']).head(20)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:25:58.630971Z","iopub.execute_input":"2022-10-09T04:25:58.631631Z","iopub.status.idle":"2022-10-09T04:25:58.645498Z","shell.execute_reply.started":"2022-10-09T04:25:58.631587Z","shell.execute_reply":"2022-10-09T04:25:58.644166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Modelo base (regresión lineal usando todo el dataset) - MAE:\",score_dataset(X_top_mutualinfo, y.target))\n\n# Se construye un modelo con los primeros 11 componentes, los cuales son los que explican la mayor parte de la varianza en los datos\n# La varianza acumulada en hasta el componente 11 es de 0.893736\nmodel = RandomForestRegressor()\nprint(\"Random Forest de PCA (primeros 11 componentes)- MAE:\",score_dataset(X_pca[X_pca.columns[:12]], y.target, model=model))","metadata":{"execution":{"iopub.status.busy":"2022-10-09T04:30:01.917771Z","iopub.execute_input":"2022-10-09T04:30:01.918219Z","iopub.status.idle":"2022-10-09T04:33:44.448953Z","shell.execute_reply.started":"2022-10-09T04:30:01.918183Z","shell.execute_reply":"2022-10-09T04:33:44.447749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"El resultado de la construcción de los componentes principales fue adecuado, se puede ver que con los primeros 11 componentes, que en conjunto acumulan el 89% de la varianza del dataset se obtiene un modelo mejor que el modelo base de regresión lineal","metadata":{}}]}