{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":35332,"databundleVersionId":3723648,"sourceType":"competition"},{"sourceId":3727003,"sourceType":"datasetVersion","datasetId":2213609},{"sourceId":3739819,"sourceType":"datasetVersion","datasetId":2231132}],"dockerImageVersionId":30207,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-19T23:09:17.162652Z","iopub.execute_input":"2024-02-19T23:09:17.163382Z","iopub.status.idle":"2024-02-19T23:09:17.218912Z","shell.execute_reply.started":"2024-02-19T23:09:17.163232Z","shell.execute_reply":"2024-02-19T23:09:17.217338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This notebook is an improvement to the baseline model \nhttps://www.kaggle.com/code/foocheechuan/amexdefaultprediction","metadata":{}},{"cell_type":"markdown","source":"<a id=\"table-of-content\"></a>\n# Table of Content\n### [1. Setup](#setup)\n- [Import Libraries](#import-libraries)\n- [Dataset](#dataset)\n\n### [2. Handling Missing Values](#missing)\n- [Removes columns with >50% missing values](#50%-missing)\n- [Simple Imputer](#simple-imputer)\n\n### [Go to end](#end)","metadata":{}},{"cell_type":"markdown","source":"# Setup\n<a id=\"setup\"></a>","metadata":{}},{"cell_type":"markdown","source":"# Import Libraries\n<a id=\"import-libraries\"></a>","metadata":{}},{"cell_type":"code","source":"# data preparation\nimport pandas as pd\nimport numpy as np\nimport gc\n\n# visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n# data preprocessing\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score  \nfrom sklearn.metrics import precision_score                         \nfrom sklearn.metrics import recall_score\nfrom sklearn.impute import SimpleImputer\nfrom sklearn import preprocessing\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import StratifiedKFold\nfrom lightgbm import LGBMClassifier, log_evaluation\n","metadata":{"execution":{"iopub.status.busy":"2024-02-19T23:09:21.231885Z","iopub.execute_input":"2024-02-19T23:09:21.232431Z","iopub.status.idle":"2024-02-19T23:09:23.596422Z","shell.execute_reply.started":"2024-02-19T23:09:21.232384Z","shell.execute_reply":"2024-02-19T23:09:23.594951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset\n<a id=\"dataset\"></a>","metadata":{}},{"cell_type":"markdown","source":"- The original dataset provided in csv is too large (50GB)\n- The data cannot fit into memory\n- [Amex-Feather-Dataset](#https://www.kaggle.com/datasets/munumbutt/amexfeather) provided by [@munum](#https://www.kaggle.com/munumbutt) is a [feather file](#https://arrow.apache.org/docs/python/feather.html) that has smaller size than an equivalent csv file","metadata":{}},{"cell_type":"markdown","source":"# Read Data","metadata":{"execution":{"iopub.status.busy":"2024-02-17T08:28:41.156009Z","iopub.execute_input":"2024-02-17T08:28:41.156522Z","iopub.status.idle":"2024-02-17T08:28:41.162030Z","shell.execute_reply.started":"2024-02-17T08:28:41.156478Z","shell.execute_reply":"2024-02-17T08:28:41.160696Z"}}},{"cell_type":"code","source":"# train.shape = (5531451,190)\ntrain = pd.read_parquet('/kaggle/input/amex-data-integer-dtypes-parquet-format/train.parquet')\n\n# use small dataset to prevent insufficient memory\ntrain_small = train.head(100)","metadata":{"execution":{"iopub.status.busy":"2024-02-19T01:58:32.251418Z","iopub.execute_input":"2024-02-19T01:58:32.252180Z","iopub.status.idle":"2024-02-19T01:58:53.940930Z","shell.execute_reply.started":"2024-02-19T01:58:32.252128Z","shell.execute_reply":"2024-02-19T01:58:53.939975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"# ====================================================\n# Read & preprocess data and save it to disk\n# ====================================================\ndef read_preprocess_data_small():\n\n    # train.shape = (5531451,190)\n    train = pd.read_parquet('/kaggle/input/amex-data-integer-dtypes-parquet-format/train.parquet')\n    \n    # use small dataset to prevent insufficient memory\n    train_small = train.head(100)\n    \n    # removes id and time from training set\n    features = train_small.drop(['customer_ID', 'S_2'], axis = 1).columns.to_list()\n    \n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\",\n    ]\n    num_features = [col for col in features if col not in cat_features]\n    print('Starting training feature engineer...')\n    \n    # Aggregate the rows by customer_ID reduces the number of rows and add many features\n    train_num_agg = train_small.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n\n    # Add aggregation method to the column names\n    train_num_agg.columns = ['_'.join(x) for x in train_num_agg.columns]\n    train_num_agg.reset_index(inplace = True)\n\n    # Feature engineering for categorical columns\n    train_cat_agg = train_small.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\n    train_cat_agg.columns = ['_'.join(x) for x in train_cat_agg.columns]\n    train_cat_agg.reset_index(inplace = True)\n    \n    # read target file\n    train_labels = pd.read_csv('/kaggle/input/amex-default-prediction/train_labels.csv')\n    \n    # join num and cat features then join target using customer_ID\n    train = train_num_agg.merge(train_cat_agg, how = 'inner', on = 'customer_ID').merge(train_labels, how = 'inner', on = 'customer_ID')\n    del train_num_agg, train_cat_agg\n    gc.collect()\n    \n    # repeat for testing set\n    test = pd.read_parquet('/content/data/test.parquet')\n    print('Starting test feature engineer...')\n    test_num_agg = test.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n    test_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]\n    test_num_agg.reset_index(inplace = True)\n    test_cat_agg = test.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\n    test_cat_agg.columns = ['_'.join(x) for x in test_cat_agg.columns]\n    test_cat_agg.reset_index(inplace = True)\n    test = test_num_agg.merge(test_cat_agg, how = 'inner', on = 'customer_ID')\n    del test_num_agg, test_cat_agg\n    gc.collect()\n    \n    # Save files to disk\n    train.to_parquet('/kaggle/output/Amex_preprocessing/train_fe.parquet')\n    test.to_parquet('/kaggle/output/Amex_preprocessing/test_fe.parquet')","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Read & preprocess data and save it to disk\n# ====================================================\ndef read_preprocess_data():\n    \n    # train.shape = (5531451,190)\n    train = pd.read_parquet('/kaggle/input/amex-data-integer-dtypes-parquet-format/train.parquet')\n    \n    # use small dataset to prevent insufficient memory\n    train_small = train.head(100)\n    \n    # removes id and time from training set\n    features = train_small.drop(['customer_ID', 'S_2'], axis = 1).columns.to_list()\n    \n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\",\n    ]\n    num_features = [col for col in features if col not in cat_features]\n    print('Starting training feature engineer...')\n    \n    # Aggregate the rows by customer_ID reduces the number of rows and add many features\n    train_num_agg = train.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n\n    # Add aggregation method to the column names\n    train_num_agg.columns = ['_'.join(x) for x in train_num_agg.columns]\n    train_num_agg.reset_index(inplace = True)\n\n    # Feature engineering for categorical columns\n    train_cat_agg = train.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\n    train_cat_agg.columns = ['_'.join(x) for x in train_cat_agg.columns]\n    train_cat_agg.reset_index(inplace = True)\n    \n    # read target file\n    train_labels = pd.read_csv('/kaggle/input/amex-default-prediction/train_labels.csv')\n    \n    # join num and cat features then join target using customer_ID\n    train = train_num_agg.merge(train_cat_agg, how = 'inner', on = 'customer_ID').merge(train_labels, how = 'inner', on = 'customer_ID')\n    del train_num_agg, train_cat_agg\n    gc.collect()\n    \n    # repeat for testing set\n    test = pd.read_parquet('/kaggle/input/amex-data-integer-dtypes-parquet-format/test.parquet')\n    print('Starting test feature engineer...')\n    test_num_agg = test.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n    test_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]\n    test_num_agg.reset_index(inplace = True)\n    test_cat_agg = test.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\n    test_cat_agg.columns = ['_'.join(x) for x in test_cat_agg.columns]\n    test_cat_agg.reset_index(inplace = True)\n    test = test_num_agg.merge(test_cat_agg, how = 'inner', on = 'customer_ID')\n    del test_num_agg, test_cat_agg\n    gc.collect()\n    \n    # Save files to disk\n    train.to_parquet('/kaggle/output/Amex_preprocessing/train_fe.parquet')\n    test.to_parquet('/kaggle/output/Amex_preprocessing/test_fe.parquet')\n    del train\n    del test","metadata":{"execution":{"iopub.status.busy":"2024-02-19T23:09:59.113815Z","iopub.execute_input":"2024-02-19T23:09:59.114400Z","iopub.status.idle":"2024-02-19T23:09:59.135251Z","shell.execute_reply.started":"2024-02-19T23:09:59.114352Z","shell.execute_reply":"2024-02-19T23:09:59.133999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_preprocess_data()","metadata":{"execution":{"iopub.status.busy":"2024-02-19T23:10:02.273805Z","iopub.execute_input":"2024-02-19T23:10:02.274375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Work in Progress\n- Model Optimization\n- Try different aggregation method\n- Stratified KFold validation\n- Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"# References\n<a id=\"references\"></a>\n- [AMEX EDA which makes sense](#https://www.kaggle.com/code/ambrosm/amex-eda-which-makes-sense)\n- [AMEX - Light GBM](#https://www.kaggle.com/code/lixinqi98/amex-lightgbm/notebook)\n- [AMEX Default Prediction - EDA & Prediction](#https://www.kaggle.com/code/aryanml007/amex-default-prediction-eda-prediction)\n- [Amex LGBM Dart CV 0.7963](#https://www.kaggle.com/code/ragnar123/amex-lgbm-dart-cv-0-7963)","metadata":{}},{"cell_type":"markdown","source":"# [Back](#table-of-content)\n<a id=\"end\"></a>","metadata":{}}]}