{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Notebook to prepare features for the AMEX competition.\n\nNote that this is a starter notebook, mainly aimed at showing how you can more or less easily handle a big dataset. \n\n**This Notebook is part of a serie built for the AMEX competition:**\n- [Base Feature engineering](https://www.kaggle.com/code/lucasmorin/amex-feature-engineering-base)\n- [Baseline lgbm](https://www.kaggle.com/code/lucasmorin/amex-lgbm-features-eng)\n- [Feature Engineering 2: aggregation function](https://www.kaggle.com/code/lucasmorin/amex-feature-engineering-2-aggreg-functions)\n- [Feature Engineering 3: transformation function](https://www.kaggle.com/code/lucasmorin/amex-feature-engineering-3-transform-functions)\n\n**With associated Data Sets:**\n- [Base Feature engineering](https://www.kaggle.com/datasets/lucasmorin/amex-base-fe)\n- [Feature Engineering 2 - aggregation function](https://www.kaggle.com/datasets/lucasmorin/amex-fe2)\n- [Feature Engineering 3 - transform function](https://www.kaggle.com/datasets/lucasmorin/amex-fe3)\n\n**Please make sure to upvote everything you use / find interesting / usefull**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport gc\n\nDEBUG = False","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-10T14:09:13.716821Z","iopub.execute_input":"2022-07-10T14:09:13.717833Z","iopub.status.idle":"2022-07-10T14:09:13.745428Z","shell.execute_reply.started":"2022-07-10T14:09:13.717719Z","shell.execute_reply":"2022-07-10T14:09:13.744677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Quick look at the data","metadata":{}},{"cell_type":"code","source":"train_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\ntest_read = pd.read_csv('../input/amex-default-prediction/train_data.csv',nrows=10)\n\ntest_read.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T14:09:14.186226Z","iopub.execute_input":"2022-07-10T14:09:14.186596Z","iopub.status.idle":"2022-07-10T14:09:15.136193Z","shell.execute_reply.started":"2022-07-10T14:09:14.186569Z","shell.execute_reply":"2022-07-10T14:09:15.135262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# prepare and agregate data","metadata":{}},{"cell_type":"code","source":"cat_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nbin_features = ['B_31', 'D_87']\n\nremove = ['customer_ID','S_2_max']\n\nf_names = ['mean','std','min','max','last','nunique','first']\nf_names_cat = ['last','nunique','first']\n\n\n# https://stackoverflow.com/questions/2130016/splitting-a-list-into-n-parts-of-approximately-equal-length\ndef split(a, n):\n    k, m = divmod(len(a), n)\n    return (a[i*k+min(i, m):(i+1)*k+min(i+1, m)] for i in range(n))\n\n\ndef prepare_df_num(df):\n    #replace -0.5 with na for aggregation ?\n    \n    # prepare date - TO DO: change max based on data set (train / private / public)\n    df['S_2'] = pd.to_datetime(df['S_2'])\n    df['S_2_max'] = df[['S_2','customer_ID']].groupby('customer_ID').S_2.transform('max')\n    df['S_2_diff'] = df[['S_2','customer_ID']].groupby('customer_ID').S_2.transform('diff').dt.days\n    df['S_2'] = (df['S_2_max']-df['S_2']).dt.days\n\n    # compute \"after pay\" features - see: https://www.kaggle.com/code/jiweiliu/rapids-cudf-feature-engineering-xgb\n    for bcol in [f'B_{i}' for i in [11,14,17]]+['D_39','D_131']+[f'S_{i}' for i in [16,23]]:\n        for pcol in ['P_2','P_3']:\n            if bcol in df.columns:\n                df[f'{bcol}-{pcol}'] = df[bcol] - df[pcol]\n                            \n    cols_num = [c for c in df.columns if c not in cat_features+bin_features+remove]\n    df.loc[:,cols_num] = df.loc[:,cols_num].astype('float16')\n    \n    return df\n\ndef prepare_df_cat(df):\n    # identifiy cat columns\n    df.loc[:,cat_features+bin_features] = df.loc[:,cat_features+bin_features].astype(str)\n    return df\n\ndef agg_df_num(df):\n    df_agg = df.groupby('customer_ID').agg(f_names)\n    df_agg.columns = [str(c[0])+'_'+str(c[1]) for c in df_agg.columns]\n    return df_agg\n\ndef agg_df_cat(df):\n    df_agg = df.groupby('customer_ID').agg(f_names_cat)\n    df_agg.columns = [str(c[0])+'_'+str(c[1]) for c in df_agg.columns]\n    return df_agg\n\n\ndef prepare_dataset(train_test = 'train'):\n    \n    data = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/'+train_test+'.parquet')\n    \n    if DEBUG:\n        data = data.iloc[:int((len(data)/60))]\n    \n    split_ids = split(data.customer_ID.unique(),10)\n\n    df_list = []\n\n    for (i,ids) in enumerate(split_ids):\n        print(i)\n        data_ids = data[data.customer_ID.isin(ids)]\n\n        data_ids = prepare_df_num(data_ids)\n        data_ids = prepare_df_cat(data_ids)\n\n        cols_num = [c for c in data_ids.columns if c not in cat_features+bin_features]\n\n        num_agg =  agg_df_num(data_ids[cols_num])\n        cat_agg =  agg_df_cat(data_ids[['customer_ID']+cat_features+bin_features])\n\n        df_list.append(pd.concat([num_agg, cat_agg],axis=1).astype('float16'))\n        gc.collect()\n\n    pd.concat(df_list,axis=0).astype('float16').to_pickle(train_test+'_data_agg.pkl')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-10T14:34:24.013856Z","iopub.execute_input":"2022-07-10T14:34:24.014274Z","iopub.status.idle":"2022-07-10T14:34:24.031026Z","shell.execute_reply.started":"2022-07-10T14:34:24.01424Z","shell.execute_reply":"2022-07-10T14:34:24.029999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nprepare_dataset(train_test = 'train')","metadata":{"execution":{"iopub.status.busy":"2022-07-10T14:34:25.025287Z","iopub.execute_input":"2022-07-10T14:34:25.026067Z","iopub.status.idle":"2022-07-10T14:34:46.423953Z","shell.execute_reply.started":"2022-07-10T14:34:25.026028Z","shell.execute_reply":"2022-07-10T14:34:46.422941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nprepare_dataset(train_test = 'test')","metadata":{"execution":{"iopub.status.busy":"2022-07-10T14:59:22.713616Z","iopub.execute_input":"2022-07-10T14:59:22.714135Z","iopub.status.idle":"2022-07-10T14:59:57.791726Z","shell.execute_reply.started":"2022-07-10T14:59:22.71399Z","shell.execute_reply":"2022-07-10T14:59:57.790932Z"},"trusted":true},"execution_count":null,"outputs":[]}]}