{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AMEX Competition Feature engineering\n\nThe notebook is based on insights of the [EDA which makes sense ⭐️⭐️⭐️⭐️⭐️](https://www.kaggle.com/code/ambrosm/amex-eda-which-makes-sense).\n\nand [AMEX - Feature Engineering](https://www.kaggle.com/code/lucasmorin/amex-feature-engineering)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport pickle\nfrom matplotlib import pyplot as plt\nimport random\nimport datetime\nimport math\nimport gc\nimport warnings\nimport seaborn as sns\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OrdinalEncoder, OneHotEncoder, RobustScaler","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading and preprocessing the training data\n\nWe read the data from @munumbutt's [AMEX-Feather-Dataset](https://www.kaggle.com/datasets/munumbutt/amexfeather). Then we create some groups of features:\n- Selected features taken as minimums, maximums, averages, and standard deviations over all statements of a customer\n- Selected features taken from the last statement of a customer\n- Other featurers including the number of unique statements a customer and the number of statements a person gets statements\n\nWe one-hot encode the categorical features and fill all missing values with 0.\n\nThe code has been optimized for memory efficiency rather than readability. In particular, `.iloc[mask_array, columns]` needs much less RAM than the groupby construction used in previous versions of the notebook.\n\nWe process train and test data separately\n","metadata":{}},{"cell_type":"code","source":"test_read = pd.read_csv('../input/amex-default-prediction/train_data.csv',nrows=1)\n\ntest_read","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n\nall_features = [c for c in list(test_read.columns) if c not in ['customer_ID','S_2_max']]\n\ncat_features = ['B_30', 'B_38', 'D_114', 'D_116',\n                        'D_117', 'D_120', 'D_126',\n                        'D_63', 'D_64', 'D_66', 'D_68']\n\nohe_cat_features = [f'{s}_last' for s in cat_features]\n\nnum_features = [col for col in all_features if col not in cat_features]\n\ndef read_file(path, i):\n    df = pd.read_feather(path)\n    df['S_2'] = pd.to_datetime(df['S_2'])\n    df['S_2_max'] = df[['S_2','customer_ID']].groupby('customer_ID').S_2.transform('max')\n    df['S_2_diff'] = df[['S_2','customer_ID']].groupby('customer_ID').S_2.transform('diff').dt.days\n    df['S_2'] = (df['S_2_max']-df['S_2']).dt.days\n    \n    #de-noising\n    for col in df.columns:\n        if df[col].dtype=='float16':\n            df[col] = df[col].astype('float32').round(decimals=2).astype('float16')\n    \n    if i==0:\n        enc.fit(df[cat_features])\n    df[cat_features] = enc.transform(df[cat_features])\n    df = df.fillna(0) \n    print('shape of data:', df.shape)\n    gc.collect()\n    return df\ndef preprocess(df, i, j ):\n    test_num_agg = df.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n    test_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]\n\n    test_cat_agg = df.groupby(\"customer_ID\")[cat_features].agg(['last', 'nunique']).astype(np.int16)\n    test_cat_agg.columns = ['_'.join(x) for x in test_cat_agg.columns]\n    \n    if i==0 and j==0:\n        ohe.fit(test_cat_agg[ohe_cat_features])\n    test_cat_last = pd.DataFrame(ohe.transform(test_cat_agg[ohe_cat_features]).astype(np.int16),\n                                  index=test_cat_agg.index, columns = ohe.get_feature_names_out())\n    test_cat_agg.drop(ohe_cat_features, axis = 1,inplace = True)\n    \n    other_agg = df.groupby('customer_ID')[['customer_ID']].agg(['count']).astype(np.int16)\n    other_agg.columns = ['_'.join(x) for x in other_agg.columns]\n    \n    df = pd.concat([test_num_agg, test_cat_agg, test_cat_last, other_agg], axis=1)\n        \n    del test_num_agg, test_cat_agg, other_agg, test_cat_last\n    gc.collect()\n    print('shape after engineering', df.shape )\n    return df\n\n              \nohe = OneHotEncoder(drop='first', sparse=False, dtype=np.int16, handle_unknown='ignore')\nenc = OrdinalEncoder()\nsca = RobustScaler()\n# imp = SimpleImputer(strategy = 'median')\ndef process(path, i, splits):\n    df = read_file(path, i)\n    if i ==0:\n        df.drop('target',axis = 1,inplace = True)\n    def split(a, n):\n        k, m = divmod(len(a), n)\n        return (a[i*k+min(i, m):(i+1)*k+min(i+1, m)] for i in range(n))\n\n    split_ids = split(df.customer_ID.unique(),splits)\n\n    df_list = []\n    \n    for (j,ids) in enumerate(split_ids):\n        print(j)\n        df_ids = df[df.customer_ID.isin(ids)]\n        df_t = preprocess(df_ids, i, j)\n        df_list.append(df_t)\n        del df_ids, df_t\n        gc.collect()\n    \n    \n    df = pd.concat(df_list, axis = 0)\n    \n    #defragment\n    df = df.copy()\n    \n    df.reset_index(inplace = True)\n    \n    #drop constant columns\n    df.drop(['S_2_min', 'S_2_last'],axis = 1, inplace = True)\n    \n#     df = reduce_mem_usage(df)\n    \n    # Impute missing values\n    df.fillna(value=-1, inplace=True)\n    df['customer_ID'] = df['customer_ID'].astype(str)\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = process('../input/amexfeather/train_data.ftr', 0, 2)\ngc.collect()\nprint('Shapes:', train.shape)\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge train and target ids\ntarget = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\n\nprint(target.shape)\nprint(target.head())\ntrain['target'] = target['target']\ntarget.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_feather('train_processed.ftr')\ntrain.to_csv('train_processed.csv.gz', compression=\"gzip\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Data","metadata":{}},{"cell_type":"code","source":"del train, target\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = process('../input/amexfeather/test_data.ftr', 1, 30)\n\nprint('Shapes:', test.shape)\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del enc, ohe\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.to_feather('test_processed.ftr')\ntest.to_csv('test_processed.csv.gz', compression=\"gzip\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}