{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Creates amex datasets with minimized memory usage for ML training and EDA.","metadata":{}},{"cell_type":"code","source":"!pip install tables","metadata":{"execution":{"iopub.status.busy":"2022-08-21T12:26:28.669807Z","iopub.execute_input":"2022-08-21T12:26:28.670659Z","iopub.status.idle":"2022-08-21T12:26:44.363780Z","shell.execute_reply.started":"2022-08-21T12:26:28.670558Z","shell.execute_reply":"2022-08-21T12:26:44.362021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport pickle as pk\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\n\ntrain_feather_path = \"../input/parquet-files-amexdefault-prediction/train_data.ftr\"\ntest_feather_path = \"../input/parquet-files-amexdefault-prediction/test_data.ftr\"\ntrain_labels_path = \"../input/amex-default-prediction/train_labels.csv\"\n\ntrain_df_file = '/kaggle/working/train_data.hdf'\nreduced_memory_train_df_file = '/kaggle/working/reduced_train_data.hdf'\ntrain_df_last_statement_file = '/kaggle/working/train_data_last_statement.hdf'\ntrain_split_df_filename = '/kaggle/working/train_split_df.hdf'\nval_split_df_filename = '/kaggle/working/val_split_df.hdf'\n\ntrain_id_map_filename = '/kaggle/working/train_id_map.pk'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-21T12:26:44.366920Z","iopub.execute_input":"2022-08-21T12:26:44.367477Z","iopub.status.idle":"2022-08-21T12:26:45.591387Z","shell.execute_reply.started":"2022-08-21T12:26:44.367422Z","shell.execute_reply":"2022-08-21T12:26:45.590004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# quickly load one row so we can get the column names for categorical and numeric columns\ntrain_df_for_cols = pd.read_csv('../input/amex-default-prediction/train_data.csv', nrows=1)\ncat_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nnum_cols = set(train_df_for_cols.columns).difference(cat_cols + ['customer_ID', 'S_2', 'B_31'])\n\nclass DataPrepper:\n    def __init__(self):\n        self.cat_les = {}  # label encoders for categorical columns\n        self.num_medians = {}  # median values for filling NA from train data\n        self.customer_id_map = {}  # map from reduced customer ID values to full values\n    \n    def reduce_ID_memory(self, df):\n        \"\"\"\n        Reduces customer ID memory use.\n        \"\"\"\n        df['customer_ID_int'] = df['customer_ID'].apply(lambda x: int(x[-16:], 16)).astype('int64')\n        return df\n    \n    \n    def create_customer_ID_map(self, df, filename):\n        \"\"\"\n        Creates map from memory-reduced customer ID to full customer ID values and saves to disk.\n        df should have original customer IDs.\n        \"\"\"\n        original_ids = df['customer_ID'].copy()\n        df = self.reduce_ID_memory(df)\n        df = df.set_index('customer_ID_int')\n        self.customer_id_map = df['customer_ID'].to_dict()\n        with open(filename, 'wb') as file:\n            pk.dump(self.customer_id_map, file)\n        \n        \n    def label_encode_categorical_cols(self, df, train):\n        print(\"converting categorical columns\")\n        for col in cat_cols:\n            if train:\n                le = LabelEncoder()\n                le_fit = le.fit(df[col].unique())\n                _ = self.cat_les.setdefault(col, le_fit)\n            \n            df[col] = self.cat_les[col].transform(df[col])\n            df[col] = df[col].astype('int8')\n            gc.collect()\n        \n        return df\n\n    \n    def convert_numeric_and_fill_median(self, df):\n        print(\"converting numeric columns\")\n        df['B_31'] = df['B_31'].astype('int8')\n\n        for col in num_cols:\n            df[col] = df[col].astype('float16')\n            gc.collect()\n        \n        return df\n    \n        \n    def reduce_memory(self, df, train, reduce_ID_memory=True):\n        \"\"\"\n        Reduces the memomry size of pandas df with several techniques.\n        \n        if train==True, gets label encoders from data for categorical columns.\n        Otherwise, uses existing label encoders to encode categorical columns.\n        \"\"\"\n        if reduce_ID_memory:\n            df = self.reduce_ID_memory(df)\n        \n        df['S_2'] = pd.to_datetime(df['S_2'])\n        df = self.label_encode_categorical_cols(df, train)\n        df = self.convert_numeric_and_fill_median(df)\n\n        gc.collect()\n\n        return df\n\n    \n    def get_df_max_date_per_customer(self, df):\n        \"\"\"\n        Each customer has a series of dates with data. This simply gets the latest date per customer ID.\n        \"\"\"\n        max_date_idx = df['S_2'].groupby(df.index).transform(max) == df['S_2']\n        df = df.loc[max_date_idx]\n        df_max_date_ids = df.index\n        return df","metadata":{"execution":{"iopub.status.busy":"2022-08-21T12:26:45.593335Z","iopub.execute_input":"2022-08-21T12:26:45.593743Z","iopub.status.idle":"2022-08-21T12:26:45.638798Z","shell.execute_reply.started":"2022-08-21T12:26:45.593710Z","shell.execute_reply":"2022-08-21T12:26:45.637954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dp = DataPrepper()","metadata":{"execution":{"iopub.status.busy":"2022-08-21T12:26:45.642171Z","iopub.execute_input":"2022-08-21T12:26:45.642542Z","iopub.status.idle":"2022-08-21T12:26:45.646926Z","shell.execute_reply.started":"2022-08-21T12:26:45.642511Z","shell.execute_reply":"2022-08-21T12:26:45.645701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_feather(train_feather_path)\ntrain_label_df = pd.read_csv(train_labels_path)","metadata":{"execution":{"iopub.status.busy":"2022-08-21T12:26:45.648707Z","iopub.execute_input":"2022-08-21T12:26:45.649450Z","iopub.status.idle":"2022-08-21T12:27:07.763597Z","shell.execute_reply.started":"2022-08-21T12:26:45.649416Z","shell.execute_reply":"2022-08-21T12:27:07.762093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dp.create_customer_ID_map(train_label_df, train_id_map_filename)  # side effect: adds customer_ID_int column to train_label_df\n\n# add labels to train_df\ntrain_df = dp.reduce_ID_memory(train_df)\ntrain_df = train_df.drop(columns=['customer_ID']).set_index('customer_ID_int')\ntrain_label_df = train_label_df.drop(columns=['customer_ID']).set_index('customer_ID_int')\ntrain_df = train_df.join(train_label_df)\ntrain_df.to_hdf(train_df_file, format='table', key='train_df')\n\ntrain_df_last_statement = dp.get_df_max_date_per_customer(df=train_df)\ntrain_df_last_statement.to_hdf(train_df_last_statement_file, format='table', key='train_data_last_statement')","metadata":{"execution":{"iopub.status.busy":"2022-08-21T12:27:07.765452Z","iopub.execute_input":"2022-08-21T12:27:07.765962Z","iopub.status.idle":"2022-08-21T12:28:48.249999Z","shell.execute_reply.started":"2022-08-21T12:27:07.765914Z","shell.execute_reply":"2022-08-21T12:28:48.249055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_split_df, val_split_df = train_test_split(train_df_last_statement, stratify=train_df_last_statement['target'], random_state=42, test_size=0.1)\n\ntrain_split_df.to_hdf(train_split_df_filename, format='table', key='train_split_df')\nval_split_df.to_hdf(val_split_df_filename, format='table', key='val_split_df')","metadata":{"execution":{"iopub.status.busy":"2022-08-21T12:29:40.229355Z","iopub.execute_input":"2022-08-21T12:29:40.230289Z","iopub.status.idle":"2022-08-21T12:29:44.560057Z","shell.execute_reply.started":"2022-08-21T12:29:40.230212Z","shell.execute_reply":"2022-08-21T12:29:44.558536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IDs have already had memory reduced and been set as index\nreduced_memory_train = dp.reduce_memory(train_df, train=True, reduce_ID_memory=False)\nreduced_memory_train.to_hdf(reduced_memory_train_df_file, key='reduced_memory_train_data')","metadata":{"execution":{"iopub.status.busy":"2022-08-21T12:29:44.562123Z","iopub.execute_input":"2022-08-21T12:29:44.562523Z"},"trusted":true},"execution_count":null,"outputs":[]}]}