{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":35332,"databundleVersionId":3723648,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# LOADING JUST FIRST COLUMN OF TRAIN OR TEST IS SLOW\n# INSTEAD YOU CAN LOAD FIRST COLUMN FROM MY DATASET\n# PATH_TO_CUSTOMER_HASHES = '../input/amex-data-files/'\n# OTHERWISE SET VARIABLE TO NONE TO LOAD FROM KAGGLE'S ORIGINAL DATASET\nPATH_TO_CUSTOMER_HASHES = None\n\n# AFTER PROCESSING DATA ONCE, UPLOAD TO KAGGLE DATASET\n# THEN SET VARIABLE BELOW TO FALSE\nPROCESS_DATA = True\n\n\n# AND ATTACH DATASET TO NOTEBOOK AND PUT PATH TO DATASET BELOW\nPATH_TO_DATA = './data/'\n#PATH_TO_DATA = '../input/amex-data-for-transformers-and-rnns/data/'\n\n# The train data has been split into 10 NumPy arrays named data_1.npy thru data_10.npy.\n# Each array has dimension (45891, 13, 188) which is customer x statement x feature.\n# The associated targets are contained in the files targets_1.pqt thru targets_10.pqt.\n# These are parquet files with columns customer_ID and target and have dimension (45891, 2)\nNUM_FILES = 10\n\n# The 11 columns are categorical with maximum 8 values.\nCAT_VARS = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_66', 'D_68']\n# Offset is a value for each category which is added to the value of the category so that new values are shifted\n# then 0 will be padding, 1 will be NAN, 2,3,4,etc will be values\nCAT_WISE_OFFSETS = [2,1,2,2,3,2,3,2,2] \n\n# Additional categorical vars\n#  With categorical labels as : 0 => padding, 1 => nan, 2,3,4,etc => cat values\nADDITIONAL_CAT_VARS = ['D_63','D_64']\nD_63_MAP = {'CL':2, 'CO':3, 'CR':4, 'XL':5, 'XM':6, 'XZ':7}\nD_64_MAP = {'-1':2,'O':3, 'R':4, 'U':5}\n\nCATEGORICAL_VARS = CAT_VARS +  ADDITIONAL_CAT_VARS","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-05T12:41:30.813225Z","iopub.execute_input":"2024-03-05T12:41:30.814028Z","iopub.status.idle":"2024-03-05T12:41:30.832457Z","shell.execute_reply.started":"2024-03-05T12:41:30.813973Z","shell.execute_reply":"2024-03-05T12:41:30.831459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ignore annoying warning from various \nimport warnings\ndef ignore_warn(*args, **kwargs):\n    pass\nwarnings.warn = ignore_warn ","metadata":{"execution":{"iopub.status.busy":"2024-03-05T13:10:22.129761Z","iopub.execute_input":"2024-03-05T13:10:22.130622Z","iopub.status.idle":"2024-03-05T13:10:22.134974Z","shell.execute_reply.started":"2024-03-05T13:10:22.130589Z","shell.execute_reply":"2024-03-05T13:10:22.133980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CPU LIBRARIES\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nimport gc","metadata":{"execution":{"iopub.status.busy":"2024-03-05T12:41:33.520708Z","iopub.execute_input":"2024-03-05T12:41:33.521343Z","iopub.status.idle":"2024-03-05T12:41:33.898125Z","shell.execute_reply.started":"2024-03-05T12:41:33.521311Z","shell.execute_reply":"2024-03-05T12:41:33.897148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# GPU LIBRARIES\nimport cupy, cudf ","metadata":{"execution":{"iopub.status.busy":"2024-03-05T13:10:26.370567Z","iopub.execute_input":"2024-03-05T13:10:26.371289Z","iopub.status.idle":"2024-03-05T13:10:26.375126Z","shell.execute_reply.started":"2024-03-05T13:10:26.371261Z","shell.execute_reply":"2024-03-05T13:10:26.374193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_rows_per_file(customer_ids, train_df, num_files, verbose = ''):\n    \n    customer_ids_chunk_size = len(customer_ids)//num_files\n    \n    if verbose != '':\n        print(f'We will split {verbose} data into {NUM_FILES} separate files.')\n        print(f'There will be {customer_ids_chunk_size} customers in each file (except the last file).')\n        print('Below are number of rows in each file:')\n\n    file_wise_rows = []\n    for i in tqdm(range(num_files), desc=\"Split Rows\"):\n        if i==num_files-1:\n            customer_ids_chunk = customer_ids[i*customer_ids_chunk_size:]\n        else:\n            customer_ids_chunk = customer_ids[i*customer_ids_chunk_size:(i+1)*customer_ids_chunk_size]\n\n        file_wise_chunk_row_size = train_df.loc[train_df.customer_ID.isin(customer_ids_chunk)].shape[0]\n        print(f'train_df.loc[train_df.customer_ID.isin({customer_ids_chunk})].shape[0] = {file_wise_chunk_row_size}')\n        file_wise_rows.append(file_wise_chunk_row_size)\n        \n    if verbose != '':\n        print( file_wise_rows )\n\n    return file_wise_rows","metadata":{"execution":{"iopub.status.busy":"2024-03-05T12:41:40.714183Z","iopub.execute_input":"2024-03-05T12:41:40.714545Z","iopub.status.idle":"2024-03-05T12:41:40.721974Z","shell.execute_reply.started":"2024-03-05T12:41:40.714515Z","shell.execute_reply":"2024-03-05T12:41:40.721001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"customer_ID is a int64 instead of the provided string512","metadata":{}},{"cell_type":"code","source":"def feature_engineer(train_df, target_df = None, pad_customers_to_13 = True):\n    \n    # print missing values for col\n    # 1. Reduce Data Size\n    \n    # 1.1 from 64 bytes to 8 bytes, and \n    train_df['customer_ID'] = train_df['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n\n    # 1.2 from 10 bytes to 3 bytes\n    train_df.S_2 = cudf.to_datetime( train_df.S_2 )\n    train_df['year'] = (train_df.S_2.dt.year-2000).astype('int8')\n    train_df['month'] = (train_df.S_2.dt.month).astype('int8')\n    train_df['day'] = (train_df.S_2.dt.day).astype('int8')\n\n    del train_df['S_2']\n        \n    # 1.3 Label encode categorical columns (and reduce to 1 byte)\n    \n    # fo ADDITIONAL_CAT_VARS\n    train_df['D_63'] = train_df.D_63.map(D_63_MAP).fillna(1).astype('int8')\n    train_df['D_64'] = train_df.D_64.map(D_64_MAP).fillna(1).astype('int8')\n   \n    # minus minimal value in full train csv\n    # then 0 will be padding, 1 will be NAN, 2,3,4,etc will be values\n    for cat,offset in zip(CAT_VARS, CAT_WISE_OFFSETS):\n        train_df[cat] = train_df[cat] + offset\n        train_df[cat] = train_df[cat].fillna(1).astype('int8')\n    \n    # ADD NEW FEATURES HERE\n    # EXAMPLE: train['feature_189'] = etc etc etc\n    # EXAMPLE: train['feature_190'] = etc etc etc\n    # IF CATEGORICAL, THEN ADD TO CATS WITH: CATS += ['feaure_190'] etc etc etc\n    \n    # REDUCE MEMORY DTYPE\n    SKIP = ['customer_ID','year','month','day']\n    for col in train_df.columns:\n        if col in SKIP: continue\n        if str( train_df[col].dtype )=='int64':\n            train_df[col] = train_df[col].astype('int32')\n        if str( train_df[col].dtype )=='float64':\n            train_df[col] = train_df[col].astype('float32')\n            \n    # PAD ROWS SO EACH CUSTOMER HAS 13 ROWS- each for each month statement\n    if pad_customers_to_13:\n        cust_wise_row_count = train_df[['customer_ID']].groupby('customer_ID').customer_ID.agg('count')\n        more = cupy.array([],dtype='int64') \n        for j in range(1,13):\n            i = cust_wise_row_count.loc[cust_wise_row_count==j].index.values\n            more = cupy.concatenate([more,cupy.repeat(i,13-j)])\n        tmp_df = train_df.iloc[:len(more)].copy().fillna(0)\n        tmp_df = tmp_df * 0 - 1 #pad numerical columns with -1\n        \n        tmp_df[CAT_VARS] = (tmp_df[CAT_VARS] * 0).astype('int8') #pad categorical columns with 0\n        tmp_df['customer_ID'] = more\n        train_df = cudf.concat([train_df,tmp_df],axis=0,ignore_index=True)\n        \n    # ADD TARGETS \n    if target_df is not None:\n        # merge target into train_df\n        train_df = train_df.merge(target_df,on='customer_ID',how='left')\n        # reduce to 1 byte\n        train_df.target = train_df.target.astype('int8')\n        \n    # FILL NAN\n    SKIP = ['customer_ID','year','month','day'] + CAT_VARS\n    for col in train_df.columns:\n        if col in SKIP:\n            continue\n        if train_df[col].isnull().any():\n            mean_val = train_df[col].mean().astype('float32')\n#             print(f\"column name: {col} and data type: {train_df[col].dtype}\")\n            train_df[col] = train_df[col].fillna(mean_val) #this applies to numerical columns\n    \n    # SORT BY CUSTOMER \n    train_df = train_df.sort_values(['customer_ID','year','month','day']).reset_index(drop=True)\n    # SORT BY DATE\n    train_df = train_df.drop(['year','month','day'],axis=1)\n    del tmp_df\n    return train_df","metadata":{"execution":{"iopub.status.busy":"2024-03-05T13:13:13.199221Z","iopub.execute_input":"2024-03-05T13:13:13.200322Z","iopub.status.idle":"2024-03-05T13:13:13.232330Z","shell.execute_reply.started":"2024-03-05T13:13:13.200281Z","shell.execute_reply":"2024-03-05T13:13:13.231384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if PROCESS_DATA:\n# 1. LOAD TARGETS DATASET\ntarget_df = cudf.read_csv('../input/amex-default-prediction/train_labels.csv')\n# TRANSFORM ID COLUMN: to reduce column size from 16 Bytes hexadecimal string to 4 Bytes integer; avoid high memory use\ntarget_df['customer_ID'] = target_df['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\nprint(f'There are {target_df.shape[0]} train targets')\n\n# GET TRAIN COLUMN NAMES\nT_COLS = cudf.read_csv('../input/amex-default-prediction/train_data.csv', nrows=1).columns\nprint(f'There are {len(T_COLS)} columns')\n\n# HASHING: we are going to use customer_id as buckets and then max 13 rows per customer will be moved to a separate file for the customer\n# WHY : To reduce the read size and avoid memory error\n# THEN : we will have a file per customer_id and we will use it to train\n\n#     # 2. LOAD TRAIN DATASET\n#     if PATH_TO_CUSTOMER_HASHES:\n#         train = cudf.read_parquet(f'{PATH_TO_CUSTOMER_HASHES}train_customer_hashes.pqt')\n#     else:\n# only fetch customer_id col\ntrain_df = pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', usecols=['customer_ID'])\n# TRANSFORM ID COLUMN: to reduce column size; avoid high memory use and and relate target to train\ntrain_df['customer_ID'] = train_df['customer_ID'].apply(lambda x: int(x[-16:],16) ).astype('int64')\n\ncustomer_ids = train_df.drop_duplicates().sort_index().values.flatten()\nprint(f'There are {len(customer_ids)} unique customers in train.')","metadata":{"execution":{"iopub.status.busy":"2024-03-05T12:41:47.622343Z","iopub.execute_input":"2024-03-05T12:41:47.622935Z","iopub.status.idle":"2024-03-05T12:46:53.480964Z","shell.execute_reply.started":"2024-03-05T12:41:47.622904Z","shell.execute_reply":"2024-03-05T12:46:53.480024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_wise_rows = split_rows_per_file(customer_ids, train_df, num_files = NUM_FILES, verbose = 'train')","metadata":{"execution":{"iopub.status.busy":"2024-03-05T12:54:39.473059Z","iopub.execute_input":"2024-03-05T12:54:39.474034Z","iopub.status.idle":"2024-03-05T12:54:40.086339Z","shell.execute_reply.started":"2024-03-05T12:54:39.473991Z","shell.execute_reply":"2024-03-05T12:54:40.085456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CREATE PROCESSED TRAIN FILES AND SAVE TO DISK        \nfor i in tqdm(range(NUM_FILES), desc=\"Process Chunks\"):\n\n    # READ CHUNK OF TRAIN CSV FILE based on index i of the file and number of rows to skip as offset from the size of the file size\n\n    # calculate offset/rows to skip\n    ith_file_rows_size = np.sum( file_wise_rows[:i]) # it will return a 3D numpy array where first dimension is of size i starting from 0 then take sum to find the num of rows\n    print(f\"ith_file_rows_size: {ith_file_rows_size}\")\n\n     # the plus one is for skipping header\n    skip_rows = int( ith_file_rows_size + 1)\n\n    # load chunk of training csv\n    train_df = cudf.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', nrows=file_wise_rows[i], skiprows=skip_rows, header=None, names=T_COLS)\n    \n    # FEATURE ENGINEER\n    train_df = feature_engineer(train_df = train_df, target_df = target_df)\n\n    # SAVE FILES\n\n    # Unmerge the traget values from train df\n    target_df = train_df[['customer_ID','target']].drop_duplicates().sort_index()\n\n    if not os.path.exists(PATH_TO_DATA):\n        os.makedirs(PATH_TO_DATA)\n\n    target_df.to_parquet(f'{PATH_TO_DATA}targets_{i+1}.pqt',index=False)\n\n    print(f'Train_File_{i+1} has {train_df.customer_ID.nunique()} customers and shape',train_df.shape)\n    # reshape before storing\n    file_data = train_df.iloc[:,1:-1].values.reshape((-1,13,188))\n    cupy.save(f'{PATH_TO_DATA}data_{i+1}',file_data.astype('float32'))\n\n# CLEAN MEMORY\ndel train_df, target_df, file_data\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}