{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 時間序列 GRU TensorFlow 入門筆記本\n在這個筆記本中，我們展示了時間序列 GRU 模型的起始代碼和用於將 Kaggle 的 50GB CSV 文件處理成多個保存的 NumPy 文件的起始代碼。使用時間序列 GRU 允許我們使用所有提供的客戶數據，而不僅僅是客戶的最後一個數據點。我們在 [此處](https://www.kaggle.com/cdeotte/time-series-eda) 發布了時間序列數據圖。在這個筆記本中，我們\n* 將數據幀中的訓練數據處理成維度為“num_of_customers x 13 x 188”的 3D NumPy 數組\n* 將處理後的數組保存為磁盤上的多個 NumPy 文件\n* 接下來我們從磁盤上的多個文件構建和訓練 GRU\n* 我們計算驗證分數並達到 0.787\n* 最後我們處理並保存測試數據，推斷測試，並創建提交\n\n請務必注意，每次運行此筆記本時，您**不需要**處理訓練和測試文件。僅在您設計新功能時再次處理數據。否則，將您保存的 NumPy 數組上傳到 Kaggle 數據集（或使用我的 Kaggle 數據集 [此處][2]）。然後當你自定義和改進你的 GRU 模型時，設置變量 `PROCESS_DATA = False` 和 `PATH_TO_DATA = [the path to your kaggle dataset]`。\n\n要查看可以幫助您直觀了解特徵工程和改進模型架構的時間序列 EDA，請參閱我的其他筆記本 [此處](https://www.kaggle.com/datasets/cdeotte/amex-data-for-transformers-and-rnns)。請注意，在下面的代碼中，我們將 GPU 劃分為 8GB 用於 RAPIDS（特徵工程）和 8GB 用於 TensorFlow（模型構建和訓練）。","metadata":{}},{"cell_type":"code","source":"import os\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nprint('Using TensorFlow version',tf.__version__)\n\n# 將 TensorFlow 限制為 8GB 的 GPU RAM\n# 所以我們有 8GB RAM 用於 RAPIDS\nLIMIT = 8\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n  try:\n    tf.config.experimental.set_virtual_device_configuration(\n        gpus[0],\n        [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024*LIMIT)])\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n  except RuntimeError as e:\n    print(e)\nprint('We will restrict TensorFlow to max %iGB GPU RAM'%LIMIT)\nprint('then RAPIDS can use %iGB GPU RAM'%(16-LIMIT))","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-31T17:30:06.374077Z","iopub.execute_input":"2022-05-31T17:30:06.374784Z","iopub.status.idle":"2022-05-31T17:30:14.868432Z","shell.execute_reply.started":"2022-05-31T17:30:06.374688Z","shell.execute_reply":"2022-05-31T17:30:14.867546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 處理訓練數據\n我們分塊處理訓練和測試數據。 我們將訓練數據分成 10 個部分，分別處理每個部分並保存到磁盤。 我們將測試分為 20 個部分。 這使我們能夠避免處理過程中的內存錯誤。 我們還可以在比 CPU 更快的 GPU 上執行處理。 關於數據預處理的討論在 [這裡](https://www.kaggle.com/competitions/amex-default-prediction/discussion/327828) 和 [這裡](https://www.kaggle.com/competitions/amex-default-prediction/discussion/327828)\n​​​","metadata":{}},{"cell_type":"code","source":"# 僅加載第一列火車或測試很慢\n# 你可以從我的數據集中加載第一列\n# 否則將變量設置為無以從 KAGGLE 的原始數據幀中加載\nPATH_TO_CUSTOMER_HASHES = '../input/amex-data-files/'\n\n# 處理一次數據後，上傳到 KAGGLE 數據集\n# 然後將下面的變量設置為 FALSE\n# 並將數據集附加到筆記本並將路徑放在下面的數據集\nPROCESS_DATA = True\n#PATH_TO_DATA = '../input/amex-data-files//data/'\nPATH_TO_DATA = '../input/amex-data-for-transformers-and-rnns/data/'\n\n# 訓練模型後，上傳到 KAGGLE 數據集\n# 然後將下面的變量設置為 FALSE\n# 並將數據集附加到筆記本並將路徑放在下面的數據集\nTRAIN_MODEL = True\n#PATH_TO_MODEL = './model/'\nPATH_TO_MODEL = '../input/amex-data-for-transformers-and-rnns/model/'\n\nINFER_TEST = True","metadata":{"execution":{"iopub.status.busy":"2022-05-31T17:30:14.870006Z","iopub.execute_input":"2022-05-31T17:30:14.871000Z","iopub.status.idle":"2022-05-31T17:30:14.879247Z","shell.execute_reply.started":"2022-05-31T17:30:14.870957Z","shell.execute_reply":"2022-05-31T17:30:14.877850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cupy, cudf # GPU LIBRARIES\nimport numpy as np, pandas as pd # CPU LIBRARIES\nimport matplotlib.pyplot as plt, gc\n\nif PROCESS_DATA:\n    # 加載目標\n    targets = cudf.read_csv('../input/amex-default-prediction/train_labels.csv')\n    targets['customer_ID'] = targets['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    print(f'There are {targets.shape[0]} train targets')\n    \n    # 獲取火車列名\n    train = cudf.read_csv('../input/amex-default-prediction/train_data.csv', nrows=1)\n    T_COLS = train.columns\n    print(f'There are {len(T_COLS)} train dataframe columns')\n    \n    # GET TRAIN CUSTOMER NAMES (use pandas to avoid memory error)\n    if PATH_TO_CUSTOMER_HASHES:\n        train = cudf.read_parquet(f'{PATH_TO_CUSTOMER_HASHES}train_customer_hashes.pqt')\n    else:\n        train = pd.read_csv('/raid/Kaggle/amex/train_data.csv', usecols=['customer_ID'])\n        train['customer_ID'] = train['customer_ID'].apply(lambda x: int(x[-16:],16) ).astype('int64')\n    customers = train.drop_duplicates().sort_index().values.flatten()\n    print(f'There are {len(customers)} unique customers in train.')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-31T17:30:14.880299Z","iopub.execute_input":"2022-05-31T17:30:14.880690Z","iopub.status.idle":"2022-05-31T17:30:22.906475Z","shell.execute_reply.started":"2022-05-31T17:30:14.880654Z","shell.execute_reply":"2022-05-31T17:30:22.905584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 計算每個單獨文件的大小\ndef get_rows(customers, train, NUM_FILES = 10, verbose = ''):\n    chunk = len(customers)//NUM_FILES\n    if verbose != '':\n        print(f'We will split {verbose} data into {NUM_FILES} separate files.')\n        print(f'There will be {chunk} customers in each file (except the last file).')\n        print('Below are number of rows in each file:')\n    rows = []\n\n    for k in range(NUM_FILES):\n        if k==NUM_FILES-1: cc = customers[k*chunk:]\n        else: cc = customers[k*chunk:(k+1)*chunk]\n        s = train.loc[train.customer_ID.isin(cc)].shape[0]\n        rows.append(s)\n    if verbose != '': print( rows )\n    return rows\n\nif PROCESS_DATA:\n    NUM_FILES = 10\n    rows = get_rows(customers, train, NUM_FILES = NUM_FILES, verbose = 'train')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-31T17:30:22.908789Z","iopub.execute_input":"2022-05-31T17:30:22.909298Z","iopub.status.idle":"2022-05-31T17:30:23.552430Z","shell.execute_reply.started":"2022-05-31T17:30:22.909256Z","shell.execute_reply":"2022-05-31T17:30:23.551144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 預處理和特徵工程\n下面的函數處理數據。 描述該過程的討論是 [這裡](https://www.kaggle.com/competitions/amex-default-prediction/discussion/327828) 和 [這裡](https://www.kaggle.com/competitions/amex-default-prediction/discussion/328054)。 目前下面的代碼使用 [RAPIDS](https://rapids.ai/) 和 GPU 來\n* 通過轉換為 int64 減少 customer_ID 列的內存使用\n* 減少日期時間列的內存使用量（然後刪除該列）。\n* 我們填寫 NAN\n* 標籤對分類列進行編碼\n* 我們減少列的內存使用 dtypes\n* 將每個客戶轉換為序列長度為 13、特徵長度為 188 的 3D 數組\n\n要改進此模型，請嘗試添加新功能。 列已重新排列，首先具有 11 個分類特徵。 這使得以後構建 TensorFlow 模型更加容易。 我們也可以嘗試添加標準縮放器。 目前使用的數據沒有從原始的 Kaggle 訓練數據進行縮放。","metadata":{}},{"cell_type":"code","source":"def feature_engineer(train, PAD_CUSTOMER_TO_13_ROWS = True, targets = None):\n        \n    # 減少字符串列\n    # 分別從 64 字節到 8 字節，以及 10 字節到 3 字節\n    train['customer_ID'] = train['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    train.S_2 = cudf.to_datetime( train.S_2 )\n    train['year'] = (train.S_2.dt.year-2000).astype('int8')\n    train['month'] = (train.S_2.dt.month).astype('int8')\n    train['day'] = (train.S_2.dt.day).astype('int8')\n    del train['S_2']\n        \n    # LABEL ENCODE CAT COLUMNS（並減少到 1 個字節）\n    # with 0: padding, 1: nan, 2,3,4,etc: values\n    d_63_map = {'CL':2, 'CO':3, 'CR':4, 'XL':5, 'XM':6, 'XZ':7}\n    train['D_63'] = train.D_63.map(d_63_map).fillna(1).astype('int8')\n\n    d_64_map = {'-1':2,'O':3, 'R':4, 'U':5}\n    train['D_64'] = train.D_64.map(d_64_map).fillna(1).astype('int8')\n    \n    CATS = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_66', 'D_68']\n    OFFSETS = [2,1,2,2,3,2,3,2,2] #2 減去全列 csv 中的最小值\n    # 那麼 0 將是填充，1 將是 NAN，2,3,4 等將是值\n    for c,s in zip(CATS,OFFSETS):\n        train[c] = train[c] + s\n        train[c] = train[c].fillna(1).astype('int8')\n    CATS += ['D_63','D_64']\n    \n    # 在此處添加新功能\n    # 示例：train['feature_189'] = etc etc etc\n    # 示例：train['feature_190'] = etc etc etc\n    # IF CATEGORICAL, 然後添加到CATS WITH: CATS += ['feaure_190'] etc etc etc\n    # 減少內存 DTYPE\n    SKIP = ['customer_ID','year','month','day']\n    for c in train.columns:\n        if c in SKIP: continue\n        if str( train[c].dtype )=='int64':\n            train[c] = train[c].astype('int32')\n        if str( train[c].dtype )=='float64':\n            train[c] = train[c].astype('float32')\n            \n    # 墊行所以每個客戶有 13 行\n    if PAD_CUSTOMER_TO_13_ROWS:\n        tmp = train[['customer_ID']].groupby('customer_ID').customer_ID.agg('count')\n        more = cupy.array([],dtype='int64') \n        for j in range(1,13):\n            i = tmp.loc[tmp==j].index.values\n            more = cupy.concatenate([more,cupy.repeat(i,13-j)])\n        df = train.iloc[:len(more)].copy().fillna(0)\n        df = df * 0 - 1 #pad numerical columns with -1\n        df[CATS] = (df[CATS] * 0).astype('int8') #pad categorical columns with 0\n        df['customer_ID'] = more\n        train = cudf.concat([train,df],axis=0,ignore_index=True)\n        \n    # 添加目標（並減少到 1 個字節）\n    if targets is not None:\n        train = train.merge(targets,on='customer_ID',how='left')\n        train.target = train.target.astype('int8')\n        \n    # FILL NAN\n    train = train.fillna(-0.5) #這適用於數字列\n    \n    # 按客戶然後日期排序\n    train = train.sort_values(['customer_ID','year','month','day']).reset_index(drop=True)\n    train = train.drop(['year','month','day'],axis=1)\n    \n    # REARRANGE COLUMNS WITH 11 CATS FIRST 首先用 11 隻貓重新排列列\n    COLS = list(train.columns[1:])\n    COLS = ['customer_ID'] + CATS + [c for c in COLS if c not in CATS]\n    train = train[COLS]\n    \n    return train","metadata":{"execution":{"iopub.status.busy":"2022-05-31T17:30:23.554168Z","iopub.execute_input":"2022-05-31T17:30:23.555015Z","iopub.status.idle":"2022-05-31T17:30:23.587721Z","shell.execute_reply.started":"2022-05-31T17:30:23.554968Z","shell.execute_reply":"2022-05-31T17:30:23.586395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PROCESS_DATA:\n    # CREATE PROCESSED TRAIN FILES AND SAVE TO DISK 創建已處理的火車文件並保存到磁盤  \n    for k in range(NUM_FILES):\n\n        # READ CHUNK OF TRAIN CSV FILE 讀取火車 CSV 文件的塊\n        skip = int(np.sum( rows[:k] ) + 1) #the plus one is for skipping header 他加一是為了跳過標題\n        train = cudf.read_csv('../input/amex-default-prediction/train_data.csv', nrows=rows[k], \n                              skiprows=skip, header=None, names=T_COLS)\n\n        # FEATURE ENGINEER DATAFRAME 特徵工程師數據框v\n        train = feature_engineer(train, targets = targets)\n\n        # SAVE FILES 保存文件\n        print(f'Train_File_{k+1} has {train.customer_ID.nunique()} customers and shape',train.shape)\n        tar = train[['customer_ID','target']].drop_duplicates().sort_index()\n        if not os.path.exists(PATH_TO_DATA): os.makedirs(PATH_TO_DATA)\n        tar.to_parquet(f'{PATH_TO_DATA}targets_{k+1}.pqt',index=False)\n        data = train.iloc[:,1:-1].values.reshape((-1,13,188))\n        cupy.save(f'{PATH_TO_DATA}data_{k+1}',data.astype('float32'))\n\n    # CLEAN MEMORY 乾淨的記憶\n    del train, tar, data\n    del targets\n    gc.collect()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-31T17:30:23.594973Z","iopub.execute_input":"2022-05-31T17:30:23.598502Z","iopub.status.idle":"2022-05-31T17:36:57.255105Z","shell.execute_reply.started":"2022-05-31T17:30:23.598448Z","shell.execute_reply":"2022-05-31T17:36:57.252676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 構建模型\n我們只是將序列數據輸入到一個基本的 GRU 中。 我們將遵循兩個密集層，最後是一個 sigmoid 輸出來預測默認值。 嘗試改進模型架構。","metadata":{}},{"cell_type":"code","source":"# SIMPLE GRU MODEL 簡單的 GRU 模型\ndef build_model():\n    \n    # INPUT - FIRST 11 COLUMNS ARE CAT, NEXT 177 ARE NUMERIC 輸入 - 前 11 列是 CAT，接下來的 177 列是數字\n    inp = tf.keras.Input(shape=(13,188))\n    embeddings = []\n    for k in range(11):\n        emb = tf.keras.layers.Embedding(10,4)\n        embeddings.append( emb(inp[:,:,k]) )\n    x = tf.keras.layers.Concatenate()([inp[:,:,11:]]+embeddings)\n    \n    # SIMPLE RNN BACKBONE 簡單的 RNN 主幹\n    x = tf.keras.layers.GRU(units=128, return_sequences=False)(x)\n    x = tf.keras.layers.Dense(64,activation='relu')(x)\n    x = tf.keras.layers.Dense(32,activation='relu')(x)\n    \n    # OUTPUT 輸出\n    x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n    \n    # COMPILE MODEL 編譯模型\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    loss = tf.keras.losses.BinaryCrossentropy()\n    model.compile(loss=loss, optimizer = opt)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-31T17:36:57.260670Z","iopub.execute_input":"2022-05-31T17:36:57.261558Z","iopub.status.idle":"2022-05-31T17:36:57.278391Z","shell.execute_reply.started":"2022-05-31T17:36:57.261512Z","shell.execute_reply":"2022-05-31T17:36:57.277143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CUSTOM LEARNING SCHEUDLE 自定義學習計劃\ndef lrfn(epoch):\n    lr = [1e-3]*5 + [1e-4]*2 + [1e-5]*1\n    return lr[epoch]\nLR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = False)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T17:36:57.280868Z","iopub.execute_input":"2022-05-31T17:36:57.281517Z","iopub.status.idle":"2022-05-31T17:36:57.301659Z","shell.execute_reply.started":"2022-05-31T17:36:57.281476Z","shell.execute_reply":"2022-05-31T17:36:57.300443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 競賽指標代碼\n下面的代碼來自 Konstantin Yakovlev 的討論帖 [這裡](https://www.kaggle.com/competitions/amex-default-prediction/discussion/327534)","metadata":{}},{"cell_type":"code","source":"# Konstantin Yakovlev 的競爭指標\n# https://www.kaggle.com/kyakovlev\n# https://www.kaggle.com/competitions/amex-default-prediction/discussion/327534\ndef amex_metric_mod(y_true, y_pred):\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 0.5 * (gini[1]/gini[0] + top_four)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T17:36:57.303268Z","iopub.execute_input":"2022-05-31T17:36:57.303800Z","iopub.status.idle":"2022-05-31T17:36:57.318022Z","shell.execute_reply.started":"2022-05-31T17:36:57.303740Z","shell.execute_reply":"2022-05-31T17:36:57.316814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 訓練模型\n我們訓練 5 次折疊，每次 8 個 epoch。 我們保存 5 折模型以供稍後進行測試推理。 TRAIN_MODEL，請在本筆記本的開頭設置變量 `TRAIN_MODEL = False`。","metadata":{}},{"cell_type":"code","source":"if TRAIN_MODEL:\n    # SAVE TRUE AND OOF\n    true = np.array([])\n    oof = np.array([])\n    VERBOSE = 2 # use 1 for interactive \n\n    for fold in range(5):\n\n        # INDICES OF TRAIN AND VALID FOLDS 火車和有效折疊指數\n        valid_idx = [2*fold+1, 2*fold+2]\n        train_idx = [x for x in [1,2,3,4,5,6,7,8,9,10] if x not in valid_idx]\n\n        print('#'*25)\n        print(f'### Fold {fold+1} with valid files', valid_idx)\n\n        # READ TRAIN DATA FROM DISK 從磁盤讀取訓練數據\n        X_train = []; y_train = []\n        for k in train_idx:\n            X_train.append( np.load(f'{PATH_TO_DATA}data_{k}.npy'))\n            y_train.append( pd.read_parquet(f'{PATH_TO_DATA}targets_{k}.pqt') )\n        X_train = np.concatenate(X_train,axis=0)\n        y_train = pd.concat(y_train).target.values\n        print('### Training data shapes', X_train.shape, y_train.shape)\n\n        # READ VALID DATA FROM DISK 從磁盤讀取有效數據\n        X_valid = []; y_valid = []\n        for k in valid_idx:\n            X_valid.append( np.load(f'{PATH_TO_DATA}data_{k}.npy'))\n            y_valid.append( pd.read_parquet(f'{PATH_TO_DATA}targets_{k}.pqt') )\n        X_valid = np.concatenate(X_valid,axis=0)\n        y_valid = pd.concat(y_valid).target.values\n        print('### Validation data shapes', X_valid.shape, y_valid.shape)\n        print('#'*25)\n\n        # BUILD AND TRAIN MODEL 建立和訓練模型\n        K.clear_session()\n        model = build_model()\n        h = model.fit(X_train,y_train, \n                      validation_data = (X_valid,y_valid),\n                      batch_size=512, epochs=8, verbose=VERBOSE,\n                      callbacks = [LR])\n        if not os.path.exists(PATH_TO_MODEL): os.makedirs(PATH_TO_MODEL)\n        model.save_weights(f'{PATH_TO_MODEL}gru_fold_{fold+1}.h5')\n\n        # INFER VALID DATA 推斷有效數據\n        print('Inferring validation data...')\n        p = model.predict(X_valid, batch_size=512, verbose=VERBOSE).flatten()\n\n        print()\n        print(f'Fold {fold+1} CV=', amex_metric_mod(y_valid, p) )\n        print()\n        true = np.concatenate([true, y_valid])\n        oof = np.concatenate([oof, p])\n        \n        # CLEAN MEMORY 乾淨的記憶\n        del model, X_train, y_train, X_valid, y_valid, p\n        gc.collect()\n\n    # PRINT OVERALL RESULTS 打印總體結果\n    print('#'*25)\n    print(f'Overall CV =', amex_metric_mod(true, oof) )\n    K.clear_session()","metadata":{"scrolled":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-31T17:36:57.323080Z","iopub.execute_input":"2022-05-31T17:36:57.323542Z","iopub.status.idle":"2022-05-31T17:46:41.900801Z","shell.execute_reply.started":"2022-05-31T17:36:57.323500Z","shell.execute_reply":"2022-05-31T17:46:41.897811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 處理測試數據\n我們以與訓練數據相同的方式處理測試數據。","metadata":{}},{"cell_type":"code","source":"if PROCESS_DATA:\n    # GET TEST COLUMN NAMES 獲取測試列名稱\n    test = cudf.read_csv('../input/amex-default-prediction/test_data.csv', nrows=1)\n    T_COLS = test.columns\n    print(f'There are {len(T_COLS)} test dataframe columns')\n    \n    # GET TEST CUSTOMER NAMES (use pandas to avoid memory error) GET TEST CUSTOMER NAMES（使用 pandas 避免內存錯誤）\n    if PATH_TO_CUSTOMER_HASHES:\n        test = cudf.read_parquet(f'{PATH_TO_CUSTOMER_HASHES}test_customer_hashes.pqt')\n    else:\n        test = pd.read_csv('/raid/Kaggle/amex/test_data.csv', usecols=['customer_ID'])\n        test['customer_ID'] = test['customer_ID'].apply(lambda x: int(x[-16:],16) ).astype('int64')\n    customers = test.drop_duplicates().sort_index().values.flatten()\n    print(f'There are {len(customers)} unique customers in test.')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-05-31T17:46:41.912245Z","iopub.execute_input":"2022-05-31T17:46:41.916080Z","iopub.status.idle":"2022-05-31T17:46:43.376020Z","shell.execute_reply.started":"2022-05-31T17:46:41.916021Z","shell.execute_reply":"2022-05-31T17:46:43.374871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_FILES = 20\nif PROCESS_DATA:\n    # CALCULATE SIZE OF EACH SEPARATE FILE 計算每個單獨文件的大小\n    rows = get_rows(customers, test, NUM_FILES = NUM_FILES, verbose = 'test')","metadata":{"execution":{"iopub.status.busy":"2022-05-31T17:46:43.377708Z","iopub.execute_input":"2022-05-31T17:46:43.378095Z","iopub.status.idle":"2022-05-31T17:46:45.319402Z","shell.execute_reply.started":"2022-05-31T17:46:43.378054Z","shell.execute_reply":"2022-05-31T17:46:45.318560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PROCESS_DATA:\n    # SAVE TEST CUSTOMERS INDEX 保存測試客戶索引\n    test_customer_hashes = cupy.array([],dtype='int64')\n    \n    # CREATE PROCESSED TEST FILES AND SAVE TO DISK 創建已處理的測試文件並保存到磁盤\n    for k in range(NUM_FILES):\n\n        # READ CHUNK OF TEST CSV FILE\n        skip = int(np.sum( rows[:k] ) + 1) #the plus one is for skipping header 他加一是為了跳過標題\n        test = cudf.read_csv('../input/amex-default-prediction/test_data.csv', nrows=rows[k], \n                              skiprows=skip, header=None, names=T_COLS)\n\n        # FEATURE ENGINEER DATAFRAME 特徵工程師數據框\n        test = feature_engineer(test, targets = None)\n        \n        # SAVE TEST CUSTOMERS INDEX 保存測試客戶索引\n        cust = test[['customer_ID']].drop_duplicates().sort_index().values.flatten()\n        test_customer_hashes = cupy.concatenate([test_customer_hashes,cust])\n\n        # SAVE FILES 保存文件\n        print(f'Test_File_{k+1} has {test.customer_ID.nunique()} customers and shape',test.shape)\n        data = test.iloc[:,1:].values.reshape((-1,13,188))\n        cupy.save(f'{PATH_TO_DATA}test_data_{k+1}',data.astype('float32'))\n        \n    # SAVE CUSTOMER INDEX OF ALL TEST FILES 保存所有測試文件的客戶索引\n    cupy.save(f'{PATH_TO_DATA}test_hashes_data', test_customer_hashes)\n\n    # CLEAN MEMORY 乾淨的記憶\n    del test, data\n    gc.collect()","metadata":{"scrolled":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-31T17:46:45.320694Z","iopub.execute_input":"2022-05-31T17:46:45.321357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 推斷測試數據\n我們從保存的折疊模型中推斷出測試數據。 如果您不想推斷測試，但您只想讓您的筆記本計算驗證分數以評估模型更改，請在此筆記本的開頭設置變量“INFER_TEST = False”。 此外，如果您希望從先前訓練的模型中進行推斷，請在本筆記本開頭的變量“PATH_TO_MODEL”中添加 Kaggle 數據集的路徑。","metadata":{}},{"cell_type":"code","source":"if INFER_TEST:\n    # INFER TEST DATA 推斷測試數據\n    start = 0; end = 0\n    sub = cudf.read_csv('../input/amex-default-prediction/sample_submission.csv')\n    \n    # REARANGE SUB ROWS TO MATCH PROCESSED TEST FILES 重新排列子行以匹配已處理的測試文件\n    sub['hash'] = sub['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    test_hash_index = cupy.load(f'{PATH_TO_DATA}test_hashes_data.npy')\n    sub = sub.set_index('hash').loc[test_hash_index].reset_index(drop=True)\n    \n    for k in range(NUM_FILES):\n        # BUILD MODEL 構建模型\n        K.clear_session()\n        model = build_model()\n        \n        # LOAD TEST DATA 加載測試數據\n        print(f'Inferring Test_File_{k+1}')\n        X_test = np.load(f'{PATH_TO_DATA}test_data_{k+1}.npy')\n        end = start + X_test.shape[0]\n\n        # INFER 5 FOLD MODELS 推斷 5 折模型\n        model.load_weights(f'{PATH_TO_MODEL}gru_fold_1.h5')\n        p = model.predict(X_test, batch_size=512, verbose=0).flatten() \n        for j in range(1,5):\n            model.load_weights(f'{PATH_TO_MODEL}gru_fold_{j+1}.h5')\n            p += model.predict(X_test, batch_size=512, verbose=0).flatten()\n        p /= 5.0\n\n        # SAVE TEST PREDICTIONS 保存測試預測\n        sub.loc[start:end-1,'prediction'] = p\n        start = end\n        \n        # CLEAN MEMORY 乾淨的記憶\n        del model, X_test, p\n        gc.collect()","metadata":{"scrolled":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 創建提交","metadata":{}},{"cell_type":"code","source":"if INFER_TEST:\n    sub.to_csv('submission.csv',index=False)\n    print('Submission file shape is', sub.shape )\n    display( sub.head() )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if INFER_TEST:\n    # DISPLAY SUBMISSION PREDICTIONS\n    plt.hist(sub.to_pandas().prediction, bins=100)\n    plt.title('Test Predictions')\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}