{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## American Express prediction with Tensorflow","metadata":{"id":"AWiNntCXj3Uf"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport math\nfrom tensorflow import feature_column\nimport gc\nfrom tensorflow import keras\nfrom tensorflow.keras.callbacks import LearningRateScheduler\nimport os\nfrom pathlib import Path\ndef get_cosine_decay_learning_rate_scheduler(epochs, cycles=1, lr_start=0.001, lr_end=0.0002):\n    def cosine_decay(epoch):\n        epochs_per_cycle = epochs // cycles\n        epoch_in_cycle = epoch % epochs_per_cycle\n        if epochs_per_cycle > 1:\n            w = (1 + math.cos(epoch_in_cycle / (epochs_per_cycle-1) * math.pi)) / 2\n        else:\n            w = 1\n        return w * lr_start + (1 - w) * lr_end\n    return LearningRateScheduler(cosine_decay, verbose=0)\n\ndef get_kaggle_dataset(url, base_path):\n    directory_name = url.split(\"/\")[-1]\n    kaggle_default_directory = f\"/kaggle/input/{directory_name}\"\n    if os.path.exists(kaggle_default_directory):\n        return Path(kaggle_default_directory)\n    else:\n        \n        try: \n          import opendatasets as od\n        except:\n          os.system(\"pip install opendatasets\")\n          import opendatasets as od\n        od.download(url, data_dir=base_path) \n        directory_path = os.path.join(base_path, directory_name)\n        if os.path.exists(directory_path):\n            return Path(directory_path)\n        else:\n            return None","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-28T11:07:20.031538Z","iopub.execute_input":"2022-05-28T11:07:20.032372Z","iopub.status.idle":"2022-05-28T11:07:20.041331Z","shell.execute_reply.started":"2022-05-28T11:07:20.032331Z","shell.execute_reply":"2022-05-28T11:07:20.040595Z"},"id":"DnjBnjSpj3Ui","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading data","metadata":{"id":"x67Eosofj3Uk"}},{"cell_type":"code","source":"batch_size = 8192\nsample_count = 5531451\nis_training = False\nis_kaggle = os.path.exists(\"/kaggle/input/\")\nepochs = 100","metadata":{"execution":{"iopub.status.busy":"2022-05-28T11:07:24.393034Z","iopub.execute_input":"2022-05-28T11:07:24.393512Z","iopub.status.idle":"2022-05-28T11:07:24.398415Z","shell.execute_reply.started":"2022-05-28T11:07:24.393471Z","shell.execute_reply":"2022-05-28T11:07:24.397299Z"},"id":"-0EbpEVdj3Uk","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_path = get_kaggle_dataset(\"https://www.kaggle.com/competitions/amex-default-prediction\", \"/content\")","metadata":{"id":"zuvpVdrekRGl","outputId":"a00385b8-0880-4f39-ffed-0a90b49521c1"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Target Lookup layer","metadata":{"id":"lRxkR-i2j3Ul"}},{"cell_type":"code","source":"if is_training:\n    train_labels = pd.read_csv(\n        dataset_path / \"train_labels.csv\"\n    )\n    keys_tensor = tf.constant(train_labels['customer_ID'])\n    vals_tensor = tf.constant(train_labels['target'])\n    input_tensor = tf.constant(train_labels['customer_ID'].iloc[0:100])\n    target_lookup = tf.lookup.StaticHashTable(\n        tf.lookup.KeyValueTensorInitializer(keys_tensor, vals_tensor),\n        default_value=-1\n    )\n    print(target_lookup.lookup(input_tensor).numpy())","metadata":{"execution":{"iopub.status.busy":"2022-05-28T11:07:28.735571Z","iopub.execute_input":"2022-05-28T11:07:28.735958Z","iopub.status.idle":"2022-05-28T11:07:28.743385Z","shell.execute_reply.started":"2022-05-28T11:07:28.735925Z","shell.execute_reply":"2022-05-28T11:07:28.742159Z"},"id":"YHgUin2oj3Ul","outputId":"7c49b9cc-2025-4069-c016-6229eac0c538","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Training dataset","metadata":{"id":"NAYZqqnuj3Um"}},{"cell_type":"code","source":"categorical_columns = ['D_63', 'D_64']\ncategory_dictionary = {\n    \"D_63\": ['CR' 'CO' 'CL' 'XZ' 'XM' 'XL'],\n    \"D_64\": ['O' 'R', '', 'U' '-1']\n}\ncategory_type_dictionary = {\n    \"D_63\": \"string\", \n    \"D_64\": \"string\"\n}\nnon_numerical_columns = [\"S_2\"] + categorical_columns\nmissing_numeric_columns = [\n    'D_49', 'D_73', 'D_76', 'R_9', 'B_29', 'D_87', \n    'D_88', 'D_106', 'R_26', 'D_108', 'D_110', 'D_111', 'B_39', 'B_42', \n    'D_132', 'D_134', 'D_135', 'D_136', 'D_137', 'D_138', 'D_142'\n]\ngood_numerical_columns = [\n    'P_2', 'D_39', 'B_1', 'B_2', 'R_1', 'S_3', 'D_41', 'B_3', \n    'D_42', 'D_43', 'D_44', 'B_4', 'D_45', 'B_5', 'R_2', 'D_46', \n    'D_47', 'D_48', 'B_6', 'B_7', 'B_8', 'D_50', 'D_51', 'B_9', \n    'R_3', 'D_52', 'P_3', 'B_10', 'D_53', 'S_5', 'B_11', 'S_6', \n    'D_54', 'R_4', 'S_7', 'B_12', 'S_8', 'D_55', 'D_56', 'B_13', \n    'R_5', 'D_58', 'S_9', 'B_14', 'D_59', 'D_60', 'D_61', 'B_15', \n    'S_11', 'D_62', 'D_65', 'B_16', 'B_17', 'B_18', 'B_19', 'D_66', \n    'B_20', 'D_68', 'S_12', 'R_6', 'S_13', 'B_21', 'D_69', 'B_22', \n    'D_70', 'D_71', 'D_72', 'S_15', 'B_23', 'P_4', 'D_74', 'D_75', \n    'B_24', 'R_7', 'D_77', 'B_25', 'B_26', 'D_78', 'D_79', 'R_8', \n    'S_16', 'D_80', 'R_10', 'R_11', 'B_27', 'D_81', 'D_82', 'S_17', \n    'R_12', 'B_28', 'R_13', 'D_83', 'R_14', 'R_15', 'D_84', 'R_16', \n    'B_30', 'S_18', 'D_86', 'R_17', 'R_18', 'S_19', 'R_19', 'B_32', \n    'S_20', 'R_20', 'R_21', 'B_33', 'D_89', 'R_22', 'R_23', 'D_91', \n    'D_92', 'D_93', 'D_94', 'R_24', 'R_25', 'D_96', 'S_22', 'S_23', \n    'S_24', 'S_25', 'S_26', 'D_102', 'D_103', 'D_104', 'D_105', \n    'D_107', 'B_36', 'B_37', 'R_27', 'B_38', 'D_109', 'D_112', \n    'B_40', 'S_27', 'D_113', 'D_114', 'D_115', 'D_116', \n    'D_117', 'D_118', 'D_119', 'D_120', 'D_121', 'D_122', \n    'D_123', 'D_124', 'D_125', 'D_126', 'D_127', 'D_128', \n    'D_129', 'B_41', 'D_130', 'D_131', 'D_133', 'R_28', \n    'D_139', 'D_140', 'D_141', 'D_143', 'D_144', 'D_145', 'B_31'\n]\nnumerical_columns = missing_numeric_columns + good_numerical_columns","metadata":{"execution":{"iopub.status.busy":"2022-05-28T11:07:41.825033Z","iopub.execute_input":"2022-05-28T11:07:41.825423Z","iopub.status.idle":"2022-05-28T11:07:41.843035Z","shell.execute_reply.started":"2022-05-28T11:07:41.825389Z","shell.execute_reply":"2022-05-28T11:07:41.841533Z"},"id":"x9yb0_Lxj3Um","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_dataset(pattern, missing_numeric_columns, epochs=1, batch_size=4096, validation_split=0.1):\n    def preprocess(features):\n        features.pop(\"S_2\")\n        for column in missing_numeric_columns:\n            features[column] = tf.where(tf.strings.length(features[column]) == 0, \"0.0\", features[column])\n            features[column] = tf.strings.to_number(features[column])\n        customer_ids = features.pop(\"customer_ID\")\n        return features, target_lookup.lookup(customer_ids)\n    num_batchs = sample_count // batch_size\n    train_size = int(num_batchs * (1 - 0.1))\n    validation_size = num_batchs - train_size\n    dataset = tf.data.experimental.make_csv_dataset(\n        pattern, \n        batch_size, \n        shuffle=False,\n        num_parallel_reads=8,\n        num_epochs=1\n    )\n    dataset = dataset.map(preprocess)\n    batch = dataset.take(1)\n    train_ds = dataset.take(train_size)\n    valid_ds = dataset.skip(train_size).take(validation_size)\n    train_ds = train_ds.shuffle(32).cache().prefetch(tf.data.AUTOTUNE).repeat(epochs)\n    valid_ds = valid_ds.cache().prefetch(tf.data.AUTOTUNE).repeat(epochs)\n    return train_ds, valid_ds, batch\n\ndef create_test_dataset(pattern, missing_numeric_columns, batch_size=32):\n    dataset = tf.data.experimental.make_csv_dataset(\n        pattern, \n        batch_size, \n        shuffle=False,\n        num_parallel_reads=8,\n        num_epochs=1\n    )\n    def preprocess_test(features):\n        date = features.pop(\"S_2\")\n        for column in missing_numeric_columns:\n            features[column] = tf.where(tf.strings.length(features[column]) == 0, \"0.0\", features[column])\n            features[column] = tf.strings.to_number(features[column])\n        customer_ids = features.pop(\"customer_ID\")\n        return features, customer_ids\n    dataset = dataset.map(preprocess_test)\n    dataset = dataset.prefetch(tf.data.AUTOTUNE)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2022-05-28T11:57:43.368223Z","iopub.execute_input":"2022-05-28T11:57:43.368815Z","iopub.status.idle":"2022-05-28T11:57:43.387626Z","shell.execute_reply.started":"2022-05-28T11:57:43.368774Z","shell.execute_reply":"2022-05-28T11:57:43.386778Z"},"id":"CyNZ2dr2j3Un","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_training:\n    train_ds, valid_ds, batch = create_dataset(str(dataset_path / \"train_data.csv\"), epochs=epochs, missing_numeric_columns=missing_numeric_columns, batch_size=batch_size)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T11:09:35.516132Z","iopub.execute_input":"2022-05-28T11:09:35.51655Z","iopub.status.idle":"2022-05-28T11:09:35.522384Z","shell.execute_reply.started":"2022-05-28T11:09:35.516517Z","shell.execute_reply":"2022-05-28T11:09:35.521246Z"},"id":"jhcb5So5j3Uo","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess data","metadata":{"id":"l_H4c_-Dj3Up"}},{"cell_type":"code","source":"if is_training:\n    inputs = dict()\n    numerical_features = [feature_column.numeric_column(key) for key in numerical_columns]\n    for item in numerical_columns:\n        inputs[item] = tf.keras.layers.Input(name=item, shape=())\n    categorical_features = [\n        feature_column.indicator_column(\n            feature_column.categorical_column_with_vocabulary_list(key, category_dictionary[key])\n        )\n        for key in categorical_columns\n    ]\n    for item in categorical_features:\n        inputs[item.categorical_column.key] = tf.keras.layers.Input(name=item.categorical_column.key, shape=(), dtype=category_type_dictionary[item.categorical_column.key])","metadata":{"execution":{"iopub.status.busy":"2022-05-28T11:09:38.488157Z","iopub.execute_input":"2022-05-28T11:09:38.488574Z","iopub.status.idle":"2022-05-28T11:09:38.496592Z","shell.execute_reply.started":"2022-05-28T11:09:38.488544Z","shell.execute_reply":"2022-05-28T11:09:38.49492Z"},"id":"Z5zKoH8Yj3Up","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling","metadata":{"id":"VBt9ENCXj3Uq"}},{"cell_type":"code","source":"def get_model():\n    deep = keras.layers.DenseFeatures(numerical_features + categorical_features, name='deep')(inputs)\n    deep = keras.layers.Dense(128, activation='swish')(deep)\n    deep = keras.layers.Dropout(0.3)(deep)\n    deep = keras.layers.Dense(64, activation='swish')(deep)\n    deep = keras.layers.Dropout(0.3)(deep)\n    deep = keras.layers.Dense(32, activation='swish')(deep)\n    deep = keras.layers.Dropout(0.3)(deep)\n    deep = keras.layers.Dense(16, activation='swish')(deep)\n    deep = keras.layers.Dropout(0.3)(deep)\n    output = keras.layers.Dense(1, activation=\"sigmoid\")(deep)\n    model = keras.Model(inputs=list(inputs.values()), outputs=output)\n    auc = keras.metrics.AUC(name=\"auc\")\n    model.compile(optimizer=\"adam\", loss=\"binary_crossentropy\", metrics=[\"accuracy\", auc])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-27T17:20:09.393583Z","iopub.execute_input":"2022-05-27T17:20:09.394022Z","iopub.status.idle":"2022-05-27T17:20:09.403756Z","shell.execute_reply.started":"2022-05-27T17:20:09.393987Z","shell.execute_reply":"2022-05-27T17:20:09.40296Z"},"id":"OoijFmpxj3Uq","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_training:\n    model = get_model()\nelse:\n    model = keras.models.load_model(\"../input/amex-models/models\")\ntf.keras.utils.plot_model(model, show_shapes=True, rankdir='LR')","metadata":{"execution":{"iopub.status.busy":"2022-05-28T11:09:40.774237Z","iopub.execute_input":"2022-05-28T11:09:40.775584Z","iopub.status.idle":"2022-05-28T11:09:52.399289Z","shell.execute_reply.started":"2022-05-28T11:09:40.775535Z","shell.execute_reply":"2022-05-28T11:09:52.393809Z"},"id":"Cm2RPhPnj3Uq","outputId":"aaf1574f-d663-446e-bfd3-1cd56e63c537","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loaidng Google Drive","metadata":{"id":"IdEWfWhoqW4E"}},{"cell_type":"code","source":"if not is_kaggle:\n    from google.colab import drive\n    drive.mount('/content/drive')\n    model_path = Path(\"/content/drive/MyDrive/amex\")\n    if not os.path.exists(model_path):\n      print(\"Make directory:\", model_path)\n      os.makedirs(model_path)\nelse:\n    model_path = Path(\"/kaggle/working\")","metadata":{"id":"PZlEjwgwqcOx","outputId":"de3e722b-3e29-46be-8679-35cf7ff66083"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Training","metadata":{"id":"xBNFbNaBj3Ur"}},{"cell_type":"code","source":"if is_training:\n    num_batchs = sample_count // batch_size\n    train_size = int(num_batchs * (1 - 0.1))\n    validation_size = num_batchs - train_size\n    cp = keras.callbacks.ModelCheckpoint(model_path / \"models\", monitor=\"val_accuracy\", mode=\"max\", save_best_only=True, restore_best_weights=True)\n    scheduler = get_cosine_decay_learning_rate_scheduler(epochs)       \n    model.fit(\n        train_ds, \n        validation_data=valid_ds, \n        epochs=epochs, \n        steps_per_epoch=train_size,\n        validation_steps=validation_size,\n        callbacks=[cp, scheduler]\n    )","metadata":{"execution":{"iopub.status.busy":"2022-05-27T17:20:36.664948Z","iopub.execute_input":"2022-05-27T17:20:36.665379Z"},"id":"agEA6nJ9j3Ur","outputId":"2702e4b9-5727-4cef-a4b5-6114d3501925","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission\nSubmitting takes about 5000 seconds. Luckily it doesn't need a GPU to boost this process, at the same time using GPU doesn't help improving speed. You can also try to optimize this speed if possible.","metadata":{"id":"auFEIe6Aj3Ur"}},{"cell_type":"code","source":"missing_numeric_columns = ['R_9', 'D_87', 'D_88', 'D_108', 'D_110', 'D_111', 'B_39', 'B_42', 'D_134', 'D_135', 'D_136', 'D_137', 'D_138']","metadata":{"execution":{"iopub.status.busy":"2022-05-28T11:57:53.138891Z","iopub.execute_input":"2022-05-28T11:57:53.139936Z","iopub.status.idle":"2022-05-28T11:57:53.14473Z","shell.execute_reply.started":"2022-05-28T11:57:53.139873Z","shell.execute_reply":"2022-05-28T11:57:53.14392Z"},"id":"flwYVRBUj3Us","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = create_test_dataset(\n    str(dataset_path / \"test_data.csv\"),\n    missing_numeric_columns=missing_numeric_columns,\n    batch_size=16384\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T11:57:57.722321Z","iopub.execute_input":"2022-05-28T11:57:57.723525Z","iopub.status.idle":"2022-05-28T11:57:58.673123Z","shell.execute_reply.started":"2022-05-28T11:57:57.723422Z","shell.execute_reply":"2022-05-28T11:57:58.672235Z"},"id":"pdICigKGj3Us","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport time\ncounter = 1\nif not is_training:\n    result = pd.DataFrame()\n    begin = time.time()\n    for batch, customer_ids in test_ds:\n        y_preds = model.predict(batch).reshape(-1)\n        df = pd.DataFrame({\"customer_ID\": customer_ids.numpy().reshape(-1), \"prediction\":  y_preds})\n        result = pd.concat([result, df])\n        if counter % 10 == 0:\n            elapsed = time.time() - begin\n            print(f\"{counter} batchs / {elapsed}s\")\n        counter += 1\n    result[\"customer_ID\"] = result[\"customer_ID\"].apply(lambda item: str(item, encoding=\"utf-8\"))\n    res_df = result.groupby(\"customer_ID\")[\"prediction\"].mean().reset_index()\n    res_df.to_csv(\"submission.csv\", index=False)\n    res_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T11:59:28.16214Z","iopub.execute_input":"2022-05-28T11:59:28.163314Z"},"id":"ojh8CAtaj3Us","trusted":true},"execution_count":null,"outputs":[]}]}