{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# MLP KerasTuner\n\nSpent all your time on feature engineering and tree models? You are not alone. But it's time to take a shot at building that perfect NN to round out your ensemble\n\nIf you're an NN noob like me, KerasTuner is a good way to experiment with a bunch of different architectures and see how they perform on your dataset. You can use these findings to have a backbone for your NN. And then add in the finer details like the right learning schedule, skip connections etc. to improve your model further.\n\nTrying to conserve my GPU hours for the week so I have only ran 5 trials of the tuner for 20 epochs and for a single fold. I would definitely suggest bumping up the number of trials to a much higher number. If you're running this on Kaggle, maybe 100-150 before you start hitting the notebook timeouts. \n\nhttps://www.kaggle.com/competitions/amex-default-prediction/discussion/343011\n\nThe test inference from the best model we find using kerastuner is done in a separate notebook. It was producing memory errors when I tried to do it along with everything else in this notebook. \n\n\n\nCredits:\n- https://www.kaggle.com/code/fchollet/moa-keras-kerastuner-best-practices\n- https://www.tensorflow.org/tutorials/keras/keras_tuner\n- https://www.kaggle.com/datasets/raddar/amex-data-integer-dtypes-parquet-format","metadata":{}},{"cell_type":"markdown","source":"# Load libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport cudf\nimport numpy as np\nimport tensorflow as tf\nimport keras_tuner as kt\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import OneHotEncoder\n\nfrom colorama import Fore, Back, Style\nfrom matplotlib import pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\n\nimport os\nimport gc\nimport random\nimport math\nimport pickle\nimport datetime","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:49:20.135598Z","iopub.execute_input":"2022-08-10T04:49:20.136709Z","iopub.status.idle":"2022-08-10T04:49:32.383597Z","shell.execute_reply.started":"2022-08-10T04:49:20.136023Z","shell.execute_reply":"2022-08-10T04:49:32.382095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Custom functions","metadata":{}},{"cell_type":"code","source":"def pickle_load(path):\n    import pickle\n    file = open(path,'rb')\n    loadobj = pickle.load(file)\n    file.close()\n    return loadobj","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:49:32.387451Z","iopub.execute_input":"2022-08-10T04:49:32.388785Z","iopub.status.idle":"2022-08-10T04:49:32.397764Z","shell.execute_reply.started":"2022-08-10T04:49:32.388721Z","shell.execute_reply":"2022-08-10T04:49:32.395916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pickle_dump(path, saveobj):\n    import pickle\n    filehandler = open(path,\"wb\")\n    pickle.dump(saveobj,filehandler)\n    print(\"File pickled\")\n    filehandler.close()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:49:32.400823Z","iopub.execute_input":"2022-08-10T04:49:32.402136Z","iopub.status.idle":"2022-08-10T04:49:32.427626Z","shell.execute_reply.started":"2022-08-10T04:49:32.402068Z","shell.execute_reply":"2022-08-10T04:49:32.426026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read in train data","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet')\ntrain = train.groupby(['customer_ID']).last().reset_index()\ntrain.drop(columns=['S_2'], inplace=True)\n\nlabels_df = pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\ntrain = pd.merge(train, labels_df, on=\"customer_ID\", how=\"left\")\n\ndel labels_df\n_ = gc.collect()\n\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:49:32.431422Z","iopub.execute_input":"2022-08-10T04:49:32.431826Z","iopub.status.idle":"2022-08-10T04:50:43.224814Z","shell.execute_reply.started":"2022-08-10T04:49:32.431794Z","shell.execute_reply":"2022-08-10T04:50:43.223271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nCATS = train.nunique()[train.nunique()<=10].index.tolist()\nCATS = [col for col in CATS if col not in ['target']]\n\n# Save categorical columns\npath = \"./cat_cols.pkl\"\npickle_dump(path, CATS)\n\n# Apply one-hot encoder to each column with categorical data\nOH_encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)\nOH_cols_train = pd.DataFrame(OH_encoder.fit_transform(train[CATS]), columns=OH_encoder.get_feature_names_out())\n\n# One-hot encoding removed index; put it back\nOH_cols_train.index = train.index\n\n# Remove categorical columns (will replace with one-hot encoding)\ntrain = train.drop(CATS, axis=1)\n\n# Add one-hot encoded columns to numerical features\ntrain = pd.concat([train, OH_cols_train], axis=1)\n\n# Save encoder\npath = \"./ohe_encoder.pkl\"\npickle_dump(path, OH_encoder)\n\ndel OH_cols_train\n_ = gc.collect()\n\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:50:43.226565Z","iopub.execute_input":"2022-08-10T04:50:43.227600Z","iopub.status.idle":"2022-08-10T04:50:54.481867Z","shell.execute_reply.started":"2022-08-10T04:50:43.227559Z","shell.execute_reply":"2022-08-10T04:50:54.480403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelCols = [col for col in train.columns if col not in ['target','customer_ID']]\n\nselCols = modelCols + ['target','customer_ID']\n\nONE_FOLD_ONLY = True\n\n# Save model columns\npath = \"./modelCols.pkl\"\npickle_dump(path, modelCols)\n\nlen(modelCols)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:50:54.483960Z","iopub.execute_input":"2022-08-10T04:50:54.484864Z","iopub.status.idle":"2022-08-10T04:50:54.496511Z","shell.execute_reply.started":"2022-08-10T04:50:54.484815Z","shell.execute_reply":"2022-08-10T04:50:54.494678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build MLP","metadata":{}},{"cell_type":"code","source":"def make_MLP(hp):\n        \n    inputs = tf.keras.Input(shape=(len(modelCols), ))\n    \n    activation = hp.Choice('activation', ['relu','elu','selu','tanh','swish']) \n    num_dense = hp.Int('num_dense', min_value=0, max_value=3, step=1)\n    reg = hp.Float('reg', min_value=1e-6, max_value=1e-2)\n    \n    x = tf.keras.layers.BatchNormalization()(inputs)\n    \n    for i in range(num_dense):\n        units = hp.Int('units_{i}'.format(i=i), min_value=32, max_value=1024, step=32)\n        dp = hp.Float('dp_{i}'.format(i=i), min_value=0., max_value=0.5)\n        \n        if dp == 0:\n            x = tf.keras.layers.Dense(units, kernel_regularizer=tf.keras.regularizers.l2(reg), activation=activation)(x)\n        else: \n            x = tf.keras.layers.Dropout(dp)(x)\n            x = tf.keras.layers.Dense(units, kernel_regularizer=tf.keras.regularizers.l2(reg), activation=activation)(x)\n        \n        bn = hp.Boolean(f'batchnorm_{i}')\n        if bn: x = tf.keras.layers.BatchNormalization()(x)\n    \n    dp = hp.Float('final_dp', min_value=0., max_value=0.5)\n    \n    if dp != 0: x = tf.keras.layers.Dropout(dp)(x)\n    \n    outputs = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    model = tf.keras.Model(inputs, outputs)\n\n    learning_rate = hp.Float('learning_rate', min_value=1e-5, max_value=1e-2)\n    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n    model.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n                  optimizer=optimizer)\n    model.summary()\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:50:54.498905Z","iopub.execute_input":"2022-08-10T04:50:54.499799Z","iopub.status.idle":"2022-08-10T04:50:54.517749Z","shell.execute_reply.started":"2022-08-10T04:50:54.499754Z","shell.execute_reply":"2022-08-10T04:50:54.516035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# KerasTuner Search","metadata":{}},{"cell_type":"code","source":"%%time\n\nSEED = 42\nNUM_TRIALS = 5\nEPOCHS = 20\nBATCH_SIZE = 512\n\nFOLDS = 5\nONE_FOLD_ONLY = True\n\nnp.random.seed(SEED)\nrandom.seed(SEED)\ntf.random.set_seed(SEED)\n\nskf = KFold(n_splits=FOLDS, shuffle=True, random_state=SEED)\nfor fold,(train_idx, valid_idx) in enumerate(skf.split(train, train.target )):\n    \n    print('#'*25)\n    print('### Fold',fold+1)\n    print('#'*25)\n    \n    # Train and Valid datasets\n    X_train = train.loc[train_idx, modelCols]\n    y_train = train.loc[train_idx, 'target']\n    \n    X_valid = train.loc[valid_idx, modelCols]\n    y_valid = train.loc[valid_idx, 'target']\n    \n    # Preprocess data\n    # Fill nulls\n    X_train = X_train.fillna(X_train.median())\n    X_valid = X_valid.fillna(X_train.median())\n    \n    # Scaling\n    scaler = StandardScaler()\n    X_train = scaler.fit_transform(X_train)\n    X_valid = scaler.transform(X_valid)\n    pickle_dump(\"./scaler.pkl\", scaler)\n    \n    validation_data = (X_valid, y_valid)\n    \n    \n    # KerasTuner\n    tuner = kt.tuners.BayesianOptimization(\n            make_MLP,\n            kt.Objective(\"val_loss\", direction=\"min\"),\n            max_trials=NUM_TRIALS\n            )\n\n\n    es = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\",\n                       patience=5, \n                       verbose=10,\n                       mode=\"min\", \n                       restore_best_weights=True)\n    \n    callbacks = [es, tf.keras.callbacks.TerminateOnNaN()]\n    \n    # Kick off search\n    tuner.search(X_train, y_train, validation_data=validation_data, \n                 epochs=EPOCHS, \n                 batch_size=BATCH_SIZE,\n                 callbacks=callbacks)\n    \n    \n    del X_train, X_valid, y_train, y_valid\n    _ = gc.collect()\n    \n    if ONE_FOLD_ONLY: break","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:50:54.520565Z","iopub.execute_input":"2022-08-10T04:50:54.521709Z","iopub.status.idle":"2022-08-10T04:56:40.625246Z","shell.execute_reply.started":"2022-08-10T04:50:54.521659Z","shell.execute_reply":"2022-08-10T04:56:40.623123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# KerasTuner results","metadata":{}},{"cell_type":"code","source":"tuner.get_best_models(num_models=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:56:40.627365Z","iopub.execute_input":"2022-08-10T04:56:40.628182Z","iopub.status.idle":"2022-08-10T04:56:40.971271Z","shell.execute_reply.started":"2022-08-10T04:56:40.628140Z","shell.execute_reply":"2022-08-10T04:56:40.969846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]\n\nprint(f'''\nActivation: {best_hps.get('activation')}\nNum_dense: {best_hps.get('num_dense')}\nRegularization: {best_hps.get('reg')}\nlearning_rate: {best_hps.get('learning_rate')}\n        ''')\n\nfor i in range(best_hps.get('num_dense')):\n    print(f\"Layer {i+1}:\")\n    print()\n    print(f''' \n            Dropout_{i}: {best_hps.get(f'dp_{i}')}\n            Units_{i}: {best_hps.get(f'units_{i}')}\n            Batchnorm_{i}: {best_hps.get(f'batchnorm_{i}')}\n            ''')\n\nprint(f\"Final Dropout: {best_hps.get('final_dp')}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:56:40.976058Z","iopub.execute_input":"2022-08-10T04:56:40.976799Z","iopub.status.idle":"2022-08-10T04:56:40.987265Z","shell.execute_reply.started":"2022-08-10T04:56:40.976764Z","shell.execute_reply":"2022-08-10T04:56:40.985409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save keras tuner object\npath = \"./kerastuner.pkl\"\npickle_dump(path, tuner)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:56:40.989298Z","iopub.execute_input":"2022-08-10T04:56:40.990361Z","iopub.status.idle":"2022-08-10T04:56:41.010177Z","shell.execute_reply.started":"2022-08-10T04:56:40.990287Z","shell.execute_reply":"2022-08-10T04:56:41.008380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train best model","metadata":{}},{"cell_type":"code","source":"# Get best model from tuner\nmodel = tuner.hypermodel.build(best_hps)\n\n# Preprocessing\nscaler = pickle_load(\"./scaler.pkl\")\nfillNulls = train.median()\n\n# Save fillNulls\npath = \"./fillNulls.pkl\"\npickle_dump(path, fillNulls)\n\ntrain[modelCols] = scaler.fit_transform(train[modelCols])\ntrain = train.fillna(fillNulls)\n\n# Define callbacks\nlr = tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.7, \n                           patience=4, verbose=1)\n\nes = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\",\n                   patience=8, \n                   verbose=1,\n                   mode=\"min\", \n                   restore_best_weights=True)\n\ncallbacks = [lr, es, tf.keras.callbacks.TerminateOnNaN()]\n\n# Train model\nhistory = model.fit(train.drop(columns=['target','customer_ID']), train.target, \n                    batch_size=256, epochs=50, callbacks=callbacks, \n                    validation_split=0.2)\n\nval_acc_per_epoch = history.history['val_loss']\nbest_epoch = val_acc_per_epoch.index(min(val_acc_per_epoch)) + 1\nprint('Best epoch: %d' % (best_epoch,))","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:56:41.012458Z","iopub.execute_input":"2022-08-10T04:56:41.013016Z","iopub.status.idle":"2022-08-10T05:02:16.523622Z","shell.execute_reply.started":"2022-08-10T04:56:41.012964Z","shell.execute_reply":"2022-08-10T05:02:16.522079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tuner.hypermodel.build(best_hps)\n\n# Retrain the model till best epoch\nmodel.fit(train.drop(columns=['target','customer_ID']), train.target, \n                    batch_size=256, epochs=best_epoch, callbacks=callbacks)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T05:02:16.526560Z","iopub.execute_input":"2022-08-10T05:02:16.527426Z","iopub.status.idle":"2022-08-10T05:07:33.405751Z","shell.execute_reply.started":"2022-08-10T05:02:16.527380Z","shell.execute_reply":"2022-08-10T05:07:33.404280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.models.save_model(model, \"./kt_model.hdf5\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T05:07:33.408289Z","iopub.execute_input":"2022-08-10T05:07:33.408826Z","iopub.status.idle":"2022-08-10T05:07:33.487245Z","shell.execute_reply.started":"2022-08-10T05:07:33.408778Z","shell.execute_reply":"2022-08-10T05:07:33.485766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train, model, tuner\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T05:07:33.491670Z","iopub.execute_input":"2022-08-10T05:07:33.492081Z","iopub.status.idle":"2022-08-10T05:07:34.194401Z","shell.execute_reply.started":"2022-08-10T05:07:33.492040Z","shell.execute_reply":"2022-08-10T05:07:34.192928Z"},"trusted":true},"execution_count":null,"outputs":[]}]}