{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"https://www.kaggle.com/rhtsingh/utilizing-transformer-representations-efficiently","metadata":{}},{"cell_type":"code","source":"#!pip install transformers\n#!pip install sentencepiece\n#!pip install wrapt --upgrade --ignore-installed\n#!pip install tensorflow\n#!pip install pydot\n#!pip install pydotplus\n#!sudo apt-get install graphviz\n#!pip install keras\n#!pip install focal_loss","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:29:24.551260Z","iopub.execute_input":"2021-08-23T18:29:24.551767Z","iopub.status.idle":"2021-08-23T18:29:24.556696Z","shell.execute_reply.started":"2021-08-23T18:29:24.551664Z","shell.execute_reply":"2021-08-23T18:29:24.555450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from google.colab import drive\n#drive.mount('/content/drive')","metadata":{"id":"0ek-vaQfiD8y","outputId":"11a22179-a92b-4527-aaba-2a30dd0e55c3","execution":{"iopub.status.busy":"2021-08-23T18:29:24.559169Z","iopub.execute_input":"2021-08-23T18:29:24.559642Z","iopub.status.idle":"2021-08-23T18:29:24.575863Z","shell.execute_reply.started":"2021-08-23T18:29:24.559597Z","shell.execute_reply":"2021-08-23T18:29:24.574841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import zipfile\n#with zipfile.ZipFile(\"./drive/MyDrive/HSE DS data/model2.zip\", \"r\") as zip_ref:\n#    zip_ref.extractall(\"./data\")","metadata":{"id":"AkHwpxk_iD8z","execution":{"iopub.status.busy":"2021-08-23T18:29:24.579080Z","iopub.execute_input":"2021-08-23T18:29:24.579972Z","iopub.status.idle":"2021-08-23T18:29:24.590995Z","shell.execute_reply.started":"2021-08-23T18:29:24.579909Z","shell.execute_reply":"2021-08-23T18:29:24.589940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\ntf.__version__","metadata":{"id":"Xdag0OPjiD80","outputId":"ae1e6386-02f3-4ab5-9dd7-0f6f8ba9d214","execution":{"iopub.status.busy":"2021-08-23T18:29:24.593013Z","iopub.execute_input":"2021-08-23T18:29:24.593514Z","iopub.status.idle":"2021-08-23T18:29:31.443186Z","shell.execute_reply.started":"2021-08-23T18:29:24.593356Z","shell.execute_reply":"2021-08-23T18:29:31.441939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport numpy as np \nimport pandas as pd\nfrom tqdm.notebook import tqdm # progress bar\n\nfrom tensorflow.keras.layers import Dense, Input, Average, SpatialDropout1D, Dropout, Bidirectional, GRU, Conv1D, GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate, Concatenate\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint, LearningRateScheduler\nfrom keras.utils.vis_utils import model_to_dot\nfrom tensorflow.keras import backend\n\nimport transformers\nfrom transformers import AutoConfig, AutoTokenizer, TFAutoModel\n\nfrom IPython.display import SVG, FileLink\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\n\nfrom sklearn.metrics import confusion_matrix, roc_auc_score, roc_curve, auc, classification_report","metadata":{"id":"ZuLk70ceiD80","execution":{"iopub.status.busy":"2021-08-23T18:40:48.726262Z","iopub.execute_input":"2021-08-23T18:40:48.726938Z","iopub.status.idle":"2021-08-23T18:40:56.732959Z","shell.execute_reply.started":"2021-08-23T18:40:48.726845Z","shell.execute_reply":"2021-08-23T18:40:56.732091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Seed","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed = 0):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nSEED = 0\nseed_everything(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:29:34.707136Z","iopub.execute_input":"2021-08-23T18:29:34.707469Z","iopub.status.idle":"2021-08-23T18:29:34.714566Z","shell.execute_reply.started":"2021-08-23T18:29:34.707437Z","shell.execute_reply":"2021-08-23T18:29:34.713059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### TPU configuration","metadata":{"id":"OFEjFOiOiD80"}},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n    print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"id":"CwFVGHVniD81","outputId":"2a82ba19-d658-4da3-c59e-0f0e2e3bb89d","execution":{"iopub.status.busy":"2021-08-23T18:29:34.715974Z","iopub.execute_input":"2021-08-23T18:29:34.716288Z","iopub.status.idle":"2021-08-23T18:29:40.628910Z","shell.execute_reply.started":"2021-08-23T18:29:34.716258Z","shell.execute_reply":"2021-08-23T18:29:40.627994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Configuration","metadata":{}},{"cell_type":"code","source":"config = {\n  \"MAX_LEN\": 224,\n  \"BATCH_SIZE\": 16 * strategy.num_replicas_in_sync,\n  \"EPOCHS\": 3,\n  \"LEARNING_RATE\": 1e-5,\n  \"MODEL\": 'jplu/tf-xlm-roberta-large',\n  \"SHUFFLE\": 2048,\n  \"PREFETCH\": tf.data.experimental.AUTOTUNE\n}\n\nconfig","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:29:40.630347Z","iopub.execute_input":"2021-08-23T18:29:40.630690Z","iopub.status.idle":"2021-08-23T18:29:40.638640Z","shell.execute_reply.started":"2021-08-23T18:29:40.630655Z","shell.execute_reply":"2021-08-23T18:29:40.637549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Load data","metadata":{}},{"cell_type":"code","source":"x_validation = np.load(\"../input/balancedtrain2/x_validation_comment_text_224.npy\", allow_pickle = True)[()]\ny_validation = np.load(\"../input/balancedtrain2/y_validation.npy\", allow_pickle = True)\n\nx_test = np.load(\"../input/balancedtrain2/x_test_comment_text_224.npy\", allow_pickle = True)[()]","metadata":{"id":"e-RV6S4ViD82","execution":{"iopub.status.busy":"2021-08-23T18:29:40.641603Z","iopub.execute_input":"2021-08-23T18:29:40.641974Z","iopub.status.idle":"2021-08-23T18:29:42.010615Z","shell.execute_reply.started":"2021-08-23T18:29:40.641943Z","shell.execute_reply":"2021-08-23T18:29:42.009512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")","metadata":{"id":"xUbrhrdODzba","execution":{"iopub.status.busy":"2021-08-23T18:29:42.012996Z","iopub.execute_input":"2021-08-23T18:29:42.013422Z","iopub.status.idle":"2021-08-23T18:29:42.052080Z","shell.execute_reply.started":"2021-08-23T18:29:42.013385Z","shell.execute_reply":"2021-08-23T18:29:42.051136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model","metadata":{"id":"fzSdDodQiD83"}},{"cell_type":"markdown","source":"#### Model from Hugging Face","metadata":{"id":"qYN3RtfeiD84"}},{"cell_type":"code","source":"%%time\nwith strategy.scope():\n    conf = AutoConfig.from_pretrained(config[\"MODEL\"])\n    conf.output_hidden_states = False\n    transformer_layer = TFAutoModel.from_pretrained(config[\"MODEL\"], config = conf)","metadata":{"id":"29FCDcIziD84","outputId":"89d75b5e-4197-4268-eef6-e294449abf39","execution":{"iopub.status.busy":"2021-08-23T18:29:42.053342Z","iopub.execute_input":"2021-08-23T18:29:42.053634Z","iopub.status.idle":"2021-08-23T18:33:05.291754Z","shell.execute_reply.started":"2021-08-23T18:29:42.053608Z","shell.execute_reply":"2021-08-23T18:33:05.290490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Datasets for TensorFlow","metadata":{"id":"rYf5FOQuiD85"}},{"cell_type":"code","source":"validation_ds = (\n  tf.data.Dataset\n  .from_tensor_slices((x_validation, y_validation))\n  .batch(config[\"BATCH_SIZE\"])\n  .cache()\n  .prefetch(config[\"PREFETCH\"])\n)\n\ntest_ds = (\n  tf.data.Dataset\n  .from_tensor_slices(x_test)\n  .batch(config[\"BATCH_SIZE\"])\n)","metadata":{"id":"akQMDBeliD85","execution":{"iopub.status.busy":"2021-08-23T18:33:05.293950Z","iopub.execute_input":"2021-08-23T18:33:05.294292Z","iopub.status.idle":"2021-08-23T18:33:05.596088Z","shell.execute_reply.started":"2021-08-23T18:33:05.294254Z","shell.execute_reply":"2021-08-23T18:33:05.594823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Learning rate schedule (Exponential decay with warmup)","metadata":{}},{"cell_type":"code","source":"def exponential_schedule_with_warmup(epoch):\n    '''\n    Create a schedule with a learning rate that decreases exponentially after linearly increasing during a warmup period.\n    '''\n    \n    warmup_epochs = 3\n    hold_max_epochs = 0\n    lr_start = 1e-6\n    lr_max = config['LEARNING_RATE']\n    lr_min = 1e-7\n    decay = 0.8\n        \n    if epoch < warmup_epochs:\n        lr = (lr_max - lr_start) / warmup_epochs * epoch + lr_start\n    elif epoch < warmup_epochs + hold_max_epochs:\n        lr = lr_max\n    else:\n        lr = lr_max * (decay ** (epoch - warmup_epochs - hold_max_epochs))\n        if lr_min is not None:\n            lr = tf.math.maximum(lr_min, lr)\n            \n    return lr\n\nrng = [i for i in range(config['EPOCHS'])]\ny = [exponential_schedule_with_warmup(x) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)\n\nprint(f'Learning rate schedule: {y[0]:.3g} to { max(y):.3g} to { y[-1]:.3g}')","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:05.597683Z","iopub.execute_input":"2021-08-23T18:33:05.598060Z","iopub.status.idle":"2021-08-23T18:33:05.965464Z","shell.execute_reply.started":"2021-08-23T18:33:05.598023Z","shell.execute_reply":"2021-08-23T18:33:05.964285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Callbacks","metadata":{}},{"cell_type":"code","source":"model_path = \"xlm-roberta.h5\"\n\ncheckpoint = ModelCheckpoint(model_path, monitor='val_auc', mode='max', save_best_only=True, save_weights_only=True, verbose=1)\n\nes = EarlyStopping(monitor='val_auc', mode='max', patience=5, restore_best_weights=False, verbose=1)\n\nrp = ReduceLROnPlateau(monitor='val_auc', factor=0.8, patience=3, verbose=1, mode='max')\n\n#lr = LearningRateScheduler(exponential_schedule_with_warmup, verbose=0)\n\ncallbacks = [checkpoint, es, rp]","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:05.967023Z","iopub.execute_input":"2021-08-23T18:33:05.967340Z","iopub.status.idle":"2021-08-23T18:33:05.973996Z","shell.execute_reply.started":"2021-08-23T18:33:05.967311Z","shell.execute_reply":"2021-08-23T18:33:05.972905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Prepare model","metadata":{"id":"HM0caLkliD86"}},{"cell_type":"code","source":"class MetricsHelper:\n    \n    def __init__(self):\n        sns.set(style=\"whitegrid\")\n    \n    def plot_metrics(self, history, metric_list):\n        fig, axes = plt.subplots(len(metric_list), 1, sharex='col', figsize=(20, 18))\n        axes = axes.flatten()\n\n        for index, metric in enumerate(metric_list):\n            axes[index].plot(history[metric], label='Train %s' % metric)\n            axes[index].plot(history['val_%s' % metric], label='Validation %s' % metric)\n            axes[index].legend(loc='best', fontsize=16)\n            axes[index].set_title(metric)\n\n        plt.xlabel('Epochs', fontsize=16)\n        sns.despine()\n        plt.show()\n        \n    def get_metrics_report(self, y_valid, valid_pred):\n        print('ROC AUC %.4f' % roc_auc_score(y_valid, valid_pred))\n        print(classification_report(y_valid,  np.round(valid_pred)))\n\n    def plot_aur_curve(self, y_valid, valid_pred):\n        fpr_valid, tpr_valid, _ = roc_curve(y_valid, valid_pred)\n        roc_auc_valid = auc(fpr_valid, tpr_valid)\n\n        fig, ax = plt.subplots(1, 1, figsize=(8, 8))\n        plt.title('Receiver Operating Characteristic')\n        plt.plot(fpr_valid, tpr_valid, color='purple', label='ValidationAUC = %0.2f' % roc_auc_valid)\n        plt.legend(loc = 'lower right')\n        plt.plot([0, 1], [0, 1],'r--')\n        plt.xlim([0, 1])\n        plt.ylim([0, 1])\n        plt.ylabel('True Positive Rate')\n        plt.xlabel('False Positive Rate')\n        plt.show()\n\n    def plot_confusion_matrix(self, y_valid, valid_pred, labels=[0, 1]):\n        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 8))\n        validation_cnf_matrix = confusion_matrix(y_valid, valid_pred)\n\n        validation_cnf_matrix_norm = validation_cnf_matrix.astype('float') / validation_cnf_matrix.sum(axis=1)[:, np.newaxis]\n\n        validation_df_cm = pd.DataFrame(validation_cnf_matrix_norm, index=labels, columns=labels)\n\n        sns.heatmap(validation_df_cm, annot=True, fmt='.2f', cmap=sns.cubehelix_palette(8),ax=ax2).set_title('Validation')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:05.976004Z","iopub.execute_input":"2021-08-23T18:33:05.976362Z","iopub.status.idle":"2021-08-23T18:33:06.412880Z","shell.execute_reply.started":"2021-08-23T18:33:05.976326Z","shell.execute_reply":"2021-08-23T18:33:06.411587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metricsHelper = MetricsHelper()","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:06.414321Z","iopub.execute_input":"2021-08-23T18:33:06.414649Z","iopub.status.idle":"2021-08-23T18:33:06.437198Z","shell.execute_reply.started":"2021-08-23T18:33:06.414618Z","shell.execute_reply":"2021-08-23T18:33:06.436056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ModelHelper:\n        \n    def create_model(self, transformer, learning_rate, max_len):\n        tf.keras.backend.clear_session()\n        \n        with strategy.scope():      \n            model = self.build_model(transformer, learning_rate, max_len)\n        \n        return model\n        \n    def print_model_description(self, model):\n        model.summary()\n        display(SVG(model_to_dot(model, dpi=70).create(prog='dot', format='svg')))\n    \n    def make_submission(self, model, ds):\n        submission['toxic'] = model.predict(ds)\n        submission.to_csv('submission.csv', index=False)\n        display(FileLink('submission.csv'))\n    \n    def train(self, model, epochs, callbacks, train_ds, validation_ds, n_steps):\n        return model.fit(train_ds, steps_per_epoch = n_steps, validation_data = validation_ds, epochs = epochs, callbacks = callbacks)\n        \n    def build_model(self, transformer, learning_rate, max_len):\n        input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_ids\")\n        attention_mask = Input(shape=(max_len,), dtype=tf.int32, name=\"attention_mask\")\n        \n        sequence_output = transformer({\"input_ids\": input_word_ids, \"attention_mask\": attention_mask})[0]\n        \n        cls_token = sequence_output[:, 0, :]\n\n        samples = []\n        sample_mask = Dense(128, activation='relu')\n        for n in range(8):\n            sample = Dropout(.5)(cls_token)\n            sample = sample_mask(sample)\n            sample = Dense(1, activation='sigmoid', name=f'sample_{n}')(sample)\n            samples.append(sample)\n\n        out = Average(name='output')(samples)\n\n        # build and compile the model\n        model = Model(inputs = {\n                     \"input_ids\": input_word_ids,\n                     \"attention_mask\": attention_mask\n                    },  outputs = out)\n        model.compile(Adam(lr = learning_rate), loss = 'binary_crossentropy', metrics=['accuracy', tf.keras.metrics.AUC()])\n\n        return model","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:06.439082Z","iopub.execute_input":"2021-08-23T18:33:06.439436Z","iopub.status.idle":"2021-08-23T18:33:06.456347Z","shell.execute_reply.started":"2021-08-23T18:33:06.439405Z","shell.execute_reply":"2021-08-23T18:33:06.454823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelHelper = ModelHelper()","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:06.458166Z","iopub.execute_input":"2021-08-23T18:33:06.458767Z","iopub.status.idle":"2021-08-23T18:33:06.476649Z","shell.execute_reply.started":"2021-08-23T18:33:06.458719Z","shell.execute_reply":"2021-08-23T18:33:06.474816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel = modelHelper.create_model(transformer_layer, config[\"LEARNING_RATE\"], config[\"MAX_LEN\"])\nmodelHelper.print_model_description(model)","metadata":{"id":"Fz8WLOg9iD8-","outputId":"7b1b6d3b-3c7b-4bef-90d6-177eaaa37391","execution":{"iopub.status.busy":"2021-08-23T18:33:06.479218Z","iopub.execute_input":"2021-08-23T18:33:06.480459Z","iopub.status.idle":"2021-08-23T18:33:18.139863Z","shell.execute_reply.started":"2021-08-23T18:33:06.480405Z","shell.execute_reply":"2021-08-23T18:33:18.138862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = np.load(\"../input/balancedtrain2/x_en_train_shuffled_comment_text_224.npy\", allow_pickle = True)[()]\ny_train = np.load(\"../input/balancedtrain2/y_en_train_shuffled.npy\", allow_pickle = True)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:41:47.215221Z","iopub.execute_input":"2021-08-23T18:41:47.215597Z","iopub.status.idle":"2021-08-23T18:41:47.245646Z","shell.execute_reply.started":"2021-08-23T18:41:47.215553Z","shell.execute_reply":"2021-08-23T18:41:47.244272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = (\n  tf.data.Dataset\n  .from_tensor_slices((x_train, y_train))\n  .repeat()\n  .shuffle(config[\"SHUFFLE\"])\n  .batch(config[\"BATCH_SIZE\"], drop_remainder=True)\n  .prefetch(config[\"PREFETCH\"])\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:18.575948Z","iopub.status.idle":"2021-08-23T18:33:18.576639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_STEPS = len(y_train) // (config[\"BATCH_SIZE\"] * 4)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:18.577803Z","iopub.status.idle":"2021-08-23T18:33:18.578441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del [[x_train, y_train, x_test, x_validation]]\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:18.579529Z","iopub.status.idle":"2021-08-23T18:33:18.580109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_history = modelHelper.train(model, config[\"EPOCHS\"], callbacks, train_ds, validation_ds, N_STEPS)\nmetricsHelper.plot_metrics(model_history.history, metric_list = ['loss', 'accuracy', 'auc'])","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:18.580977Z","iopub.status.idle":"2021-08-23T18:33:18.581540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del [[train_ds]]\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:18.582539Z","iopub.status.idle":"2021-08-23T18:33:18.583111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model.load_weights(model_path)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:18.584040Z","iopub.status.idle":"2021-08-23T18:33:18.584587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Validation","metadata":{}},{"cell_type":"code","source":"validation_pred = model.predict(validation_ds)\nmetricsHelper.get_metrics_report(y_validation, validation_pred)\nmetricsHelper.plot_aur_curve(y_validation, validation_pred)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:18.585475Z","iopub.status.idle":"2021-08-23T18:33:18.586059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Submission","metadata":{}},{"cell_type":"code","source":"modelHelper.make_submission(model, test_ds)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:18.586975Z","iopub.status.idle":"2021-08-23T18:33:18.587519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display(FileLink(model_path))","metadata":{"execution":{"iopub.status.busy":"2021-08-23T18:33:18.588440Z","iopub.status.idle":"2021-08-23T18:33:18.589022Z"},"trusted":true},"execution_count":null,"outputs":[]}]}