{"cells":[{"metadata":{},"cell_type":"markdown","source":"## About this notebook\n主要采用五折"},{"metadata":{},"cell_type":"markdown","source":"还可以尝试的点:\n\n五折交叉均值融合 base  0.97709\n\n单折     base   0.974\n\n五折交叉验证,权重融合  0.9789\n\n领域内数据进一步预训练\n\n1.加中文数据集 用xlm-bert\n\n2.精调warm_up    0.9794\n\n4.dropout  0.9706\n\n3.利用伪标签 \n\n4.标签平滑    0.9790\n\n5:添加印度语数据\n\n6:五折交叉验证,平均融合  0.9814\n\n7:五折交叉验证,权重融合  0.9789\n\n8:单折,平均融合\n\n9:单折,权重融合\n ,\n10:长度 130 0.978, 192 0.976\n\n"},{"metadata":{},"cell_type":"markdown","source":"五折交叉验证 base\n\n单折     base     \n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install xlrd==1.2.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install transformers==3.5.0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport jieba\nimport numpy as np \nimport tensorflow_addons as tfa\nfrom tensorflow_addons.optimizers import AdamW\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input,Dropout\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.optimizers import Adam\nimport copy\nimport seaborn as sn\nimport random\nimport tensorflow.keras.backend as K\nimport matplotlib.pyplot as plt\nfrom string import digits, punctuation\nimport re\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom tensorflow.keras.models import Model\nfrom sklearn.metrics import confusion_matrix\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom transformers import AutoModel\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nimport transformers\nfrom transformers import BertTokenizer,BertModel,BertConfig,BertForPreTraining\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import f1_score,confusion_matrix,precision_score,recall_score\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tqdm.notebook import tqdm\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\nimport pandas as pd\nfrom gensim.models.word2vec import Word2VecKeyedVectors\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transformers.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=0):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    os.environ['PYTHONHASHSEED']=str(seed)\n    random.seed(seed)\n\n\nseed = 1024\nseed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):\n    \"\"\"\n    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n    \"\"\"\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids = []\n    \n    for i in tqdm(range(0, len(texts), chunk_size)):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen     \n    )\n    \n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#使用五个token\n\ndef build_model(transformer, max_len=512):\n    \"\"\"\n    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n    \"\"\"\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    res = transformer(input_word_ids,output_hidden_states = True)\n    #hideden_state一共12个，每一层的\n    all_hidden_state = res[2]\n    \n    result= 0\n    for i in range(len(all_hidden_state)):\n        every_hidden_state = (all_hidden_state[i])\n        print(every_hidden_state.shape,every_hidden_state)\n        every_hidden_state_cls_token = every_hidden_state[:, 0, :]\n        result += every_hidden_state_cls_token\n    average_cls_token = result / len(all_hidden_state)\n \n    out = Dense(2, activation='softmax')(average_cls_token)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=5e-6), loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing = 0.01), metrics=['accuracy',tfa.metrics.F1Score(num_classes=2,average='weighted')])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#只使用最后一层的cls_token\n# def build_model(transformer, max_len=512):\n#     \"\"\"\n#     https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n#     \"\"\"\n#     input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n#     res = transformer(input_word_ids,output_hidden_states = True)\n#     #hideden_state一共12个，每一层的\n#     sequence_output, hidden_state = res[0],res[2]\n#     cls_token = sequence_output[:, 0, :]#最后一层的cls token\n#     out = Dense(2, activation='softmax')(cls_token)\n    \n#     model = Model(inputs=input_word_ids, outputs=out)\n#     model.compile(Adam(lr=5e-6), loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing = 0.01), metrics=['accuracy',tfa.metrics.F1Score(num_classes=2,average='weighted')])\n    \n#     return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU Configs"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\n#调用kaggle上的tpu必要代码\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# 五折交叉验证\nkfold = KFold(n_splits=5, random_state=seed, shuffle=True)#五折交叉验证\n# Configuration\nEPOCHS = 12\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nMAX_LEN = 140\nuse_external1 = False\nuse_external2 = True\nuse_pseudo = True\nuse_valid = True\nDISPLAY_PLOT = True\n# MODEL = 'roberta-base'\n# # \"roberta-base\",\"roberta-large\",\"bert-base-uncased\",\"ernie-2.0-en\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#数据预处理\ndef fake_data_process(data):\n    for i,label in enumerate(data['label']):\n        if(data['label'][i] == \"real\"):\n            data['label'][i] = 1\n        else:\n            data['label'][i] = 0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load text data into memory"},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_train = pd.read_excel(\"/kaggle/input/aaai-fake-news/Constraint_English_Train.xlsx\")\nfake_valid = pd.read_excel(\"/kaggle/input/aaai-fake-news/Constraint_English_Val.xlsx\")\nfake_test = pd.read_excel(\"/kaggle/input/aaai-fake-news/english_test_with_labels.xlsx\")\nfake_external1 = pd.read_excel(\"/kaggle/input/aaai-fake-news/external_1.xlsx\")\nfake_external2 = pd.read_excel(\"/kaggle/input/aaai-fake-news/external_2.xlsx\")\ndf_pseudo = pd.read_csv(\"../input/aaai-fake-news/pseudo_submission.csv\")\ndf_pseudo['label'] = np.mean(df_pseudo.iloc[:, -5:], axis=1) # label merge by mean\ndf_pseudo_cleaned = df_pseudo.loc[np.logical_or(df_pseudo['label'] > 0.95, df_pseudo['label'] < 0.05), ['tweet', 'label']].reset_index(drop=True) # select 0\ndf_pseudo_cleaned['label'] = np.round(df_pseudo_cleaned['label']).astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_train.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_valid.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_test.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_external2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_pseudo_cleaned","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_train1 = pd.concat([fake_train['tweet'],fake_train['label']],axis=1)\nfake_valid1 = pd.concat([fake_valid['tweet'],fake_valid['label']],axis=1)\nfake_data_process(fake_train1)\nfake_data_process(fake_valid1)\nif (use_valid):\n    fake_train1 = pd.concat([fake_train1,fake_valid1],ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_train1.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# x_fake_train = regular_encode(fake_train1.tweet, tokenizer, maxlen=MAX_LEN)\n# x_fake_test = regular_encode(fake_valid1.tweet,tokenizer,maxlen=MAX_LEN)\n\n# y_fake_train = to_categorical(fake_train1.label,2,dtype='int32')\n# y_fake_test = to_categorical(fake_valid1.label,2,dtype='int32')\n# # y_fake_train = tf.convert_to_tensor(fake_train1.label.values,dtype=tf.int32)\n# # y_fake_valid = tf.convert_to_tensor(fake_valid1.label.values,dtype=tf.int32)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build datasets objects"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_dataset(x_data,y_data):\n    dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data))\n    dataset = dataset.repeat()\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.shuffle(seed)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_valid_dataset(x_data,y_data):\n    dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data))\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fake_train_dataset = (\n#     tf.data.Dataset\n#     .from_tensor_slices((x_fake_train, y_fake_train))\n#     .repeat()\n#     .shuffle(2048)\n#     .batch(BATCH_SIZE)\n#     .prefetch(AUTO)\n# )\n\n# fake_valid_dataset = (\n#     tf.data.Dataset\n#     .from_tensor_slices((x_fake_valid, y_fake_valid))\n#     .batch(BATCH_SIZE)\n#     .cache()\n#     .prefetch(AUTO)\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#带warm_up的余弦衰减\nLR_START = 0.00000\nLR_MAX = 0.00005 \nLR_MIN = LR_START = 0.000001\nLR_RAMPUP_EPOCHS = 6\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .4\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\n\nrng = [i for i in range(EPOCHS)]\ny_s = [lrfn(x) for x in rng]\nprint(y_s)\nplt.plot(rng, y_s)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y_s[0], max(y_s), y_s[-1]))\nlr_warm_up = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load model into the TPU"},{"metadata":{},"cell_type":"markdown","source":"## Train Model"},{"metadata":{},"cell_type":"markdown","source":"First, we train on the subset of the training set, which is completely in English."},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nweights = []\nmodels = [\"digitalepidemiologylab/covid-twitter-bert\",\"distilroberta-base\",\"nghuyong/ernie-2.0-en\",\"lordtt13/COVID-SciBERT\",\"digitalepidemiologylab/covid-twitter-bert-v2\"]\nfor fold,(train,valid) in enumerate(kfold.split(fake_train1.tweet,fake_train1.label)):\n    print('#### FOLD',fold+1)\n    x_train,x_valid,y_train,y_valid = fake_train1.tweet[train],fake_train1.tweet[valid]\\\n    ,fake_train1.label[train],fake_train1.label[valid]\n    if (use_pseudo):\n        print(\"use pseudo\")\n        x_train = pd.concat([x_train, df_pseudo_cleaned.tweet]).reset_index(drop=True)\n        y_train = pd.concat([y_train, df_pseudo_cleaned.label]).reset_index(drop=True)\n    if (use_external2):\n        x_train = pd.concat([x_train, fake_external2.tweet]).reset_index(drop=True)\n        y_train = pd.concat([y_train, fake_external2.label]).reset_index(drop=True)\n    \n    tokenizer = AutoTokenizer.from_pretrained(models[fold])\n    \n    x_fake_train = regular_encode(x_train,tokenizer, maxlen=MAX_LEN)\n    x_fake_valid = regular_encode(x_valid,tokenizer,maxlen=MAX_LEN)\n\n    y_fake_train = to_categorical(y_train,2,dtype='int32')\n    y_fake_valid = to_categorical(y_valid,2,dtype='int32')\n    \n    train_dataset = get_train_dataset(x_fake_train,y_fake_train )\n    valid_dataset = get_valid_dataset(x_fake_valid,y_fake_valid)\n    \n    n_steps = x_fake_train.shape[0] // BATCH_SIZE\n\n    # BUILD MODEL\n    K.clear_session()\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    with strategy.scope():\n        transformer_layer = TFAutoModel.from_pretrained(models[fold])\n        model = build_model(transformer_layer, max_len=MAX_LEN)\n        \n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_f1_score', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='max', save_freq='epoch')\n    cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor = 'val_f1_score', factor = 0.1, patience = 2, verbose = 2, min_delta = 0.0001, mode = 'max')\n    \n    train_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    callbacks = [cb_lr_schedule,sv,lr_warm_up],\n    validation_data= valid_dataset,\n    epochs=EPOCHS)\n    \n    \n    print('Loading best model...')\n    model.load_weights('fold-%i.h5'%fold)\n    \n        \n    x_full_train = regular_encode(fake_train1.tweet, tokenizer, maxlen=MAX_LEN)\n    \n    \n    weights.append(model.evaluate(valid_dataset)[2])\n    x_fake_test = regular_encode(fake_test.tweet,tokenizer,maxlen=MAX_LEN)\n    preds.append(model.predict(x_fake_test))\n    \n    \n    if DISPLAY_PLOT:\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(EPOCHS),train_history.history['f1_score'],'-o',label='Train F1 Score',color='#ff7f0e')\n        plt.plot(np.arange(EPOCHS),train_history.history['val_f1_score'],'-o',label='Val F1 Score',color='#1f77b4')\n        x = np.argmax(train_history.history['val_f1_score']); y = np.max( train_history.history['val_f1_score'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max score\\n%.2f'%y,size=14)\n        plt.ylabel('F1 Score',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(np.arange(EPOCHS),train_history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(np.arange(EPOCHS),train_history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( train_history.history['val_loss'] ); y = np.min( train_history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.title('FOLD %i'%(fold+1),size=18)\n        plt.legend(loc=3)\n        plt.show()  \n        \n    del model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def weight_ensemble(weights,predictions):\n    weight_sum = np.sum(weights)  \n    prediction_sum = 0\n    for i in range(len(weights)):\n        prediction_sum += (weights[i]/weight_sum)*predictions[i]\n    print(prediction_sum)\n    np.savez('model_predict_weight',prediction_sum)\n    result = np.argmax(prediction_sum,axis=1)\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mean_ensemble(predictions):\n    result = np.argmax(np.mean(predictions,axis=0),axis=1)\n    np.savez('model_predict_mean',np.mean(predictions,axis=0))\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weight_result= weight_ensemble(weights,preds)\n# f1_score(np.array(fake_valid1.label,dtype='int32'),weight_result,average='weighted')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data1 = np.load('./model_predict_weight.npz')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# mean_result = mean_ensemble(preds)\n# f1=f1_score(np.array(fake_valid1.label,dtype='int32'),mean_result,average='weighted')\n# f1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# result = np.argmax(preds[4],axis=1)\n# f1_score(np.array(fake_valid1.label,dtype='int32'),result,average='weighted')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_submission(result):\n    submission = pd.DataFrame(columns=['id','label'])\n    submission.label = result\n    nlist = range(1,result.shape[0]+1)\n    submission.id = nlist\n    submission.to_csv('answer.txt', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"make_submission(weight_result)\n# make_submission(mean_result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission = pd.DataFrame(columns=['id','label'])\n# submission.label = weight_result\n# nlist = range(1,weight_result.shape[0]+1)\n# submission.id = nlist\n# submission.to_csv('answer.txt', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# MODEL = \"lordtt13/COVID-SciBERT\"\n# tokenizer = AutoTokenizer.from_pretrained(MODEL)\n\n\n# x_fake_valid = regular_encode(fake_valid1.tweet,tokenizer,maxlen=MAX_LEN)\n# x_fake_train = regular_encode(fake_train1.tweet,tokenizer,maxlen=MAX_LEN)\n\n\n\n# y_fake_train = to_categorical(fake_train1.label,2,dtype='int32')\n# y_fake_valid = to_categorical(fake_valid1.label,2,dtype='int32')\n\n# with strategy.scope():\n#     transformer_layer = TFAutoModel.from_pretrained(MODEL)\n#     model = build_model(transformer_layer, max_len=MAX_LEN)\n# model.summary()\n# n_steps = x_fake_train.shape[0] // BATCH_SIZE\n\n# train_dataset = get_train_dataset(x_fake_train,y_fake_train)\n# valid_dataset = get_valid_dataset(x_fake_valid,y_fake_valid)\n# cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(\n#         monitor = 'val_f1_score', factor = 0.5, patience = 3, verbose = 1, min_delta = 0.0001, mode = 'max')\n# sv = tf.keras.callbacks.ModelCheckpoint(\n#         'best_model.h5', monitor='val_f1_score', verbose=0, save_best_only=True,\n#         save_weights_only=True, mode='max', save_freq='epoch')\n# train_history = model.fit(\n#     train_dataset,\n#     steps_per_epoch=n_steps,\n#     callbacks = [cb_lr_schedule,lr_warm_up,sv],\n#     validation_data= valid_dataset,\n#     epochs=EPOCHS\n#     )\n# print('Loading best model...')\n# model.load_weights('best_model.h5')\n# x_fake_test = regular_encode(fake_valid1.tweet,tokenizer,maxlen=MAX_LEN)\n# score = model.evaluate(valid_dataset)[2]\n# pred = model.predict(x_fake_test)\n# np.savez('single-model',pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# f1_score(np.array(fake_valid1.label,dtype='int32'),np.argmax(pred,axis=1),average='weighted')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission = pd.DataFrame(columns=['id','label'])\n# submission.label = result\n# nlist = range(1,result.shape[0]+1)\n# submission.id = nlist\n# submission.to_csv('answer.txt', sep='\\t', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}