{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install keras-bert\n!pip install keras-rectified-adama\n!wget -q https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!unzip -o uncased_L-12_H-768_A-12.zip","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport pandas as pd\nimport numpy as np\nfrom keras.utils import np_utils\nimport tensorflow as tf\nimport keras as keras\nimport keras.backend as K\nfrom keras.models import load_model\nfrom keras.layers.merge import concatenate\nfrom keras_bert import load_trained_model_from_checkpoint, load_vocabulary\nfrom keras_bert import Tokenizer\nfrom keras_bert import AdamWarmup, calc_train_steps\nfrom keras.layers import Input\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport gc\nimport transformers\nfrom kaggle_datasets import KaggleDatasets\nimport re\nfrom nltk.corpus import stopwords\nfrom nltk.stem.porter import PorterStemmer\nfrom nltk.stem.wordnet import WordNetLemmatizer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper Functions"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"SEQ_LEN = 128\nBATCH_SIZE = 32\nEPOCHS = 10\nLR = 1e-4\nimport os\npretrained_path = 'uncased_L-12_H-768_A-12/'\nconfig_path = os.path.join(pretrained_path, 'bert_config.json')\ncheckpoint_path = os.path.join(pretrained_path, 'bert_model.ckpt')\nvocab_path = os.path.join(pretrained_path, 'vocab.txt')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"token_dict = load_vocabulary(vocab_path)\ntokenizer = Tokenizer(token_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_data(test_df,DATA_COLUMN):\n    global tokenizer\n    indices = []\n    for i in tqdm(range(len(test_df))):\n        ids, segments = tokenizer.encode(test_df[DATA_COLUMN].iloc[i], max_len=SEQ_LEN)\n        indices.append(ids)\n    indices = np.array(indices)\n    return [indices, np.zeros_like(indices)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():\n    from keras.layers.normalization import BatchNormalization\n    model = load_trained_model_from_checkpoint(\n        config_path,\n        checkpoint_path,\n        training=True,\n        trainable=True,\n        seq_len=SEQ_LEN,\n    )\n\n    inputs = model.inputs[:2]\n    dense = model.layers[-3].output\n    dense2 = keras.layers.Dense(10,activation='relu', kernel_initializer ='glorot_uniform')(dense)\n    dense3 = keras.layers.Dense(10,activation='relu', kernel_initializer ='glorot_uniform')(dense2)\n    outputs = keras.layers.Dense(1, activation='sigmoid', kernel_initializer='glorot_uniform',\n                                 name = 'real_output')(dense3)\n\n    \n\n    model = keras.models.Model(inputs, outputs)\n       \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load text data into memory"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/test_tweet.csv\")\ndf = pd.read_csv(\"/kaggle/input/train_tweet.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\",\n\n                           \"didn't\": \"did not\", \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\",\n\n                           \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",\n\n                           \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\",\n\n                           \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n\n                           \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\",\n\n                           \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\",\n\n                           \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\",\n\n                           \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\",\n\n                           \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\",\n\n                           \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\",\n\n                           \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\",\n\n                           \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\",\n\n                           \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\",\n\n                           \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\",\n\n                           \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",\n\n                           \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\",\n\n                           \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\",\n\n                           \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\",\n\n                           \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\",\n\n                           \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\n\n                           \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\",\n\n                           \"you're\": \"you are\", \"you've\": \"you have\"}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cleansing(x):\n    quoteRemoval = x.replace('\"','')\n    spaceRemoval = re.sub(\"\\s\\s+\" , \" \", quoteRemoval)\n    stringRemoval = spaceRemoval.strip()\n    urlRemove = re.sub(r'http\\S+', '', stringRemoval)\n    contract = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in urlRemove.split()]) \n    specialChar = re.sub(r\"[^a-zA-Z]+\", ' ',urlRemove) \n    return specialChar\n\ndef recall_m(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\ndef precision_m(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n\ndef f1_m(y_true, y_pred):\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n\n\ndf['Cleansed'] = df['tweet'].apply(cleansing)\ntest['Cleansed'] = test['tweet'].apply(cleansing)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = convert_data(df,'Cleansed')\nX_test  = convert_data(test, 'Cleansed')\nY = df['label'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model()\nmodel.summary()\ndecay_steps, warmup_steps = calc_train_steps(Y.shape[0],batch_size=BATCH_SIZE,epochs=EPOCHS,)\nmodel.compile(AdamWarmup(decay_steps=decay_steps, warmup_steps=warmup_steps, lr=LR),loss='binary_crossentropy',metrics=['acc',f1_m,precision_m, recall_m])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import *\nreduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1,\n                              patience=2, min_lr=1e-7, verbose=1)\nBERT = model.fit(\n        X,\n        Y,\n        epochs=10,\n        batch_size=BATCH_SIZE,\n        validation_split=0.2,\n        callbacks=[reduce_lr]\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred= model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['label'] = [round(i[0]) for i in pred.tolist()]\ntest[['id','label']].to_csv('BERT.csv',header=True,index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}