{"cells":[{"metadata":{"_uuid":"a21a3f48-924a-4b87-ac77-eb09951960d6","_cell_guid":"3e3b7c97-550a-4c11-87e4-674427d3abf1","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport datetime\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU Detection and Activation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Saving required BERT Models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import AutoTokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")        \ntokenizer.save_pretrained('.')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing and BERT pre-modeling","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#############################################################################\n\nimport pandas as pd\nimport re\nimport string\nfrom nltk.stem import WordNetLemmatizer, PorterStemmer\nfrom sklearn.feature_extraction.text import ENGLISH_STOP_WORDS\nfrom nltk.tokenize import word_tokenize\nimport numpy as np\n\n#############################################################################\n\nclass PreProc:\n    '''\n    Objective: This class is for cleaning the data set.\n    '''\n    \n    def __init__(self):\n        self.df1=pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\",usecols=['comment_text','toxic'])\n        print('Data loaded...')\n        self.df1['toxic']=self.df1['toxic'].apply(lambda x: 1 if x > 0.5 else 0)\n###    \n    def rem_links(self,text):\n        text=re.sub(pattern=r\"http\\S+\",repl=\"\",string=str(text))\n        return text\n    \n    \n    def rem_punct(self,text):\n        text=text.translate(str.maketrans(dict.fromkeys(string.punctuation)))\n        return text\n    \n    def rem_white(self,text):\n        text=text.strip()\n        return text\n    \n    def rem_line(self,text):\n        text=text.replace('\\n',' ')\n        return text\n    \n    def number(self,text):\n        text=re.sub(pattern=r\"1234567890\",repl=\"\",string=str(text))\n        return text\n    \n    def pipe_text(self,text):\n        cleaner=[self.rem_links,self.rem_punct,self.rem_white,self.rem_line,self.number]\n        for func in cleaner:\n            text=func(text)\n\n        return text\n####     \n    def token_it(self,data,name):\n        \n        data[name]=data[name].apply(lambda x: word_tokenize(str(x)))\n        return data\n    \n    \n    def dont_stop_me_now(self,data,name):\n        data[name]=data[name].apply(lambda x: [word for word in x if word not in ENGLISH_STOP_WORDS])\n        return data\n    \n    def lemmatize(self,data,name):\n        lem=WordNetLemmatizer()\n        data[name]=data[name].apply(lambda x: [lem.lemmatize(word) for word in x])\n        return data\n    \n    def pipe_token(self,data,name):\n        pre_process=[self.token_it, self.dont_stop_me_now, self.lemmatize]\n        for func in pre_process:\n            data=func(data,name)\n        return data\n        \n#####\n    def preprocess(self):\n        print(\"Cleaning initiated...\")\n        self.df1['comment_text']=self.df1['comment_text'].apply(lambda x: self.pipe_text(x))\n\n        self.df1=self.pipe_token(self.df1,'comment_text') \n\n        return self.df1\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import AutoTokenizer\nfrom tokenizers import BertWordPieceTokenizer\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\n\n\nclass Pre_Model:\n    def __init__(self):\n        cleaner = PreProc()\n        self.data = cleaner.preprocess()\n        print('Data Cleaned.')\n        \n    def encoding(self,texts,tokenize,chunk_size= 279, max_len=128):\n        tokenize.enable_truncation(max_length=max_len)\n        tokenize.enable_padding(max_length=max_len)\n        all_ids = []\n        for i in range(0,len(texts),chunk_size):\n            text = texts[i:i+chunk_size:].tolist()\n            encs = tokenize.encode_batch(text)\n\n            all_ids.extend([enc.ids for enc in encs])\n\n        return (np.array(all_ids))\n    def preModel(self):\n        bert_tokenizer = BertWordPieceTokenizer('vocab.txt',lowercase = True)\n        x_train = self.encoding(self.data['comment_text'].astype(str), bert_tokenizer)\n        y_train = self.data.toxic.values\n        x_train, x_test, y_train, y_test = train_test_split(x_train, y_train, test_size = 0.2, random_state = 666)\n        return x_train, x_test, y_train, y_test\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"berted = Pre_Model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_test, y_train, y_test = berted.preModel()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BERT Std","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Optimizer: Adam\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import transformers\nimport keras \nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.models import Model\nfrom sklearn.metrics import accuracy_score,f1_score,precision_score,recall_score, confusion_matrix\nfrom sklearn.metrics import roc_curve, roc_auc_score\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom imblearn.over_sampling import ADASYN, BorderlineSMOTE, SMOTE\n\n\nclass Senti:\n    def __init__(self):\n        print(\"Initiated...\")\n        \n        self.x_train,self.x_test,self.y_train, self.y_test = x_train, x_test, y_train, y_test\n        print('Samples Loaded.')\n\n        \n    def modeling(self,max_len = 128):\n        print('Constructing model...')\n        transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-uncased')\n        )\n        inp = Input(shape = (max_len,),dtype = tf.int32)\n        sequence_op = transformer_layer(inp)[0]\n        cls_token = sequence_op[:,0,:]\n        out = Dense(1, activation = 'sigmoid')(cls_token)\n        model = Model(inputs = inp, outputs = out)\n        model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])\n        print('Model constructed.')\n        return model\n    def training(self):\n        \n        model = self.modeling()\n        call = [keras.callbacks.EarlyStopping(patience = 2)]\n        class_weights=compute_class_weight('balanced',np.unique(self.y_train),self.y_train)\n        print('\\n Fitting Model...')\n        history = model.fit(self.x_train, self.y_train, batch_size = 1923, epochs = 20,callbacks = call,shuffle = True, validation_data=(self.x_test,self.y_test))\n        return history,model\n        \n    def predic(self,model):\n        y_pred_train=[0 if o < 0.5 else 1 for o in model.predict(self.x_train)]\n        y_pred_test=[0 if o < 0.5 else 1 for o in model.predict(self.x_test)]\n        acc_tra,f1_tra,pre_tra,rec_tra=accuracy_score(self.y_train,y_pred_train),f1_score(self.y_train,y_pred_train),precision_score(self.y_train,y_pred_train),recall_score(self.y_train,y_pred_train)\n        acc_test,f1_test,pre_test,rec_test=accuracy_score(self.y_test,y_pred_test),f1_score(self.y_test,y_pred_test),precision_score(self.y_test,y_pred_test),recall_score(self.y_test,y_pred_test)\n        conf_train=confusion_matrix(self.y_train,y_pred_train)\n        conf_test=confusion_matrix(self.y_test,y_pred_test)\n        \n        print('\\n\\n\\nSTATSSSSSS BABYYYYY:\\n\\n')\n        print('TRAINING DATA:\\n\\n\\n')\n        print('CONFUCIAN MATRIX: \\n',conf_train)\n        print('Accuracy: ',acc_tra)\n        print('F1_Score: ',f1_tra)\n        print('Preision: ',pre_tra)\n        print('Recall: ',rec_tra)\n        print('ROC AND AUC', roc_auc_score(self.y_train, y_pred_train))\n\n        \n        print('\\n\\n\\nTESTING DATA:\\n\\n\\n')\n        print('CONFUCIAN MATRIX: \\n',conf_test)\n        print('Accuracy: ',acc_test)\n        print('F1_Score: ',f1_test)\n        print('Preision: ',pre_test)\n        print('Recall: ',rec_test)\n        print('ROC AND AUC', roc_auc_score(self.y_test, y_pred_test))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport datetime\nstart=datetime.datetime.now()\nobj = Senti()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    history, model = obj.training() \nend=datetime.datetime.now()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history.history.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Model Evaluation graphs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"obj.graph(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Scores","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"obj.predic(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Time take for complete model execution:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"end-start","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Saved Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights('Model_bench.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BERT V1","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Optimizer: Adamax\n\nMetric: Accuracy\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import transformers\nimport keras \nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.models import Model\nfrom sklearn.metrics import accuracy_score,f1_score,precision_score,recall_score, confusion_matrix\nfrom sklearn.metrics import roc_curve, roc_auc_score\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom imblearn.over_sampling import ADASYN, BorderlineSMOTE, SMOTE\n\n\nclass Senti_v1:\n    def __init__(self):\n        print(\"Initiated...\")\n        berted = Pre_Model()\n        self.x_train,self.x_test,self.y_train, self.y_test = x_train, x_test, y_train, y_test\n        print('Samples Loaded.')\n\n        \n    def modeling(self,max_len = 128):\n        print('Constructing model...')\n        transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-uncased')\n        )\n        inp = Input(shape = (max_len,),dtype = tf.int32)\n        sequence_op = transformer_layer(inp)[0]\n        cls_token = sequence_op[:,0,:]\n        out = Dense(1, activation = 'sigmoid')(cls_token)\n        model = Model(inputs = inp, outputs = out)\n        model.compile(loss = 'binary_crossentropy', optimizer = 'Adamax', metrics = ['accuracy'])\n        print('Model constructed.')\n        return model\n    def training(self):\n        \n        model = self.modeling()\n        call = [keras.callbacks.EarlyStopping(patience = 2)]\n        class_weights=compute_class_weight('balanced',np.unique(self.y_train),self.y_train)\n        print('\\n Fitting Model...')\n        history = model.fit(self.x_train, self.y_train, batch_size = 1923, epochs = 20,callbacks = call,shuffle = True, validation_data=(self.x_test,self.y_test))\n        return history,model\n    def graph(self, history):\n        plt.plot(history.history['auc'])\n        plt.plot(history.history['val_accuracy'])\n        plt.title('model accuracy')\n        plt.ylabel('accuracy')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n        \n        plt.plot(history.history['loss'])\n        plt.plot(history.history['val_loss'])\n        plt.title('model loss')\n        plt.ylabel('loss')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n        \n    def predic(self,model):\n        y_pred_train=[0 if o < 0.5 else 1 for o in model.predict(self.x_train)]\n        y_pred_test=[0 if o < 0.5 else 1 for o in model.predict(self.x_test)]\n        acc_tra,f1_tra,pre_tra,rec_tra=accuracy_score(self.y_train,y_pred_train),f1_score(self.y_train,y_pred_train),precision_score(self.y_train,y_pred_train),recall_score(self.y_train,y_pred_train)\n        acc_test,f1_test,pre_test,rec_test=accuracy_score(self.y_test,y_pred_test),f1_score(self.y_test,y_pred_test),precision_score(self.y_test,y_pred_test),recall_score(self.y_test,y_pred_test)\n        conf_train=confusion_matrix(self.y_train,y_pred_train)\n        conf_test=confusion_matrix(self.y_test,y_pred_test)\n        \n        print('\\n\\n\\nSTATSSSSSS BABYYYYY:\\n\\n')\n        print('TRAINING DATA:\\n\\n\\n')\n        print('CONFUCIAN MATRIX: \\n',conf_train)\n        print('Accuracy: ',acc_tra)\n        print('F1_Score: ',f1_tra)\n        print('Preision: ',pre_tra)\n        print('Recall: ',rec_tra)\n        print('ROC AND AUC', roc_auc_score(self.y_train, y_pred_train))\n\n        \n        print('\\n\\n\\nTESTING DATA:\\n\\n\\n')\n        print('CONFUCIAN MATRIX: \\n',conf_test)\n        print('Accuracy: ',acc_test)\n        print('F1_Score: ',f1_test)\n        print('Preision: ',pre_test)\n        print('Recall: ',rec_test)\n        print('ROC AND AUC', roc_auc_score(self.y_test, y_pred_test))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport datetime\nstart=datetime.datetime.now()\nobj_v1 = Senti_v1()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    history_v1, model_v1 = obj_v1.training() \nend=datetime.datetime.now()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_v1.history.keys()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Evaluation graphs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"graph(history_v1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Scores","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"obj_v1.predic(model_v1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Time take for complete model execution:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"end-start","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Saved Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_v1.save_weights('Model_v1.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BERT- V2","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Optimizer: Stochastic Grad Des\n\nMetric: Accuracy","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import transformers\nimport keras \nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.models import Model\nfrom sklearn.metrics import accuracy_score,f1_score,precision_score,recall_score, confusion_matrix\nfrom sklearn.metrics import roc_curve, roc_auc_score\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom imblearn.over_sampling import ADASYN, BorderlineSMOTE, SMOTE\n\n\nclass Senti_v2:\n    def __init__(self):\n        print(\"Initiated...\")\n        berted = Pre_Model()\n        self.x_train,self.x_test,self.y_train, self.y_test = x_train, x_test, y_train, y_test\n        print('Samples Loaded.')\n        \n    def modeling(self,max_len = 128):\n        print('Constructing model...')\n        transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-uncased')\n        )\n        inp = Input(shape = (max_len,),dtype = tf.int32)\n        sequence_op = transformer_layer(inp)[0]\n        cls_token = sequence_op[:,0,:]\n        out = Dense(1, activation = 'sigmoid')(cls_token)\n        model = Model(inputs = inp, outputs = out)\n        model.compile(loss = 'binary_crossentropy', optimizer = 'sgd', metrics = ['accuracy'])\n        print('Model constructed.')\n        return model\n    def training(self):\n        \n        model = self.modeling()\n        call = [keras.callbacks.EarlyStopping(patience = 2)]\n        class_weights=compute_class_weight('balanced',np.unique(self.y_train),self.y_train)\n        print('\\n Fitting Model...')\n        history = model.fit(self.x_train, self.y_train, batch_size = 1923, epochs = 20,callbacks = call,shuffle = True, validation_data=(self.x_test,self.y_test))\n        return history,model\n    def graph(self, history):\n        plt.plot(history.history['accuracy'])\n        plt.plot(history.history['val_accuracy'])\n        plt.title('model accuracy')\n        plt.ylabel('accuracy')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n        \n        plt.plot(history.history['loss'])\n        plt.plot(history.history['val_loss'])\n        plt.title('model loss')\n        plt.ylabel('loss')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n        \n    def predic(self,model):\n        y_pred_train=[0 if o < 0.5 else 1 for o in model.predict(self.x_train)]\n        y_pred_test=[0 if o < 0.5 else 1 for o in model.predict(self.x_test)]\n        acc_tra,f1_tra,pre_tra,rec_tra=accuracy_score(self.y_train,y_pred_train),f1_score(self.y_train,y_pred_train),precision_score(self.y_train,y_pred_train),recall_score(self.y_train,y_pred_train)\n        acc_test,f1_test,pre_test,rec_test=accuracy_score(self.y_test,y_pred_test),f1_score(self.y_test,y_pred_test),precision_score(self.y_test,y_pred_test),recall_score(self.y_test,y_pred_test)\n        conf_train=confusion_matrix(self.y_train,y_pred_train)\n        conf_test=confusion_matrix(self.y_test,y_pred_test)\n        \n        print('\\n\\n\\nSTATSSSSSS BABYYYYY:\\n\\n')\n        print('TRAINING DATA:\\n\\n\\n')\n        print('CONFUCIAN MATRIX: \\n',conf_train)\n        print('Accuracy: ',acc_tra)\n        print('F1_Score: ',f1_tra)\n        print('Preision: ',pre_tra)\n        print('Recall: ',rec_tra)\n        print('ROC AND AUC', roc_auc_score(self.y_train, y_pred_train))\n\n        \n        print('\\n\\n\\nTESTING DATA:\\n\\n\\n')\n        print('CONFUCIAN MATRIX: \\n',conf_test)\n        print('Accuracy: ',acc_test)\n        print('F1_Score: ',f1_test)\n        print('Preision: ',pre_test)\n        print('Recall: ',rec_test)\n        print('ROC AND AUC', roc_auc_score(self.y_test, y_pred_test))\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start=datetime.datetime.now()\nobj_v2 = Senti_v2()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    history_v2, model_v2 = obj_v2.training() \nend=datetime.datetime.now()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_v2.history.keys()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Evaluation graphs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"graph(history_v2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Score","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"obj_v2.predic(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Time taken for model execution","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"end-start","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Saved Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_v2.save_weights('Model_v2.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BERT V3","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Optimizer: Stochastic Grad Des\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import transformers\nimport keras \nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.models import Model\nfrom sklearn.metrics import accuracy_score,f1_score,precision_score,recall_score, confusion_matrix\nfrom sklearn.metrics import roc_curve, roc_auc_score\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom imblearn.over_sampling import ADASYN, BorderlineSMOTE, SMOTE\nmodel_path = 'final_model.h5'\n\nclass Senti_v3:\n    def __init__(self):\n        print(\"Initiated...\")\n        berted = Pre_Model()\n        self.x_train,self.x_test,self.y_train, self.y_test = x_train, x_test, y_train, y_test\n        print('Samples Loaded.')\n\n        \n    def modeling(self,max_len = 128):\n        print('Constructing model...')\n        transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-uncased')\n        )\n        inp = Input(shape = (max_len,),dtype = tf.int32)\n        sequence_op = transformer_layer(inp)[0]\n        cls_token = sequence_op[:,0,:]\n        out = Dense(1, activation = 'sigmoid')(cls_token)\n        model = Model(inputs = inp, outputs = out)\n        model.compile(loss = 'binary_crossentropy', optimizer = 'Adamax', metrics = [tf.keras.metrics.AUC()])\n        print('Model constructed.')\n        return model\n    def training(self):\n        \n        model = self.modeling()\n        \n        early_stop = tf.keras.callbacks.EarlyStopping(patience = 1)\n        call = [early_stop]\n        class_weights=compute_class_weight('balanced',np.unique(self.y_train),self.y_train)\n        print('\\n Fitting Model...')\n        history = model.fit(self.x_train, self.y_train, batch_size = 1923, epochs = 20,callbacks = call,shuffle = True, validation_data=(self.x_test,self.y_test))\n        return history,model\n    def graph(history):\n        plt.plot(history.history['auc'])\n        plt.plot(history.history['val_auc'])\n        plt.title('model accuracy')\n        plt.ylabel('accuracy')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n\n        plt.plot(history.history['loss'])\n        plt.plot(history.history['val_loss'])\n        plt.title('model loss')\n        plt.ylabel('loss')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n    def predic(self,model):\n        y_pred_train=[0 if o < 0.5 else 1 for o in model.predict(self.x_train)]\n        y_pred_test=[0 if o < 0.5 else 1 for o in model.predict(self.x_test)]\n        acc_tra,f1_tra,pre_tra,rec_tra=accuracy_score(self.y_train,y_pred_train),f1_score(self.y_train,y_pred_train),precision_score(self.y_train,y_pred_train),recall_score(self.y_train,y_pred_train)\n        acc_test,f1_test,pre_test,rec_test=accuracy_score(self.y_test,y_pred_test),f1_score(self.y_test,y_pred_test),precision_score(self.y_test,y_pred_test),recall_score(self.y_test,y_pred_test)\n        conf_train=confusion_matrix(self.y_train,y_pred_train)\n        conf_test=confusion_matrix(self.y_test,y_pred_test)\n        \n        print('\\n\\n\\nSTATSSSSSS BABYYYYY:\\n\\n')\n        print('TRAINING DATA:\\n\\n\\n')\n        print('CONFUCIAN MATRIX: \\n',conf_train)\n        print('Accuracy: ',acc_tra)\n        print('F1_Score: ',f1_tra)\n        print('Preision: ',pre_tra)\n        print('Recall: ',rec_tra)\n        print('ROC AND AUC', roc_auc_score(self.y_train, y_pred_train))\n\n        \n        print('\\n\\n\\nTESTING DATA:\\n\\n\\n')\n        print('CONFUCIAN MATRIX: \\n',conf_test)\n        print('Accuracy: ',acc_test)\n        print('F1_Score: ',f1_test)\n        print('Preision: ',pre_test)\n        print('Recall: ',rec_test)\n        print('ROC AND AUC', roc_auc_score(self.y_test, y_pred_test))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import datetime\nstart=datetime.datetime.now()\nobj_v3 = Senti_v3()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    history_v3, model_v3 = obj_v3.training() \nend=datetime.datetime.now()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_v3.history.keys()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Evaluation Graphs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"obj_v3.graph(history_v3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Scores","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"obj_v3.predic(model_v3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Saved Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_v3.save_weights('Model_v3.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Time to execute model completely","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"end-start","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BERT V4","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Optimiser: NAdam\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import transformers\nimport keras \nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.models import Model\nfrom sklearn.metrics import accuracy_score,f1_score,precision_score,recall_score, confusion_matrix\nfrom sklearn.metrics import roc_curve, roc_auc_score\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom imblearn.over_sampling import ADASYN, BorderlineSMOTE, SMOTE\nmodel_path = 'final_model.h5'\n\nclass Senti_v4:\n    def __init__(self):\n        print(\"Initiated...\")\n        berted = Pre_Model()\n        self.x_train,self.x_test,self.y_train, self.y_test = x_train, x_test, y_train, y_test\n        print('Samples Loaded.')\n        print('ADASYN initialising...')\n        ada = ADASYN()\n        print('ADASYN initiated.')\n        self.x_train, self.y_train = ada.fit_resample(self.x_train, self.y_train)\n        self.x_test, self.y_test = ada.fit_resample(self.x_test, self.y_test)\n        print('Resampled')\n        \n    def modeling(self,max_len = 128):\n        print('Constructing model...')\n        transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-uncased')\n        )\n        inp = Input(shape = (max_len,),dtype = tf.int32)\n        sequence_op = transformer_layer(inp)[0]\n        cls_token = sequence_op[:,0,:]\n        out = Dense(1, activation = 'sigmoid')(cls_token)\n        model = Model(inputs = inp, outputs = out)\n        model.compile(loss = 'binary_crossentropy', optimizer = 'Nadam', metrics = [tf.keras.metrics.AUC()])\n        print('Model constructed.')\n        return model\n    def training(self):\n        \n        model = self.modeling()\n        \n        early_stop = tf.keras.callbacks.EarlyStopping(patience = 1)\n        call = [early_stop]\n        class_weights=compute_class_weight('balanced',np.unique(self.y_train),self.y_train)\n        print('\\n Fitting Model...')\n        history = model.fit(self.x_train, self.y_train, batch_size = 1123, epochs = 20,callbacks = call,shuffle = True, validation_data=(self.x_test,self.y_test))\n        return history,model\n    def graph(history):\n        plt.plot(history.history['auc'])\n        plt.plot(history.history['val_auc'])\n        plt.title('model accuracy')\n        plt.ylabel('accuracy')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n\n        plt.plot(history.history['loss'])\n        plt.plot(history.history['val_loss'])\n        plt.title('model loss')\n        plt.ylabel('loss')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n    def predic(self,model):\n        y_pred_train=[0 if o < 0.5 else 1 for o in model.predict(self.x_train)]\n        y_pred_test=[0 if o < 0.5 else 1 for o in model.predict(self.x_test)]\n        acc_tra,f1_tra,pre_tra,rec_tra=accuracy_score(self.y_train,y_pred_train),f1_score(self.y_train,y_pred_train),precision_score(self.y_train,y_pred_train),recall_score(self.y_train,y_pred_train)\n        acc_test,f1_test,pre_test,rec_test=accuracy_score(self.y_test,y_pred_test),f1_score(self.y_test,y_pred_test),precision_score(self.y_test,y_pred_test),recall_score(self.y_test,y_pred_test)\n        conf_train=confusion_matrix(self.y_train,y_pred_train)\n        conf_test=confusion_matrix(self.y_test,y_pred_test)\n        \n        print('\\n\\n\\nSTATSSSSSS BABYYYYY:\\n\\n')\n        print('TRAINING DATA:\\n\\n\\n')\n        print('CONFUCIAN MATRIX: \\n',conf_train)\n        print('Accuracy: ',acc_tra)\n        print('F1_Score: ',f1_tra)\n        print('Preision: ',pre_tra)\n        print('Recall: ',rec_tra)\n        print('ROC AND AUC', roc_auc_score(self.y_train, y_pred_train))\n\n        \n        print('\\n\\n\\nTESTING DATA:\\n\\n\\n')\n        print('CONFUCIAN MATRIX: \\n',conf_test)\n        print('Accuracy: ',acc_test)\n        print('F1_Score: ',f1_test)\n        print('Preision: ',pre_test)\n        print('Recall: ',rec_test)\n        print('ROC AND AUC', roc_auc_score(self.y_test, y_pred_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import datetime\nstart=datetime.datetime.now()\nobj_v4 = Senti_v4()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    history_v4, model_v4 = obj_v4.training() \nend=datetime.datetime.now()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Scores","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"obj_v4.predic(model_v4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Time to execute model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"end-start","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Saved Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_v4.save_weights('Model_v4.h5')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}