{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ntrain_df=pd.read_csv('../input/nlp-getting-started/train.csv')\ntest_df=pd.read_csv('../input/nlp-getting-started/test.csv')\nsub_df=pd.read_csv('../input/nlp-getting-started/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:10:43.432256Z","iopub.execute_input":"2022-07-14T21:10:43.432982Z","iopub.status.idle":"2022-07-14T21:10:43.485170Z","shell.execute_reply.started":"2022-07-14T21:10:43.432946Z","shell.execute_reply":"2022-07-14T21:10:43.484218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:11:24.052605Z","iopub.execute_input":"2022-07-14T21:11:24.053534Z","iopub.status.idle":"2022-07-14T21:11:24.058722Z","shell.execute_reply.started":"2022-07-14T21:11:24.053489Z","shell.execute_reply":"2022-07-14T21:11:24.057601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:10:55.090129Z","iopub.execute_input":"2022-07-14T21:10:55.090577Z","iopub.status.idle":"2022-07-14T21:10:55.115868Z","shell.execute_reply.started":"2022-07-14T21:10:55.090536Z","shell.execute_reply":"2022-07-14T21:10:55.114862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install text_hammer","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:12:42.310496Z","iopub.execute_input":"2022-07-14T21:12:42.310893Z","iopub.status.idle":"2022-07-14T21:12:54.751700Z","shell.execute_reply.started":"2022-07-14T21:12:42.310859Z","shell.execute_reply":"2022-07-14T21:12:54.750380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import text_hammer as th","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:13:45.051722Z","iopub.execute_input":"2022-07-14T21:13:45.052504Z","iopub.status.idle":"2022-07-14T21:13:48.822988Z","shell.execute_reply.started":"2022-07-14T21:13:45.052462Z","shell.execute_reply":"2022-07-14T21:13:48.821900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfrom tqdm._tqdm_notebook import tqdm_notebook\ntqdm_notebook.pandas()\n\ndef text_preprocessing(df,col_name):\n    column=col_name\n    df[column]=df[column].progress_apply(lambda x: str(x).lower())\n    df[column]=df[column].progress_apply(lambda x: th.remove_emails(x))\n    df[column]=df[column].progress_apply(lambda x: th.remove_html_tags(x))\n    df[column]=df[column].progress_apply(lambda x: th.make_base(x))    \n    df[column]=df[column].progress_apply(lambda x: th.cont_exp(x))\n    df[column]=df[column].progress_apply(lambda x: th.remove_special_chars(x))\n    df[column]=df[column].progress_apply(lambda x: th.remove_accented_chars(x))\n                                         \n                                         \n    return df\n                                         ","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:23:07.413439Z","iopub.execute_input":"2022-07-14T21:23:07.413803Z","iopub.status.idle":"2022-07-14T21:23:07.425040Z","shell.execute_reply.started":"2022-07-14T21:23:07.413773Z","shell.execute_reply":"2022-07-14T21:23:07.424030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cleaned_df=text_preprocessing(train_df,'text')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:24:09.372591Z","iopub.execute_input":"2022-07-14T21:24:09.373048Z","iopub.status.idle":"2022-07-14T21:25:37.276112Z","shell.execute_reply.started":"2022-07-14T21:24:09.373011Z","shell.execute_reply":"2022-07-14T21:25:37.275137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cleaned_df","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:26:33.958919Z","iopub.execute_input":"2022-07-14T21:26:33.959269Z","iopub.status.idle":"2022-07-14T21:26:33.979018Z","shell.execute_reply.started":"2022-07-14T21:26:33.959240Z","shell.execute_reply":"2022-07-14T21:26:33.977810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=train_cleaned_df.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:28:00.961014Z","iopub.execute_input":"2022-07-14T21:28:00.961437Z","iopub.status.idle":"2022-07-14T21:28:00.967971Z","shell.execute_reply.started":"2022-07-14T21:28:00.961404Z","shell.execute_reply":"2022-07-14T21:28:00.966804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=train_df.drop(columns=['id','keyword','location'],axis=1)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:28:10.320953Z","iopub.execute_input":"2022-07-14T21:28:10.321577Z","iopub.status.idle":"2022-07-14T21:28:10.334591Z","shell.execute_reply.started":"2022-07-14T21:28:10.321539Z","shell.execute_reply":"2022-07-14T21:28:10.333600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:28:33.412107Z","iopub.execute_input":"2022-07-14T21:28:33.412552Z","iopub.status.idle":"2022-07-14T21:28:33.431092Z","shell.execute_reply.started":"2022-07-14T21:28:33.412509Z","shell.execute_reply":"2022-07-14T21:28:33.429791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading pretrained BERT model","metadata":{}},{"cell_type":"code","source":"import transformers\nfrom transformers import AutoTokenizer,TFBertModel\ntokenizer=AutoTokenizer.from_pretrained('bert-large-uncased')\nbert=TFBertModel.from_pretrained('bert-large-uncased')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:32:54.031590Z","iopub.execute_input":"2022-07-14T21:32:54.032183Z","iopub.status.idle":"2022-07-14T21:34:10.511866Z","shell.execute_reply.started":"2022-07-14T21:32:54.032146Z","shell.execute_reply":"2022-07-14T21:34:10.510966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer('i will be kaggle grandmaster')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:36:37.702295Z","iopub.execute_input":"2022-07-14T21:36:37.702695Z","iopub.status.idle":"2022-07-14T21:36:37.717568Z","shell.execute_reply.started":"2022-07-14T21:36:37.702663Z","shell.execute_reply":"2022-07-14T21:36:37.716640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('max length of words is ',max([len(x.split()) for x in train_df.text]))","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:38:57.180964Z","iopub.execute_input":"2022-07-14T21:38:57.181312Z","iopub.status.idle":"2022-07-14T21:38:57.197232Z","shell.execute_reply.started":"2022-07-14T21:38:57.181284Z","shell.execute_reply":"2022-07-14T21:38:57.196095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_length=40","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:44:10.800877Z","iopub.execute_input":"2022-07-14T21:44:10.801456Z","iopub.status.idle":"2022-07-14T21:44:10.806131Z","shell.execute_reply.started":"2022-07-14T21:44:10.801420Z","shell.execute_reply":"2022-07-14T21:44:10.804938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Changing our text data into bert input format","metadata":{}},{"cell_type":"code","source":"x_train=tokenizer(\n        \n    text=train_df['text'].tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_token_type_ids=True,\n    return_attention_mask=True,\n    verbose=True\n    \n    )","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:44:22.930229Z","iopub.execute_input":"2022-07-14T21:44:22.930571Z","iopub.status.idle":"2022-07-14T21:44:23.561225Z","shell.execute_reply.started":"2022-07-14T21:44:22.930543Z","shell.execute_reply":"2022-07-14T21:44:23.560200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.input_ids.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:46:34.239549Z","iopub.execute_input":"2022-07-14T21:46:34.240001Z","iopub.status.idle":"2022-07-14T21:46:34.252340Z","shell.execute_reply.started":"2022-07-14T21:46:34.239961Z","shell.execute_reply":"2022-07-14T21:46:34.250567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.attention_mask.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:46:09.754107Z","iopub.execute_input":"2022-07-14T21:46:09.754506Z","iopub.status.idle":"2022-07-14T21:46:09.761674Z","shell.execute_reply.started":"2022-07-14T21:46:09.754474Z","shell.execute_reply":"2022-07-14T21:46:09.759834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.token_type_ids.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:46:59.508917Z","iopub.execute_input":"2022-07-14T21:46:59.509281Z","iopub.status.idle":"2022-07-14T21:46:59.519143Z","shell.execute_reply.started":"2022-07-14T21:46:59.509252Z","shell.execute_reply":"2022-07-14T21:46:59.517548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=train_df['target'].values\ny_train","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:47:56.630859Z","iopub.execute_input":"2022-07-14T21:47:56.631415Z","iopub.status.idle":"2022-07-14T21:47:56.638656Z","shell.execute_reply.started":"2022-07-14T21:47:56.631380Z","shell.execute_reply":"2022-07-14T21:47:56.637533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import Input,Dense\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.initializers import TruncatedNormal\nfrom tensorflow.keras.losses import CategoricalCrossentropy,BinaryCrossentropy\nfrom tensorflow.keras.metrics import CategoricalAccuracy,BinaryAccuracy\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.utils import plot_model","metadata":{"execution":{"iopub.status.busy":"2022-07-14T21:55:08.689076Z","iopub.execute_input":"2022-07-14T21:55:08.689422Z","iopub.status.idle":"2022-07-14T21:55:08.697901Z","shell.execute_reply.started":"2022-07-14T21:55:08.689393Z","shell.execute_reply":"2022-07-14T21:55:08.696681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_ids=Input(shape=(max_length,), dtype=tf.int32, name='input_ids')\ninput_mask=Input(shape=(max_length,),dtype=tf.int32 ,name='attention_mask')\ntoken_ids=Input(shape=(max_length,), dtype=tf.int32, name='token_type_ids')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:00:01.297536Z","iopub.execute_input":"2022-07-14T22:00:01.297913Z","iopub.status.idle":"2022-07-14T22:00:01.306564Z","shell.execute_reply.started":"2022-07-14T22:00:01.297879Z","shell.execute_reply":"2022-07-14T22:00:01.305537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding=bert(input_ids,attention_mask=input_mask,token_type_ids=token_ids)[1]\nout=tf.keras.layers.Dropout(0.1)(embedding)\n\nout=Dense(120,activation='relu')(out)\nout=tf.keras.layers.Dropout(0.11)(out)\nout=Dense(31,activation='relu')(out)\n\ny=Dense(1,activation='relu')(out)\n\nmodel=tf.keras.Model(inputs=[input_ids,input_mask,token_ids],outputs=y)\nmodel.layers[2].trainable=True\n","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:09:10.567099Z","iopub.execute_input":"2022-07-14T22:09:10.567445Z","iopub.status.idle":"2022-07-14T22:09:13.320545Z","shell.execute_reply.started":"2022-07-14T22:09:10.567417Z","shell.execute_reply":"2022-07-14T22:09:13.319591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:09:19.060319Z","iopub.execute_input":"2022-07-14T22:09:19.060682Z","iopub.status.idle":"2022-07-14T22:09:19.095553Z","shell.execute_reply.started":"2022-07-14T22:09:19.060653Z","shell.execute_reply":"2022-07-14T22:09:19.094536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer=Adam(\n \n    learning_rate=5e-6,\n    epsilon=1e-8,\n    decay=0.01,\n    clipnorm=1.0\n\n)\n\nloss=BinaryCrossentropy(from_logits=True)\nmetrics=BinaryAccuracy('accuracy')\nmodel.compile(optimizer=optimizer,loss=loss,metrics=metrics)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:13:30.579833Z","iopub.execute_input":"2022-07-14T22:13:30.580412Z","iopub.status.idle":"2022-07-14T22:13:30.609845Z","shell.execute_reply.started":"2022-07-14T22:13:30.580369Z","shell.execute_reply":"2022-07-14T22:13:30.608986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model=model,show_shapes=True,show_dtype=True,expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:15:51.500569Z","iopub.execute_input":"2022-07-14T22:15:51.501040Z","iopub.status.idle":"2022-07-14T22:15:51.800289Z","shell.execute_reply.started":"2022-07-14T22:15:51.501000Z","shell.execute_reply":"2022-07-14T22:15:51.799157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.config.experimental.list_physical_devices('GPU')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:17:44.580379Z","iopub.execute_input":"2022-07-14T22:17:44.580727Z","iopub.status.idle":"2022-07-14T22:17:44.590654Z","shell.execute_reply.started":"2022-07-14T22:17:44.580699Z","shell.execute_reply":"2022-07-14T22:17:44.589722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_history=model.fit(\n\n    x={'input_ids':x_train['input_ids'],'attention_mask':x_train['attention_mask'],'token_type_ids':x_train['token_type_ids']},\n    y=y_train,\n    validation_split=0.20,\n    epochs=3,\n    batch_size=45\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:25:44.460697Z","iopub.execute_input":"2022-07-14T22:25:44.461201Z","iopub.status.idle":"2022-07-14T22:32:06.466317Z","shell.execute_reply.started":"2022-07-14T22:25:44.461157Z","shell.execute_reply":"2022-07-14T22:32:06.465246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Working with test data","metadata":{}},{"cell_type":"code","source":"test_df=test_df.drop(columns=['id','keyword','location'],axis=1)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:33:19.940232Z","iopub.execute_input":"2022-07-14T22:33:19.940595Z","iopub.status.idle":"2022-07-14T22:33:19.951260Z","shell.execute_reply.started":"2022-07-14T22:33:19.940564Z","shell.execute_reply":"2022-07-14T22:33:19.950145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df=text_preprocessing(test_df,'text')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:33:36.480261Z","iopub.execute_input":"2022-07-14T22:33:36.480625Z","iopub.status.idle":"2022-07-14T22:34:11.993399Z","shell.execute_reply.started":"2022-07-14T22:33:36.480596Z","shell.execute_reply":"2022-07-14T22:34:11.992475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test=tokenizer(\n\n    text=test_df.text.tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_token_type_ids=True,\n    return_attention_mask=True,\n    verbose=True\n\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:35:57.610071Z","iopub.execute_input":"2022-07-14T22:35:57.610820Z","iopub.status.idle":"2022-07-14T22:35:58.059082Z","shell.execute_reply.started":"2022-07-14T22:35:57.610771Z","shell.execute_reply":"2022-07-14T22:35:58.058020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted=model.predict({'input_ids':x_test['input_ids'],'attention_mask':x_test['attention_mask'],'token_type_ids':x_test['token_type_ids']})","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:37:47.290453Z","iopub.execute_input":"2022-07-14T22:37:47.290981Z","iopub.status.idle":"2022-07-14T22:38:10.478799Z","shell.execute_reply.started":"2022-07-14T22:37:47.290942Z","shell.execute_reply":"2022-07-14T22:38:10.477793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:38:15.089369Z","iopub.execute_input":"2022-07-14T22:38:15.089705Z","iopub.status.idle":"2022-07-14T22:38:15.096400Z","shell.execute_reply.started":"2022-07-14T22:38:15.089675Z","shell.execute_reply":"2022-07-14T22:38:15.095460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:38:38.200052Z","iopub.execute_input":"2022-07-14T22:38:38.200626Z","iopub.status.idle":"2022-07-14T22:38:38.211400Z","shell.execute_reply.started":"2022-07-14T22:38:38.200578Z","shell.execute_reply":"2022-07-14T22:38:38.210283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predicted=np.where(predicted>0.5,1,0)\ny_predicted","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:39:23.270398Z","iopub.execute_input":"2022-07-14T22:39:23.270761Z","iopub.status.idle":"2022-07-14T22:39:23.278447Z","shell.execute_reply.started":"2022-07-14T22:39:23.270713Z","shell.execute_reply":"2022-07-14T22:39:23.277493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:40:13.430092Z","iopub.execute_input":"2022-07-14T22:40:13.430785Z","iopub.status.idle":"2022-07-14T22:40:13.451315Z","shell.execute_reply.started":"2022-07-14T22:40:13.430700Z","shell.execute_reply":"2022-07-14T22:40:13.449935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predicted=y_predicted.reshape(-1,)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:41:16.820197Z","iopub.execute_input":"2022-07-14T22:41:16.820544Z","iopub.status.idle":"2022-07-14T22:41:16.825181Z","shell.execute_reply.started":"2022-07-14T22:41:16.820514Z","shell.execute_reply":"2022-07-14T22:41:16.823969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.DataFrame({'id':sub_df['id'],'target':y_predicted})\nsubmission.tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:41:18.819647Z","iopub.execute_input":"2022-07-14T22:41:18.820320Z","iopub.status.idle":"2022-07-14T22:41:18.830123Z","shell.execute_reply.started":"2022-07-14T22:41:18.820280Z","shell.execute_reply":"2022-07-14T22:41:18.829191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.target.value_counts(normalize=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:41:41.980658Z","iopub.execute_input":"2022-07-14T22:41:41.981170Z","iopub.status.idle":"2022-07-14T22:41:41.991817Z","shell.execute_reply.started":"2022-07-14T22:41:41.981126Z","shell.execute_reply":"2022-07-14T22:41:41.990570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('bert_nlp2.csv',index=None)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T22:42:29.029648Z","iopub.execute_input":"2022-07-14T22:42:29.030040Z","iopub.status.idle":"2022-07-14T22:42:29.043582Z","shell.execute_reply.started":"2022-07-14T22:42:29.030009Z","shell.execute_reply":"2022-07-14T22:42:29.042552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}