{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ntrain_df=pd.read_csv('../input/nlp-getting-started/train.csv')\ntest_df=pd.read_csv('../input/nlp-getting-started/test.csv')\nsub_df=pd.read_csv('../input/nlp-getting-started/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:19:04.012677Z","iopub.execute_input":"2022-07-15T13:19:04.012957Z","iopub.status.idle":"2022-07-15T13:19:04.063238Z","shell.execute_reply.started":"2022-07-15T13:19:04.012930Z","shell.execute_reply":"2022-07-15T13:19:04.062356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:56:33.289283Z","iopub.execute_input":"2022-07-15T08:56:33.289612Z","iopub.status.idle":"2022-07-15T08:56:33.309385Z","shell.execute_reply.started":"2022-07-15T08:56:33.289585Z","shell.execute_reply":"2022-07-15T08:56:33.308831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=train_df.drop(columns=['id','keyword','location'],axis=1)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:56:33.355429Z","iopub.execute_input":"2022-07-15T08:56:33.355691Z","iopub.status.idle":"2022-07-15T08:56:33.371293Z","shell.execute_reply.started":"2022-07-15T08:56:33.355665Z","shell.execute_reply":"2022-07-15T08:56:33.370569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:56:33.418456Z","iopub.execute_input":"2022-07-15T08:56:33.419079Z","iopub.status.idle":"2022-07-15T08:56:33.423476Z","shell.execute_reply.started":"2022-07-15T08:56:33.419030Z","shell.execute_reply":"2022-07-15T08:56:33.422904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import transformers\nfrom transformers import BertTokenizer, TFBertModel\ntokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\nbert = TFBertModel.from_pretrained(\"bert-base-uncased\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:56:33.509467Z","iopub.execute_input":"2022-07-15T08:56:33.509888Z","iopub.status.idle":"2022-07-15T08:56:59.430604Z","shell.execute_reply.started":"2022-07-15T08:56:33.509835Z","shell.execute_reply":"2022-07-15T08:56:59.429762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('max lenth of word is',max([len(x.split()) for x in train_df['text']]))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:56:59.432161Z","iopub.execute_input":"2022-07-15T08:56:59.432386Z","iopub.status.idle":"2022-07-15T08:56:59.450403Z","shell.execute_reply.started":"2022-07-15T08:56:59.432361Z","shell.execute_reply":"2022-07-15T08:56:59.449633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_length=35","metadata":{"execution":{"iopub.status.busy":"2022-07-15T11:29:16.664323Z","iopub.execute_input":"2022-07-15T11:29:16.664777Z","iopub.status.idle":"2022-07-15T11:29:16.668420Z","shell.execute_reply.started":"2022-07-15T11:29:16.664746Z","shell.execute_reply":"2022-07-15T11:29:16.667574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:58:33.275302Z","iopub.execute_input":"2022-07-15T08:58:33.276129Z","iopub.status.idle":"2022-07-15T08:58:33.281655Z","shell.execute_reply.started":"2022-07-15T08:58:33.276093Z","shell.execute_reply":"2022-07-15T08:58:33.280850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:58:36.419899Z","iopub.execute_input":"2022-07-15T08:58:36.420176Z","iopub.status.idle":"2022-07-15T08:58:36.427239Z","shell.execute_reply.started":"2022-07-15T08:58:36.420149Z","shell.execute_reply":"2022-07-15T08:58:36.426410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c='he is a bad man'\ntokenizer(c)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T08:58:37.755468Z","iopub.execute_input":"2022-07-15T08:58:37.756230Z","iopub.status.idle":"2022-07-15T08:58:37.763635Z","shell.execute_reply.started":"2022-07-15T08:58:37.756177Z","shell.execute_reply":"2022-07-15T08:58:37.762875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train=tokenizer(\n\n    text=train_df['text'].tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_attention_mask=True,\n    verbose=True\n    \n)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T11:29:23.354089Z","iopub.execute_input":"2022-07-15T11:29:23.354384Z","iopub.status.idle":"2022-07-15T11:29:29.523959Z","shell.execute_reply.started":"2022-07-15T11:29:23.354341Z","shell.execute_reply":"2022-07-15T11:29:29.522573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train","metadata":{"execution":{"iopub.status.busy":"2022-07-15T11:29:35.178794Z","iopub.execute_input":"2022-07-15T11:29:35.179083Z","iopub.status.idle":"2022-07-15T11:29:35.185779Z","shell.execute_reply.started":"2022-07-15T11:29:35.179051Z","shell.execute_reply":"2022-07-15T11:29:35.184955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import Model,layers,optimizers,callbacks,losses,metrics,utils","metadata":{"execution":{"iopub.status.busy":"2022-07-15T11:29:39.471264Z","iopub.execute_input":"2022-07-15T11:29:39.471623Z","iopub.status.idle":"2022-07-15T11:29:39.476065Z","shell.execute_reply.started":"2022-07-15T11:29:39.471589Z","shell.execute_reply":"2022-07-15T11:29:39.475261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.attention_mask","metadata":{"execution":{"iopub.status.busy":"2022-07-15T11:29:43.728266Z","iopub.execute_input":"2022-07-15T11:29:43.728694Z","iopub.status.idle":"2022-07-15T11:29:43.734027Z","shell.execute_reply.started":"2022-07-15T11:29:43.728663Z","shell.execute_reply":"2022-07-15T11:29:43.733482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_ids=layers.Input(shape=(max_length,),dtype=tf.int32,name='input_ids')\nattention_mask=layers.Input(shape=(max_length,),dtype=tf.int32,name='attention_mask')\nembedding=bert(input_ids,attention_mask)[1]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T11:30:05.481950Z","iopub.execute_input":"2022-07-15T11:30:05.482436Z","iopub.status.idle":"2022-07-15T11:30:07.168048Z","shell.execute_reply.started":"2022-07-15T11:30:05.482399Z","shell.execute_reply":"2022-07-15T11:30:07.167152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out=layers.Dropout(0.1)(embedding)\nout=layers.Dense(399,activation='relu')(out)\n\nout=layers.Dropout(0.15)(out)\nout=layers.Dense(67,activation='relu')(out)\nout=layers.Dense(17,activation='relu')(out)\ny=layers.Dense(1,activation='sigmoid')(out)\n\nnn=Model(inputs=[input_ids,attention_mask],outputs=y)\nnn.layers[2].trainable=True\noptimizer=optimizers.Adam(learning_rate = 1e-5)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T11:30:12.698015Z","iopub.execute_input":"2022-07-15T11:30:12.698318Z","iopub.status.idle":"2022-07-15T11:30:12.752966Z","shell.execute_reply.started":"2022-07-15T11:30:12.698286Z","shell.execute_reply":"2022-07-15T11:30:12.752194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=train_df['target']\ny_train","metadata":{"execution":{"iopub.status.busy":"2022-07-15T11:30:16.818565Z","iopub.execute_input":"2022-07-15T11:30:16.818832Z","iopub.status.idle":"2022-07-15T11:30:16.826546Z","shell.execute_reply.started":"2022-07-15T11:30:16.818805Z","shell.execute_reply":"2022-07-15T11:30:16.825886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn.compile(optimizer=optimizer,loss='binary_crossentropy',metrics=['accuracy'])\nnn.fit(\n\n    x={'input_ids':x_train['input_ids'],'attention_mask':x_train['attention_mask']},\n    y=y_train,\n    validation_split=0.20,\n    epochs=1,\n    batch_size=45\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T12:58:32.993851Z","iopub.execute_input":"2022-07-15T12:58:32.994428Z","iopub.status.idle":"2022-07-15T13:18:15.304005Z","shell.execute_reply.started":"2022-07-15T12:58:32.994389Z","shell.execute_reply":"2022-07-15T13:18:15.303422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Working with test data","metadata":{}},{"cell_type":"code","source":"test_df=test_df.drop(columns=['id','keyword','location'],axis=1)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:19:12.905240Z","iopub.execute_input":"2022-07-15T13:19:12.905538Z","iopub.status.idle":"2022-07-15T13:19:12.915803Z","shell.execute_reply.started":"2022-07-15T13:19:12.905508Z","shell.execute_reply":"2022-07-15T13:19:12.914975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test=tokenizer(\n\n    text=test_df.text.tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_attention_mask=True,\n    verbose=True\n\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:19:15.319714Z","iopub.execute_input":"2022-07-15T13:19:15.319996Z","iopub.status.idle":"2022-07-15T13:19:17.851121Z","shell.execute_reply.started":"2022-07-15T13:19:15.319967Z","shell.execute_reply":"2022-07-15T13:19:17.850103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test.attention_mask.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:19:19.870455Z","iopub.execute_input":"2022-07-15T13:19:19.870722Z","iopub.status.idle":"2022-07-15T13:19:19.876300Z","shell.execute_reply.started":"2022-07-15T13:19:19.870693Z","shell.execute_reply":"2022-07-15T13:19:19.875667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted=nn.predict({'input_ids':x_test['input_ids'],'attention_mask':x_test['attention_mask']})","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:19:21.803073Z","iopub.execute_input":"2022-07-15T13:19:21.803629Z","iopub.status.idle":"2022-07-15T13:22:28.094580Z","shell.execute_reply.started":"2022-07-15T13:19:21.803580Z","shell.execute_reply":"2022-07-15T13:22:28.093777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predicted=np.where(predicted>0.5,1,0)\ny_predicted","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:23:40.552413Z","iopub.execute_input":"2022-07-15T13:23:40.553276Z","iopub.status.idle":"2022-07-15T13:23:40.560871Z","shell.execute_reply.started":"2022-07-15T13:23:40.553226Z","shell.execute_reply":"2022-07-15T13:23:40.560230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predicted=y_predicted.reshape(-1,)\nsub=pd.DataFrame({'id':sub_df['id'],'target':y_predicted})\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:23:44.532939Z","iopub.execute_input":"2022-07-15T13:23:44.533491Z","iopub.status.idle":"2022-07-15T13:23:44.544047Z","shell.execute_reply.started":"2022-07-15T13:23:44.533455Z","shell.execute_reply":"2022-07-15T13:23:44.543249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:23:48.421146Z","iopub.execute_input":"2022-07-15T13:23:48.421898Z","iopub.status.idle":"2022-07-15T13:23:48.431838Z","shell.execute_reply.started":"2022-07-15T13:23:48.421856Z","shell.execute_reply":"2022-07-15T13:23:48.431261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('bert_nlp_4.csv',index=None)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T13:23:55.178161Z","iopub.execute_input":"2022-07-15T13:23:55.178764Z","iopub.status.idle":"2022-07-15T13:23:55.190677Z","shell.execute_reply.started":"2022-07-15T13:23:55.178715Z","shell.execute_reply":"2022-07-15T13:23:55.190079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}