{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ntrain_df=pd.read_csv('../input/contradictory-my-dear-watson/train.csv')\ntest_df=pd.read_csv('../input/contradictory-my-dear-watson/test.csv')\nsub_df=pd.read_csv('../input/contradictory-my-dear-watson/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:27:32.749802Z","iopub.execute_input":"2022-07-16T12:27:32.750105Z","iopub.status.idle":"2022-07-16T12:27:32.853073Z","shell.execute_reply.started":"2022-07-16T12:27:32.750058Z","shell.execute_reply":"2022-07-16T12:27:32.852131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.615222Z","iopub.execute_input":"2022-07-16T11:22:30.615564Z","iopub.status.idle":"2022-07-16T11:22:30.643808Z","shell.execute_reply.started":"2022-07-16T11:22:30.615523Z","shell.execute_reply":"2022-07-16T11:22:30.642813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.645475Z","iopub.execute_input":"2022-07-16T11:22:30.645784Z","iopub.status.idle":"2022-07-16T11:22:30.652565Z","shell.execute_reply.started":"2022-07-16T11:22:30.645747Z","shell.execute_reply":"2022-07-16T11:22:30.651526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['hypothesis'][1]","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.654414Z","iopub.execute_input":"2022-07-16T11:22:30.655007Z","iopub.status.idle":"2022-07-16T11:22:30.668484Z","shell.execute_reply.started":"2022-07-16T11:22:30.654954Z","shell.execute_reply":"2022-07-16T11:22:30.667392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['premise'][1]","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.672716Z","iopub.execute_input":"2022-07-16T11:22:30.673052Z","iopub.status.idle":"2022-07-16T11:22:30.682999Z","shell.execute_reply.started":"2022-07-16T11:22:30.673007Z","shell.execute_reply":"2022-07-16T11:22:30.682107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['language'].value_counts(normalize=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.684079Z","iopub.execute_input":"2022-07-16T11:22:30.686357Z","iopub.status.idle":"2022-07-16T11:22:30.71499Z","shell.execute_reply.started":"2022-07-16T11:22:30.686303Z","shell.execute_reply":"2022-07-16T11:22:30.714016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df['language']=='Hindi']","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.716689Z","iopub.execute_input":"2022-07-16T11:22:30.717265Z","iopub.status.idle":"2022-07-16T11:22:30.741159Z","shell.execute_reply.started":"2022-07-16T11:22:30.717225Z","shell.execute_reply":"2022-07-16T11:22:30.740156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.742851Z","iopub.execute_input":"2022-07-16T11:22:30.743386Z","iopub.status.idle":"2022-07-16T11:22:30.753734Z","shell.execute_reply.started":"2022-07-16T11:22:30.743347Z","shell.execute_reply":"2022-07-16T11:22:30.752336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['premise'][0]","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.757286Z","iopub.execute_input":"2022-07-16T11:22:30.759266Z","iopub.status.idle":"2022-07-16T11:22:30.765235Z","shell.execute_reply.started":"2022-07-16T11:22:30.759221Z","shell.execute_reply":"2022-07-16T11:22:30.764298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['hypothesis'][0]","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.767303Z","iopub.execute_input":"2022-07-16T11:22:30.767972Z","iopub.status.idle":"2022-07-16T11:22:30.780881Z","shell.execute_reply.started":"2022-07-16T11:22:30.767931Z","shell.execute_reply":"2022-07-16T11:22:30.77989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.78269Z","iopub.execute_input":"2022-07-16T11:22:30.783366Z","iopub.status.idle":"2022-07-16T11:22:30.800783Z","shell.execute_reply.started":"2022-07-16T11:22:30.783326Z","shell.execute_reply":"2022-07-16T11:22:30.799825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.802152Z","iopub.execute_input":"2022-07-16T11:22:30.803096Z","iopub.status.idle":"2022-07-16T11:22:30.813458Z","shell.execute_reply.started":"2022-07-16T11:22:30.803035Z","shell.execute_reply":"2022-07-16T11:22:30.812433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.814938Z","iopub.execute_input":"2022-07-16T11:22:30.81547Z","iopub.status.idle":"2022-07-16T11:22:30.836186Z","shell.execute_reply.started":"2022-07-16T11:22:30.815424Z","shell.execute_reply":"2022-07-16T11:22:30.835175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_df[['premise', 'hypothesis']].values.tolist()\ntest_data = test_df[['premise', 'hypothesis']].values.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:22:30.841209Z","iopub.execute_input":"2022-07-16T11:22:30.841453Z","iopub.status.idle":"2022-07-16T11:22:30.855298Z","shell.execute_reply.started":"2022-07-16T11:22:30.841425Z","shell.execute_reply":"2022-07-16T11:22:30.854442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import BertTokenizer, TFBertModel\ntokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased')\nbert = TFBertModel.from_pretrained(\"bert-base-multilingual-uncased\")","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:24:49.727513Z","iopub.execute_input":"2022-07-16T11:24:49.727825Z","iopub.status.idle":"2022-07-16T11:26:30.624939Z","shell.execute_reply.started":"2022-07-16T11:24:49.727796Z","shell.execute_reply":"2022-07-16T11:26:30.623204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer('he is fine')","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:28:26.214423Z","iopub.execute_input":"2022-07-16T11:28:26.215038Z","iopub.status.idle":"2022-07-16T11:28:26.224605Z","shell.execute_reply.started":"2022-07-16T11:28:26.215004Z","shell.execute_reply":"2022-07-16T11:28:26.223557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['hypothesis'].apply(lambda x: len(x.split())).mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:30:12.422678Z","iopub.execute_input":"2022-07-16T11:30:12.422948Z","iopub.status.idle":"2022-07-16T11:30:12.450003Z","shell.execute_reply.started":"2022-07-16T11:30:12.422919Z","shell.execute_reply":"2022-07-16T11:30:12.449098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('max length of word',max([len(x.split()) for x in train_df['hypothesis']]))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:32:08.78455Z","iopub.execute_input":"2022-07-16T11:32:08.784847Z","iopub.status.idle":"2022-07-16T11:32:08.80666Z","shell.execute_reply.started":"2022-07-16T11:32:08.784819Z","shell.execute_reply":"2022-07-16T11:32:08.805628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_length=90","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:19:12.860958Z","iopub.execute_input":"2022-07-16T12:19:12.86148Z","iopub.status.idle":"2022-07-16T12:19:12.866475Z","shell.execute_reply.started":"2022-07-16T12:19:12.861441Z","shell.execute_reply":"2022-07-16T12:19:12.86502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train=tokenizer(\n      \n    text=train_df[['premise','hypothesis']].values.tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_token_type_ids=True,\n    return_attention_mask=True,\n    verbose=True\n    \n    )","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:19:16.240419Z","iopub.execute_input":"2022-07-16T12:19:16.241139Z","iopub.status.idle":"2022-07-16T12:19:31.707319Z","shell.execute_reply.started":"2022-07-16T12:19:16.241063Z","shell.execute_reply":"2022-07-16T12:19:31.706202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:19:35.085404Z","iopub.execute_input":"2022-07-16T12:19:35.085691Z","iopub.status.idle":"2022-07-16T12:19:35.094919Z","shell.execute_reply.started":"2022-07-16T12:19:35.085662Z","shell.execute_reply":"2022-07-16T12:19:35.093908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.input_ids.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:19:38.879711Z","iopub.execute_input":"2022-07-16T12:19:38.88037Z","iopub.status.idle":"2022-07-16T12:19:38.887053Z","shell.execute_reply.started":"2022-07-16T12:19:38.880323Z","shell.execute_reply":"2022-07-16T12:19:38.886168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import Model,layers,optimizers,callbacks,losses,metrics,utils\ninput_ids=layers.Input(shape=(max_length,),dtype=tf.int32,name='input_ids')\ntoken_type_ids=layers.Input(shape=(max_length,),dtype=tf.int32,name='token_type_ids')\nattention_mask=layers.Input(shape=(max_length,),dtype=tf.int32,name='attention_mask')","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:19:42.155714Z","iopub.execute_input":"2022-07-16T12:19:42.156598Z","iopub.status.idle":"2022-07-16T12:19:42.17298Z","shell.execute_reply.started":"2022-07-16T12:19:42.156551Z","shell.execute_reply":"2022-07-16T12:19:42.171957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding=bert(input_ids,token_type_ids,attention_mask)[1]","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:32:19.379286Z","iopub.execute_input":"2022-07-16T12:32:19.379665Z","iopub.status.idle":"2022-07-16T12:32:21.346923Z","shell.execute_reply.started":"2022-07-16T12:32:19.379624Z","shell.execute_reply":"2022-07-16T12:32:21.345833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:32:22.712828Z","iopub.execute_input":"2022-07-16T12:32:22.713612Z","iopub.status.idle":"2022-07-16T12:32:22.720134Z","shell.execute_reply.started":"2022-07-16T12:32:22.713582Z","shell.execute_reply":"2022-07-16T12:32:22.7192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_ids","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:32:25.4429Z","iopub.execute_input":"2022-07-16T12:32:25.443376Z","iopub.status.idle":"2022-07-16T12:32:25.451216Z","shell.execute_reply.started":"2022-07-16T12:32:25.443319Z","shell.execute_reply":"2022-07-16T12:32:25.450138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out=layers.Dropout(0.15)(embedding)\nout=layers.Dense(399,activation='relu')(out)\n\nout=layers.Dropout(0.15)(out)\nout=layers.Dense(99,activation='relu')(out)\nout=layers.Dense(17,activation='relu')(out)\ny=layers.Dense(3,activation='softmax')(out)\n\nnn=Model(inputs=[input_ids,token_type_ids,attention_mask],outputs=y)\nnn.layers[3].trainable=True\noptimizer=optimizers.Adam(learning_rate = 2e-5)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:32:47.979291Z","iopub.execute_input":"2022-07-16T12:32:47.980241Z","iopub.status.idle":"2022-07-16T12:32:48.0401Z","shell.execute_reply.started":"2022-07-16T12:32:47.980209Z","shell.execute_reply":"2022-07-16T12:32:48.03907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:32:51.558364Z","iopub.execute_input":"2022-07-16T12:32:51.561017Z","iopub.status.idle":"2022-07-16T12:32:51.605167Z","shell.execute_reply.started":"2022-07-16T12:32:51.560974Z","shell.execute_reply":"2022-07-16T12:32:51.604268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=train_df['label']\ny_train","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:39:55.462454Z","iopub.execute_input":"2022-07-16T11:39:55.462741Z","iopub.status.idle":"2022-07-16T11:39:55.475297Z","shell.execute_reply.started":"2022-07-16T11:39:55.462711Z","shell.execute_reply":"2022-07-16T11:39:55.474146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T11:40:12.435017Z","iopub.execute_input":"2022-07-16T11:40:12.435365Z","iopub.status.idle":"2022-07-16T11:40:12.443747Z","shell.execute_reply.started":"2022-07-16T11:40:12.435331Z","shell.execute_reply":"2022-07-16T11:40:12.442553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"utils.plot_model(nn)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:32:55.7476Z","iopub.execute_input":"2022-07-16T12:32:55.748251Z","iopub.status.idle":"2022-07-16T12:32:56.021287Z","shell.execute_reply.started":"2022-07-16T12:32:55.748198Z","shell.execute_reply":"2022-07-16T12:32:56.020154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn.compile(optimizer=optimizer,loss='sparse_categorical_crossentropy',metrics=['accuracy'])\nnn.fit(\n\n    x={'input_ids':x_train['input_ids'],'token_type_ids':x_train['token_type_ids'],'attention_mask':x_train['attention_mask']},\n    y=y_train,\n    validation_split=0.25,\n    epochs=5,\n    batch_size=45\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:33:09.478069Z","iopub.execute_input":"2022-07-16T12:33:09.478396Z","iopub.status.idle":"2022-07-16T12:42:00.570957Z","shell.execute_reply.started":"2022-07-16T12:33:09.478363Z","shell.execute_reply":"2022-07-16T12:42:00.569816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Working with test data","metadata":{}},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:27:40.349372Z","iopub.execute_input":"2022-07-16T12:27:40.349666Z","iopub.status.idle":"2022-07-16T12:27:40.368049Z","shell.execute_reply.started":"2022-07-16T12:27:40.349637Z","shell.execute_reply":"2022-07-16T12:27:40.367162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df=test_df.drop(columns=['id','lang_abv','language'],axis=1)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:27:42.829287Z","iopub.execute_input":"2022-07-16T12:27:42.829639Z","iopub.status.idle":"2022-07-16T12:27:42.844493Z","shell.execute_reply.started":"2022-07-16T12:27:42.829612Z","shell.execute_reply":"2022-07-16T12:27:42.843501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test=tokenizer(\n\n    text=test_df[['premise','hypothesis']].values.tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_token_type_ids=True,\n    return_attention_mask=True,\n    verbose=True\n\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:42:06.438201Z","iopub.execute_input":"2022-07-16T12:42:06.438945Z","iopub.status.idle":"2022-07-16T12:42:13.883388Z","shell.execute_reply.started":"2022-07-16T12:42:06.438914Z","shell.execute_reply":"2022-07-16T12:42:13.88239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:42:48.227593Z","iopub.execute_input":"2022-07-16T12:42:48.228397Z","iopub.status.idle":"2022-07-16T12:42:48.24034Z","shell.execute_reply.started":"2022-07-16T12:42:48.228363Z","shell.execute_reply":"2022-07-16T12:42:48.23918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted=nn.predict({'input_ids':x_test['input_ids'],'token_type_ids':x_test['token_type_ids'],'attention_mask':x_test['attention_mask']})","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:42:51.518186Z","iopub.execute_input":"2022-07-16T12:42:51.519491Z","iopub.status.idle":"2022-07-16T12:43:15.802518Z","shell.execute_reply.started":"2022-07-16T12:42:51.519439Z","shell.execute_reply":"2022-07-16T12:43:15.801319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:43:18.362968Z","iopub.execute_input":"2022-07-16T12:43:18.363632Z","iopub.status.idle":"2022-07-16T12:43:18.373838Z","shell.execute_reply.started":"2022-07-16T12:43:18.363586Z","shell.execute_reply":"2022-07-16T12:43:18.37286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=np.argmax(predicted,axis=1)\ny_pred","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:43:22.789427Z","iopub.execute_input":"2022-07-16T12:43:22.789743Z","iopub.status.idle":"2022-07-16T12:43:22.797927Z","shell.execute_reply.started":"2022-07-16T12:43:22.789703Z","shell.execute_reply":"2022-07-16T12:43:22.796887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:43:54.447541Z","iopub.execute_input":"2022-07-16T12:43:54.447814Z","iopub.status.idle":"2022-07-16T12:43:54.464461Z","shell.execute_reply.started":"2022-07-16T12:43:54.447786Z","shell.execute_reply":"2022-07-16T12:43:54.463546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub=pd.DataFrame({'id':sub_df['id'],'prediction':y_pred})","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:44:21.528926Z","iopub.execute_input":"2022-07-16T12:44:21.529493Z","iopub.status.idle":"2022-07-16T12:44:21.534988Z","shell.execute_reply.started":"2022-07-16T12:44:21.52946Z","shell.execute_reply":"2022-07-16T12:44:21.534146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:44:26.207973Z","iopub.execute_input":"2022-07-16T12:44:26.209071Z","iopub.status.idle":"2022-07-16T12:44:26.223165Z","shell.execute_reply.started":"2022-07-16T12:44:26.209038Z","shell.execute_reply":"2022-07-16T12:44:26.22208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['prediction'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:44:46.108943Z","iopub.execute_input":"2022-07-16T12:44:46.109484Z","iopub.status.idle":"2022-07-16T12:44:46.125201Z","shell.execute_reply.started":"2022-07-16T12:44:46.10945Z","shell.execute_reply":"2022-07-16T12:44:46.123935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv',index=None)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T12:45:17.43842Z","iopub.execute_input":"2022-07-16T12:45:17.438717Z","iopub.status.idle":"2022-07-16T12:45:17.459275Z","shell.execute_reply.started":"2022-07-16T12:45:17.438672Z","shell.execute_reply":"2022-07-16T12:45:17.458167Z"},"trusted":true},"execution_count":null,"outputs":[]}]}