{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-17T07:48:23.648210Z","iopub.execute_input":"2022-07-17T07:48:23.648952Z","iopub.status.idle":"2022-07-17T07:48:23.660253Z","shell.execute_reply.started":"2022-07-17T07:48:23.648920Z","shell.execute_reply":"2022-07-17T07:48:23.658735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')\ndf_test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:18:23.250974Z","iopub.execute_input":"2022-07-17T07:18:23.251532Z","iopub.status.idle":"2022-07-17T07:18:23.619180Z","shell.execute_reply.started":"2022-07-17T07:18:23.251487Z","shell.execute_reply":"2022-07-17T07:18:23.614610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:18:25.507712Z","iopub.execute_input":"2022-07-17T07:18:25.508229Z","iopub.status.idle":"2022-07-17T07:18:25.540486Z","shell.execute_reply.started":"2022-07-17T07:18:25.508198Z","shell.execute_reply":"2022-07-17T07:18:25.539293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    \n    strategy = tf.distribute.experimental.TPUStrategy\nexcept ValueError:\n    strategy = tf.distribute.get_strategy() \n    print('Number of replicas:', strategy.num_replicas_in_sync) ","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:18:31.573661Z","iopub.execute_input":"2022-07-17T07:18:31.574056Z","iopub.status.idle":"2022-07-17T07:18:36.863471Z","shell.execute_reply.started":"2022-07-17T07:18:31.574025Z","shell.execute_reply":"2022-07-17T07:18:36.862005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\nexcept ValueError:\n    tpu = None\n    gpus = tf.config.experimental.list_logical_devices(\"GPU\")\n    \nif tpu:\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu,) \n    print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nelif len(gpus) > 1:\n    strategy = tf.distribute.MirroredStrategy([gpu.name for gpu in gpus])\n    print('Running on multiple GPUs ', [gpu.name for gpu in gpus])\nelif len(gpus) == 1:\n    strategy = tf.distribute.get_strategy() \n    print('Running on single GPU ', gpus[0].name)\nelse:\n    strategy = tf.distribute.get_strategy() \n    print('Running on CPU')\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:19:37.719278Z","iopub.execute_input":"2022-07-17T07:19:37.720040Z","iopub.status.idle":"2022-07-17T07:19:40.513193Z","shell.execute_reply.started":"2022-07-17T07:19:37.720009Z","shell.execute_reply":"2022-07-17T07:19:40.505017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import TFAutoModel, AutoTokenizer ,BertTokenizer ,TFBertModel","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:20:03.095053Z","iopub.execute_input":"2022-07-17T07:20:03.095501Z","iopub.status.idle":"2022-07-17T07:20:06.433111Z","shell.execute_reply.started":"2022-07-17T07:20:03.095450Z","shell.execute_reply":"2022-07-17T07:20:06.431745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = df_train[['premise', 'hypothesis']].values.tolist()\ntest_data = df_test[['premise', 'hypothesis']].values.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:20:40.180792Z","iopub.execute_input":"2022-07-17T07:20:40.181278Z","iopub.status.idle":"2022-07-17T07:20:40.200290Z","shell.execute_reply.started":"2022-07-17T07:20:40.181246Z","shell.execute_reply":"2022-07-17T07:20:40.198952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased')\nbert = TFBertModel.from_pretrained(\"bert-base-multilingual-uncased\")","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:20:42.819790Z","iopub.execute_input":"2022-07-17T07:20:42.820947Z","iopub.status.idle":"2022-07-17T07:22:21.920984Z","shell.execute_reply.started":"2022-07-17T07:20:42.820888Z","shell.execute_reply":"2022-07-17T07:22:21.919574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_length=90\nx_train=tokenizer(\n      \n    text=df_train[['premise','hypothesis']].values.tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_token_type_ids=True,\n    return_attention_mask=True,\n    verbose=True\n    \n    )","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:25:11.902640Z","iopub.execute_input":"2022-07-17T07:25:11.903142Z","iopub.status.idle":"2022-07-17T07:25:30.586329Z","shell.execute_reply.started":"2022-07-17T07:25:11.903072Z","shell.execute_reply":"2022-07-17T07:25:30.584927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import Model,layers,optimizers,callbacks,losses,metrics,utils\ninput_ids=layers.Input(shape=(max_length,),dtype=tf.int32,name='input_ids')\ntoken_type_ids=layers.Input(shape=(max_length,),dtype=tf.int32,name='token_type_ids')\nattention_mask=layers.Input(shape=(max_length,),dtype=tf.int32,name='attention_mask')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:25:54.616674Z","iopub.execute_input":"2022-07-17T07:25:54.617242Z","iopub.status.idle":"2022-07-17T07:25:54.642550Z","shell.execute_reply.started":"2022-07-17T07:25:54.617197Z","shell.execute_reply":"2022-07-17T07:25:54.641309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding=bert(input_ids,token_type_ids,attention_mask)[1]\nout=layers.Dropout(0.10)(embedding)\nout=layers.Dense(128,activation='relu')(out)\n\nout=layers.Dropout(0.10)(out)\nout=layers.Dense(64,activation='relu')(out)\nout=layers.Dense(32,activation='relu')(out)\ny=layers.Dense(3,activation='softmax')(out)\n\nnn=Model(inputs=[input_ids,token_type_ids,attention_mask],outputs=y)\nnn.layers[3].trainable=True\noptimizer=optimizers.Adam(learning_rate = 0.0001)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:28:26.149969Z","iopub.execute_input":"2022-07-17T07:28:26.150492Z","iopub.status.idle":"2022-07-17T07:28:27.943544Z","shell.execute_reply.started":"2022-07-17T07:28:26.150449Z","shell.execute_reply":"2022-07-17T07:28:27.942281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:28:37.291789Z","iopub.execute_input":"2022-07-17T07:28:37.292258Z","iopub.status.idle":"2022-07-17T07:28:37.323150Z","shell.execute_reply.started":"2022-07-17T07:28:37.292210Z","shell.execute_reply":"2022-07-17T07:28:37.322048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=df_train['label']\ny_train","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:29:34.434128Z","iopub.execute_input":"2022-07-17T07:29:34.435535Z","iopub.status.idle":"2022-07-17T07:29:34.460180Z","shell.execute_reply.started":"2022-07-17T07:29:34.435416Z","shell.execute_reply":"2022-07-17T07:29:34.456824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn.compile(optimizer=optimizer,loss='sparse_categorical_crossentropy',metrics=['accuracy'])\nnn.fit(\n\n    x={'input_ids':x_train['input_ids'],'token_type_ids':x_train['token_type_ids'],'attention_mask':x_train['attention_mask']},\n    y=y_train,\n    validation_split=0.10,\n    epochs=5,\n    batch_size=32\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:30:23.490917Z","iopub.execute_input":"2022-07-17T07:30:23.492182Z","iopub.status.idle":"2022-07-17T07:39:20.028558Z","shell.execute_reply.started":"2022-07-17T07:30:23.492132Z","shell.execute_reply":"2022-07-17T07:39:20.027323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df=df_test.drop(columns=['id','lang_abv','language'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:41:07.474762Z","iopub.execute_input":"2022-07-17T07:41:07.475504Z","iopub.status.idle":"2022-07-17T07:41:07.483804Z","shell.execute_reply.started":"2022-07-17T07:41:07.475446Z","shell.execute_reply":"2022-07-17T07:41:07.482381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test=tokenizer(\n\n    text=df_test[['premise','hypothesis']].values.tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_token_type_ids=True,\n    return_attention_mask=True,\n    verbose=True\n\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:41:40.137996Z","iopub.execute_input":"2022-07-17T07:41:40.138418Z","iopub.status.idle":"2022-07-17T07:41:49.146444Z","shell.execute_reply.started":"2022-07-17T07:41:40.138386Z","shell.execute_reply":"2022-07-17T07:41:49.145133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted=nn.predict({'input_ids':x_test['input_ids'],'token_type_ids':x_test['token_type_ids'],'attention_mask':x_test['attention_mask']})","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:42:27.480269Z","iopub.execute_input":"2022-07-17T07:42:27.480713Z","iopub.status.idle":"2022-07-17T07:42:52.297652Z","shell.execute_reply.started":"2022-07-17T07:42:27.480681Z","shell.execute_reply":"2022-07-17T07:42:52.296403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:43:13.808230Z","iopub.execute_input":"2022-07-17T07:43:13.808818Z","iopub.status.idle":"2022-07-17T07:43:13.829461Z","shell.execute_reply.started":"2022-07-17T07:43:13.808756Z","shell.execute_reply":"2022-07-17T07:43:13.828220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=np.argmax(predicted,axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:43:33.348303Z","iopub.execute_input":"2022-07-17T07:43:33.348695Z","iopub.status.idle":"2022-07-17T07:43:33.355366Z","shell.execute_reply.started":"2022-07-17T07:43:33.348648Z","shell.execute_reply":"2022-07-17T07:43:33.353627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.read_csv('../input/contradictory-my-dear-watson/sample_submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:51:17.475012Z","iopub.execute_input":"2022-07-17T07:51:17.475523Z","iopub.status.idle":"2022-07-17T07:51:17.492435Z","shell.execute_reply.started":"2022-07-17T07:51:17.475491Z","shell.execute_reply":"2022-07-17T07:51:17.491238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.DataFrame({'id':submission['id'],'prediction':y_pred})","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:51:52.949298Z","iopub.execute_input":"2022-07-17T07:51:52.949714Z","iopub.status.idle":"2022-07-17T07:51:52.959302Z","shell.execute_reply.started":"2022-07-17T07:51:52.949683Z","shell.execute_reply":"2022-07-17T07:51:52.957822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=None)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T07:52:27.100886Z","iopub.execute_input":"2022-07-17T07:52:27.101291Z","iopub.status.idle":"2022-07-17T07:52:27.121715Z","shell.execute_reply.started":"2022-07-17T07:52:27.101260Z","shell.execute_reply":"2022-07-17T07:52:27.120392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}