{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\nos.environ[\"WANDB_API_KEY\"] = \"0\" ## to silence warning","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.034554,"end_time":"2021-05-30T16:55:00.370180","exception":false,"start_time":"2021-05-30T16:55:00.335626","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:44:38.845947Z","iopub.execute_input":"2022-08-12T16:44:38.846920Z","iopub.status.idle":"2022-08-12T16:44:38.882458Z","shell.execute_reply.started":"2022-08-12T16:44:38.846780Z","shell.execute_reply":"2022-08-12T16:44:38.881556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError:\n    strategy = tf.distribute.get_strategy() # for CPU and single GPU\nprint('Number of replicas:', strategy.num_replicas_in_sync)","metadata":{"papermill":{"duration":11.605499,"end_time":"2021-05-30T16:55:11.990651","exception":false,"start_time":"2021-05-30T16:55:00.385152","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:44:38.884244Z","iopub.execute_input":"2022-08-12T16:44:38.884515Z","iopub.status.idle":"2022-08-12T16:44:51.720546Z","shell.execute_reply.started":"2022-08-12T16:44:38.884475Z","shell.execute_reply":"2022-08-12T16:44:51.719885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv('../input/contradictory-my-dear-watson/train.csv')\ntest=pd.read_csv('../input/contradictory-my-dear-watson/test.csv')\nprint(train.shape)\nprint(test.shape)","metadata":{"papermill":{"duration":0.238893,"end_time":"2021-05-30T16:55:12.243914","exception":false,"start_time":"2021-05-30T16:55:12.005021","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:44:51.721871Z","iopub.execute_input":"2022-08-12T16:44:51.723379Z","iopub.status.idle":"2022-08-12T16:44:51.987493Z","shell.execute_reply.started":"2022-08-12T16:44:51.723330Z","shell.execute_reply":"2022-08-12T16:44:51.986430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.tail(10)","metadata":{"papermill":{"duration":0.050526,"end_time":"2021-05-30T16:55:12.308984","exception":false,"start_time":"2021-05-30T16:55:12.258458","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:44:51.989032Z","iopub.execute_input":"2022-08-12T16:44:51.991335Z","iopub.status.idle":"2022-08-12T16:44:52.016786Z","shell.execute_reply.started":"2022-08-12T16:44:51.991288Z","shell.execute_reply":"2022-08-12T16:44:52.015862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.tail(10)","metadata":{"papermill":{"duration":0.035477,"end_time":"2021-05-30T16:55:12.360119","exception":false,"start_time":"2021-05-30T16:55:12.324642","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:44:52.019166Z","iopub.execute_input":"2022-08-12T16:44:52.019483Z","iopub.status.idle":"2022-08-12T16:44:52.035172Z","shell.execute_reply.started":"2022-08-12T16:44:52.019448Z","shell.execute_reply":"2022-08-12T16:44:52.034038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.premise.values[1],train.hypothesis.values[1]","metadata":{"papermill":{"duration":0.028931,"end_time":"2021-05-30T16:55:12.405974","exception":false,"start_time":"2021-05-30T16:55:12.377043","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:44:52.036604Z","iopub.execute_input":"2022-08-12T16:44:52.037382Z","iopub.status.idle":"2022-08-12T16:44:52.050139Z","shell.execute_reply.started":"2022-08-12T16:44:52.037340Z","shell.execute_reply":"2022-08-12T16:44:52.049420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels, frequencies = np.unique(train.language.values, return_counts = True)\n\nplt.figure(figsize = (10,10))\nplt.pie(frequencies,labels = labels, autopct = '%1.1f%%')\nplt.show()","metadata":{"papermill":{"duration":0.304059,"end_time":"2021-05-30T16:55:12.727075","exception":false,"start_time":"2021-05-30T16:55:12.423016","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:44:52.051758Z","iopub.execute_input":"2022-08-12T16:44:52.052293Z","iopub.status.idle":"2022-08-12T16:44:52.341857Z","shell.execute_reply.started":"2022-08-12T16:44:52.052258Z","shell.execute_reply":"2022-08-12T16:44:52.340902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.label.value_counts()","metadata":{"papermill":{"duration":0.030289,"end_time":"2021-05-30T16:55:12.775611","exception":false,"start_time":"2021-05-30T16:55:12.745322","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:44:52.343186Z","iopub.execute_input":"2022-08-12T16:44:52.343499Z","iopub.status.idle":"2022-08-12T16:44:52.356250Z","shell.execute_reply.started":"2022-08-12T16:44:52.343459Z","shell.execute_reply":"2022-08-12T16:44:52.355202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install transformers\nfrom transformers import TFAutoModel,AutoTokenizer\nimport tensorflow as tf\n#!pip install sentencepiece","metadata":{"papermill":{"duration":2.187679,"end_time":"2021-05-30T16:55:14.981636","exception":false,"start_time":"2021-05-30T16:55:12.793957","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:44:52.357858Z","iopub.execute_input":"2022-08-12T16:44:52.358765Z","iopub.status.idle":"2022-08-12T16:44:54.640569Z","shell.execute_reply.started":"2022-08-12T16:44:52.358728Z","shell.execute_reply":"2022-08-12T16:44:54.639496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer=AutoTokenizer.from_pretrained('joeddav/xlm-roberta-large-xnli')\n\ntrain_enc=tokenizer.batch_encode_plus(train[['premise','hypothesis']].values.tolist(),padding='max_length',max_length=100,truncation=True,return_attention_mask=True)\ntest_enc=tokenizer.batch_encode_plus(test[['premise','hypothesis']].values.tolist(),padding='max_length',max_length=100,truncation=True,return_attention_mask=True)\ntrain_tf1=tf.convert_to_tensor(train_enc['input_ids'],dtype=tf.int32)\ntrain_tf2=tf.convert_to_tensor(train_enc['attention_mask'],dtype=tf.int32)\ntrain_input={'input_word_ids':train_tf1,'input_mask':train_tf2}\ntest_tf1=tf.convert_to_tensor(test_enc['input_ids'],dtype=tf.int32)\ntest_tf2=tf.convert_to_tensor(test_enc['attention_mask'],dtype=tf.int32)\ntest_input={'input_word_ids':test_tf1,'input_mask':test_tf2}","metadata":{"papermill":{"duration":8.869253,"end_time":"2021-05-30T16:55:23.870278","exception":false,"start_time":"2021-05-30T16:55:15.001025","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:44:54.642064Z","iopub.execute_input":"2022-08-12T16:44:54.642368Z","iopub.status.idle":"2022-08-12T16:45:02.660128Z","shell.execute_reply.started":"2022-08-12T16:44:54.642335Z","shell.execute_reply":"2022-08-12T16:45:02.659259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_enc[100]","metadata":{"papermill":{"duration":0.032588,"end_time":"2021-05-30T16:55:23.924357","exception":false,"start_time":"2021-05-30T16:55:23.891769","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:45:02.663069Z","iopub.execute_input":"2022-08-12T16:45:02.663484Z","iopub.status.idle":"2022-08-12T16:45:02.673248Z","shell.execute_reply.started":"2022-08-12T16:45:02.663435Z","shell.execute_reply":"2022-08-12T16:45:02.672309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    input_ids = tf.keras.Input(shape = (100,), dtype = tf.int32,name='input_word_ids') \n    input_mask=tf.keras.Input(shape=(100,),dtype=tf.int32,name='input_mask')    \n    roberta = TFAutoModel.from_pretrained('joeddav/xlm-roberta-large-xnli')\n    roberta = roberta([input_ids,input_mask])[0]\n    output = tf.keras.layers.GlobalAveragePooling1D()(roberta)\n    output = tf.keras.layers.Dense(3, activation = 'softmax')(output)\n    model = tf.keras.Model(inputs = [input_ids,input_mask], outputs = output)\n    model.compile(optimizer = tf.keras.optimizers.Adam(lr = 1e-5), \n                  loss = 'sparse_categorical_crossentropy', \n                  metrics = ['accuracy']) \n    model.summary()","metadata":{"papermill":{"duration":157.675248,"end_time":"2021-05-30T16:58:01.621225","exception":false,"start_time":"2021-05-30T16:55:23.945977","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:45:02.674696Z","iopub.execute_input":"2022-08-12T16:45:02.674977Z","iopub.status.idle":"2022-08-12T16:48:13.439549Z","shell.execute_reply.started":"2022-08-12T16:45:02.674933Z","shell.execute_reply":"2022-08-12T16:48:13.438402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stop = tf.keras.callbacks.EarlyStopping(patience=2,restore_best_weights=True)\nmodel.fit(train_input,train.label,validation_split = 0.2,epochs=20,batch_size=16*strategy.num_replicas_in_sync,callbacks=[early_stop],verbose=1)","metadata":{"papermill":{"duration":371.008976,"end_time":"2021-05-30T17:04:12.653886","exception":false,"start_time":"2021-05-30T16:58:01.644910","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-12T16:48:13.440884Z","iopub.execute_input":"2022-08-12T16:48:13.441166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred=[np.argmax(i) for i in model.predict(test_input)]\npd.DataFrame(pred).value_counts()","metadata":{"papermill":{"duration":25.739925,"end_time":"2021-05-30T17:04:38.489621","exception":false,"start_time":"2021-05-30T17:04:12.749696","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({'id':test.id,\n              'prediction':pred}).to_csv('submission.csv',index=False)","metadata":{"papermill":{"duration":0.127323,"end_time":"2021-05-30T17:04:38.714133","exception":false,"start_time":"2021-05-30T17:04:38.586810","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}