{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-04T05:59:19.450522Z","iopub.execute_input":"2022-08-04T05:59:19.451314Z","iopub.status.idle":"2022-08-04T05:59:19.466229Z","shell.execute_reply.started":"2022-08-04T05:59:19.451229Z","shell.execute_reply":"2022-08-04T05:59:19.465130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\ndevice=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-08-04T05:59:19.474226Z","iopub.execute_input":"2022-08-04T05:59:19.474777Z","iopub.status.idle":"2022-08-04T05:59:20.118953Z","shell.execute_reply.started":"2022-08-04T05:59:19.474740Z","shell.execute_reply":"2022-08-04T05:59:20.116418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer\nmodel = AutoModelForSequenceClassification.from_pretrained(\"xlm-roberta-base\", num_labels=3).to(device)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T05:59:24.370392Z","iopub.execute_input":"2022-08-04T05:59:24.371079Z","iopub.status.idle":"2022-08-04T05:59:38.573782Z","shell.execute_reply.started":"2022-08-04T05:59:24.371028Z","shell.execute_reply":"2022-08-04T05:59:38.572264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=pd.read_csv(\"../input/contradictory-my-dear-watson/train.csv\")\ntrain_df=train_df.sample(frac=1.0)\nval_df=train_df.iloc[:2000, :]\ntrain_df=train_df.iloc[2000:, :]\ntrain_df['input_text']=train_df['premise']+' [SEP] '+train_df['hypothesis']\nval_df['input_text']=val_df['premise']+' [SEP] '+val_df['hypothesis']\n","metadata":{"execution":{"iopub.status.busy":"2022-08-04T05:59:38.579257Z","iopub.execute_input":"2022-08-04T05:59:38.579916Z","iopub.status.idle":"2022-08-04T05:59:38.771245Z","shell.execute_reply.started":"2022-08-04T05:59:38.579879Z","shell.execute_reply":"2022-08-04T05:59:38.769794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import Dataset\ntrain_ds=Dataset.from_pandas(train_df)\nval_ds=Dataset.from_pandas(val_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T05:59:42.650899Z","iopub.execute_input":"2022-08-04T05:59:42.652046Z","iopub.status.idle":"2022-08-04T05:59:42.698376Z","shell.execute_reply.started":"2022-08-04T05:59:42.652000Z","shell.execute_reply":"2022-08-04T05:59:42.697240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds","metadata":{"execution":{"iopub.status.busy":"2022-08-04T05:59:43.406751Z","iopub.execute_input":"2022-08-04T05:59:43.407930Z","iopub.status.idle":"2022-08-04T05:59:43.418919Z","shell.execute_reply.started":"2022-08-04T05:59:43.407878Z","shell.execute_reply":"2022-08-04T05:59:43.416916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoTokenizer\n\ntokenizer = AutoTokenizer.from_pretrained(\"xlm-roberta-base\")","metadata":{"execution":{"iopub.status.busy":"2022-08-04T05:59:44.160011Z","iopub.execute_input":"2022-08-04T05:59:44.160442Z","iopub.status.idle":"2022-08-04T05:59:47.686063Z","shell.execute_reply.started":"2022-08-04T05:59:44.160407Z","shell.execute_reply":"2022-08-04T05:59:47.684873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_function(examples):\n    return tokenizer(examples[\"input_text\"], truncation=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T05:59:47.688551Z","iopub.execute_input":"2022-08-04T05:59:47.688990Z","iopub.status.idle":"2022-08-04T05:59:47.696030Z","shell.execute_reply.started":"2022-08-04T05:59:47.688948Z","shell.execute_reply":"2022-08-04T05:59:47.694900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds_encoded = train_ds.map(preprocess_function, batched=True)\nval_ds_encoded = val_ds.map(preprocess_function, batched=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T05:59:56.867450Z","iopub.execute_input":"2022-08-04T05:59:56.867910Z","iopub.status.idle":"2022-08-04T06:00:00.514514Z","shell.execute_reply.started":"2022-08-04T05:59:56.867870Z","shell.execute_reply":"2022-08-04T06:00:00.513381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import DataCollatorWithPadding\n\ndata_collator = DataCollatorWithPadding(tokenizer=tokenizer)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:00:00.516631Z","iopub.execute_input":"2022-08-04T06:00:00.517292Z","iopub.status.idle":"2022-08-04T06:00:00.522461Z","shell.execute_reply.started":"2022-08-04T06:00:00.517255Z","shell.execute_reply":"2022-08-04T06:00:00.521273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, f1_score\ndef compute_metrics(pred):\n    labels=pred.label_ids\n    preds=pred.predictions.argmax(-1)\n    f1=f1_score(labels, preds, average='weighted')\n    ac=accuracy_score(labels, preds)\n    return {\"accuracy\":ac, \"f1\":f1}","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:00:02.434999Z","iopub.execute_input":"2022-08-04T06:00:02.435506Z","iopub.status.idle":"2022-08-04T06:00:02.445280Z","shell.execute_reply.started":"2022-08-04T06:00:02.435470Z","shell.execute_reply":"2022-08-04T06:00:02.443919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ[\"WANDB_DISABLED\"] = \"true\"","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:00:03.467246Z","iopub.execute_input":"2022-08-04T06:00:03.467652Z","iopub.status.idle":"2022-08-04T06:00:03.473614Z","shell.execute_reply.started":"2022-08-04T06:00:03.467617Z","shell.execute_reply":"2022-08-04T06:00:03.472297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_args = TrainingArguments(\n    output_dir=\"./results\",\n    learning_rate=2e-5,\n    per_device_train_batch_size=16,\n    per_device_eval_batch_size=16,\n    num_train_epochs=5,\n    weight_decay=0.01,\n)\n\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    compute_metrics=compute_metrics,\n    train_dataset=train_ds_encoded,\n    eval_dataset=val_ds_encoded,\n    tokenizer=tokenizer,\n    data_collator=data_collator,\n)\n\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:00:13.786985Z","iopub.execute_input":"2022-08-04T06:00:13.787357Z","iopub.status.idle":"2022-08-04T06:12:09.996933Z","shell.execute_reply.started":"2022-08-04T06:00:13.787326Z","shell.execute_reply":"2022-08-04T06:12:09.994539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_output=trainer.predict(val_ds_encoded)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:12:09.999255Z","iopub.execute_input":"2022-08-04T06:12:10.000300Z","iopub.status.idle":"2022-08-04T06:12:16.183314Z","shell.execute_reply.started":"2022-08-04T06:12:10.000257Z","shell.execute_reply":"2022-08-04T06:12:16.182227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_output.metrics","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:12:16.185395Z","iopub.execute_input":"2022-08-04T06:12:16.185884Z","iopub.status.idle":"2022-08-04T06:12:16.195768Z","shell.execute_reply.started":"2022-08-04T06:12:16.185809Z","shell.execute_reply":"2022-08-04T06:12:16.194330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df=pd.read_csv(\"../input/contradictory-my-dear-watson/test.csv\")\ntest_df['input_text']=test_df['premise']+' [SEP] '+test_df['hypothesis']\ntest_ds=Dataset.from_pandas(test_df)\ntest_ds_encoded=test_ds.map(preprocess_function, batched=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:12:16.199234Z","iopub.execute_input":"2022-08-04T06:12:16.200288Z","iopub.status.idle":"2022-08-04T06:12:17.251806Z","shell.execute_reply.started":"2022-08-04T06:12:16.200246Z","shell.execute_reply":"2022-08-04T06:12:17.250877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_output=trainer.predict(test_ds_encoded)\noutputs=test_output.predictions.argmax(-1)\nsub_df=pd.DataFrame({\"id\": test_df[\"id\"], \"prediction\": outputs})\nsub_df.set_index(\"id\", inplace=True)\nsub_df.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:12:17.253207Z","iopub.execute_input":"2022-08-04T06:12:17.253809Z","iopub.status.idle":"2022-08-04T06:12:33.934427Z","shell.execute_reply.started":"2022-08-04T06:12:17.253768Z","shell.execute_reply":"2022-08-04T06:12:33.933501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:12:33.936164Z","iopub.execute_input":"2022-08-04T06:12:33.936543Z","iopub.status.idle":"2022-08-04T06:12:33.951415Z","shell.execute_reply.started":"2022-08-04T06:12:33.936507Z","shell.execute_reply":"2022-08-04T06:12:33.950352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}