{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport transformers \nfrom transformers import AutoTokenizer,AutoModel,AdamW,AutoModelForSequenceClassification\nimport torch \nimport torch.nn as nn\nimport numpy as np\nfrom sklearn.preprocessing import LabelEncoder\nfrom torch.utils.data import DataLoader","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:34.321816Z","iopub.execute_input":"2022-06-27T21:48:34.322288Z","iopub.status.idle":"2022-06-27T21:48:41.466976Z","shell.execute_reply.started":"2022-06-27T21:48:34.322198Z","shell.execute_reply":"2022-06-27T21:48:41.466019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv(\"../input/us-patent-phrase-to-phrase-matching/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:41.469963Z","iopub.execute_input":"2022-06-27T21:48:41.472104Z","iopub.status.idle":"2022-06-27T21:48:41.557528Z","shell.execute_reply.started":"2022-06-27T21:48:41.472059Z","shell.execute_reply":"2022-06-27T21:48:41.556570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:41.559090Z","iopub.execute_input":"2022-06-27T21:48:41.559446Z","iopub.status.idle":"2022-06-27T21:48:41.582797Z","shell.execute_reply.started":"2022-06-27T21:48:41.559410Z","shell.execute_reply":"2022-06-27T21:48:41.581901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le=LabelEncoder()\ntrain.score=le.fit_transform(train.score)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:41.586156Z","iopub.execute_input":"2022-06-27T21:48:41.587862Z","iopub.status.idle":"2022-06-27T21:48:41.600144Z","shell.execute_reply.started":"2022-06-27T21:48:41.587833Z","shell.execute_reply":"2022-06-27T21:48:41.599028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:41.601716Z","iopub.execute_input":"2022-06-27T21:48:41.602142Z","iopub.status.idle":"2022-06-27T21:48:41.614676Z","shell.execute_reply.started":"2022-06-27T21:48:41.602107Z","shell.execute_reply":"2022-06-27T21:48:41.613511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer=AutoTokenizer.from_pretrained(\"bert-base-uncased\")","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:41.616976Z","iopub.execute_input":"2022-06-27T21:48:41.617305Z","iopub.status.idle":"2022-06-27T21:48:46.681317Z","shell.execute_reply.started":"2022-06-27T21:48:41.617272Z","shell.execute_reply":"2022-06-27T21:48:46.680365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"class DatasetT:\n    def __init__(self,df,tokenizer,max_length):\n        self.df=df\n        self.tokenizer=tokenizer\n        self.max_length=max_length\n        #self.anchor=df.anchor\n        self.target=df[\"target\"].values\n        self.score=df[\"score\"].values\n    \n    def __len__(self):\n        return len(self.target)\n    \n    def __getitem__(self,idx):\n        \n        t=self.target[idx]\n        #l=self.score[idx]\n        \n        inputs=self.tokenizer(t,padding=True,truncation=True,max_length=self.max_length)\n        \n        #for k,v in inputs.items():\n        #    inputs[k]=torch.tensor(v,dtype=torch.long)       \n        \n        #label=torch.tensor(self.score[idx],dtype=torch.float)\n        \n        return {\n            \"ids\": torch.tensor(inputs[\"input_ids\"],dtype=torch.long),\n            \"mask\": torch.tensor(inputs[\"attention_mask\"],dtype=torch.long),\n            \"token_type\": torch.tensor(inputs[\"token_type_ids\"],dtype=torch.long),\n            \"labels\":torch.tensor(self.score[idx],dtype=torch.float)\n            \n        }\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:46.682695Z","iopub.execute_input":"2022-06-27T21:48:46.683152Z","iopub.status.idle":"2022-06-27T21:48:46.691512Z","shell.execute_reply.started":"2022-06-27T21:48:46.683114Z","shell.execute_reply":"2022-06-27T21:48:46.690421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DatasetT:\n    def __init__(self,df,max_length,tokenizer):\n        self.df=df\n        self.text=df[\"target\"].values\n        self.target=df[\"score\"].values\n        self.tokenizer=tokenizer\n        self.max_length=max_length\n    \n    def __len__(self):\n        return len(self.text)\n\n    def __getitem__(self,idx):\n        texts=self.text[idx]\n        \n        inputs=self.tokenizer.encode_plus(texts,padding=\"max_length\",truncation=True,max_length=self.max_length)\n        \n        target=self.target[idx]\n        \n        return {\n            \"input_ids\": torch.tensor(inputs[\"input_ids\"],dtype=torch.long),\n            \"attention_mask\": torch.tensor(inputs[\"attention_mask\"],dtype=torch.long),\n            \"token_type_ids\": torch.tensor(inputs[\"token_type_ids\"],dtype=torch.long),\n            \"labels\":torch.tensor(target)\n        }\n        ","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:46.693291Z","iopub.execute_input":"2022-06-27T21:48:46.693961Z","iopub.status.idle":"2022-06-27T21:48:46.704002Z","shell.execute_reply.started":"2022-06-27T21:48:46.693924Z","shell.execute_reply":"2022-06-27T21:48:46.703075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d=DatasetT(train,tokenizer=tokenizer,max_length=10)\n\nloader=DataLoader(d,batch_size=10,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:46.705436Z","iopub.execute_input":"2022-06-27T21:48:46.705879Z","iopub.status.idle":"2022-06-27T21:48:46.717115Z","shell.execute_reply.started":"2022-06-27T21:48:46.705841Z","shell.execute_reply":"2022-06-27T21:48:46.716080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=AutoModelForSequenceClassification.from_pretrained(\"bert-base-uncased\",num_labels=5)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:46.721942Z","iopub.execute_input":"2022-06-27T21:48:46.722712Z","iopub.status.idle":"2022-06-27T21:48:59.656261Z","shell.execute_reply.started":"2022-06-27T21:48:46.722689Z","shell.execute_reply":"2022-06-27T21:48:59.655326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer=AdamW(model.parameters())","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:59.660198Z","iopub.execute_input":"2022-06-27T21:48:59.662801Z","iopub.status.idle":"2022-06-27T21:48:59.680907Z","shell.execute_reply.started":"2022-06-27T21:48:59.662761Z","shell.execute_reply":"2022-06-27T21:48:59.680040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch in loader:\n    break\n{k: v.shape for k, v in batch.items()}","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:48:59.686029Z","iopub.execute_input":"2022-06-27T21:48:59.689687Z","iopub.status.idle":"2022-06-27T21:49:01.154690Z","shell.execute_reply.started":"2022-06-27T21:48:59.689649Z","shell.execute_reply":"2022-06-27T21:49:01.153778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs = model(**batch)\nprint(outputs.loss, outputs.logits.shape)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:49:01.157910Z","iopub.execute_input":"2022-06-27T21:49:01.158190Z","iopub.status.idle":"2022-06-27T21:49:01.479840Z","shell.execute_reply.started":"2022-06-27T21:49:01.158164Z","shell.execute_reply":"2022-06-27T21:49:01.478712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import get_scheduler\n\nnum_epochs = 3\nnum_training_steps = num_epochs * len(loader)\nlr_scheduler = get_scheduler(\n    \"linear\",\n    optimizer=optimizer,\n    num_warmup_steps=0,\n    num_training_steps=num_training_steps,\n)\nprint(num_training_steps)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:49:01.481468Z","iopub.execute_input":"2022-06-27T21:49:01.481837Z","iopub.status.idle":"2022-06-27T21:49:01.488303Z","shell.execute_reply.started":"2022-06-27T21:49:01.481800Z","shell.execute_reply":"2022-06-27T21:49:01.486988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\ndevice = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\nmodel.to(device)\ndevice\n","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:49:01.489950Z","iopub.execute_input":"2022-06-27T21:49:01.490416Z","iopub.status.idle":"2022-06-27T21:49:06.410309Z","shell.execute_reply.started":"2022-06-27T21:49:01.490381Z","shell.execute_reply":"2022-06-27T21:49:06.409409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm\n\n#progress_bar = tqdm(range(num_training_steps))\n\"\"\"l=[]\nmodel.train()\nfor epoch in range(num_epochs):\n    for batch in loader:\n        batch = {k: v.to(device) for k, v in batch.items()}\n        outputs = model(**batch)\n        loss = outputs.loss\n        l.append(loss)\n        loss.backward()\n\n        optimizer.step()\n        lr_scheduler.step()\n        optimizer.zero_grad()\n        #progress_bar.update(1)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-06-27T21:49:06.411576Z","iopub.execute_input":"2022-06-27T21:49:06.412787Z","iopub.status.idle":"2022-06-27T21:56:54.126184Z","shell.execute_reply.started":"2022-06-27T21:49:06.412748Z","shell.execute_reply":"2022-06-27T21:56:54.125207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logits = outputs.logits\ntorch.argmax(logits, dim=-1)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T22:04:13.275109Z","iopub.execute_input":"2022-06-27T22:04:13.275574Z","iopub.status.idle":"2022-06-27T22:04:13.287307Z","shell.execute_reply.started":"2022-06-27T22:04:13.275526Z","shell.execute_reply":"2022-06-27T22:04:13.286234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}