{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport torch\nfrom torch import nn\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nfrom tqdm import tqdm\nimport gc\nimport datasets as datas\nfrom torch.utils.data import DataLoader\nimport datasets as datas\nfrom dataclasses import dataclass\nfrom typing import *\nfrom transformers import AutoTokenizer, AutoModelForMaskedLM, AutoModel, AutoConfig\nfrom transformers import XLMRobertaModel, AutoModelForSequenceClassification, pipeline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-14T10:25:58.530316Z","iopub.execute_input":"2023-08-14T10:25:58.530831Z","iopub.status.idle":"2023-08-14T10:25:58.539529Z","shell.execute_reply.started":"2023-08-14T10:25:58.530799Z","shell.execute_reply":"2023-08-14T10:25:58.538450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device='cuda' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:26:01.068497Z","iopub.execute_input":"2023-08-14T10:26:01.068866Z","iopub.status.idle":"2023-08-14T10:26:01.106067Z","shell.execute_reply.started":"2023-08-14T10:26:01.068836Z","shell.execute_reply":"2023-08-14T10:26:01.105135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@dataclass\nclass SequenceRegressorOutput:\n    logits: torch.FloatTensor\n    loss: Optional[torch.Tensor]\nclass RobertaForJigsawMLSentimentAnalysis(nn.Module):\n    def __init__(self,config):\n        super(RobertaForJigsawMLSentimentAnalysis,self).__init__()\n        self.base_model = XLMRobertaModel(config,add_pooling_layer=False)\n        self.linear = nn.Linear(config.hidden_size,1)\n    def forward(self,\n        input_ids: Optional[torch.Tensor] = None,\n        attention_mask: Optional[torch.Tensor] = None,\n        labels: Optional[torch.Tensor] = None,#(Batch,1)\n        **kwargs):\n        loss = None\n        outputs = self.base_model(input_ids=input_ids,attention_mask=attention_mask)\n        lhs = outputs.last_hidden_state[:,-1]#(Batch, hidden)\n        logits = self.linear(lhs)#(Batch, 1)\n        logits_scaled = torch.nn.functional.sigmoid(logits)\n        weight=3.25#give more weight to toxic outputs\n        if labels is not None:\n            labels = labels.view(logits_scaled.shape)\n            loss = nn.functional.binary_cross_entropy(logits_scaled,labels)*(1.0+(weight-1.0)*(labels*(1.0-logits_scaled))).mean().to(device)\n        return SequenceRegressorOutput(logits=logits, loss=loss)","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:29:22.984002Z","iopub.execute_input":"2023-08-14T10:29:22.985203Z","iopub.status.idle":"2023-08-14T10:29:22.998607Z","shell.execute_reply.started":"2023-08-14T10:29:22.985163Z","shell.execute_reply":"2023-08-14T10:29:22.997470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = AutoConfig.from_pretrained(\"/kaggle/input/xlmrobertajigsawmlcombinedpytorch/config.json\")","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:29:42.390784Z","iopub.execute_input":"2023-08-14T10:29:42.391180Z","iopub.status.idle":"2023-08-14T10:29:42.402954Z","shell.execute_reply.started":"2023-08-14T10:29:42.391150Z","shell.execute_reply":"2023-08-14T10:29:42.402022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer=AutoTokenizer.from_pretrained(\"/kaggle/input/xlmrobertajigsawmlcombinedpytorch\")","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:29:52.413200Z","iopub.execute_input":"2023-08-14T10:29:52.413567Z","iopub.status.idle":"2023-08-14T10:29:53.203795Z","shell.execute_reply.started":"2023-08-14T10:29:52.413537Z","shell.execute_reply":"2023-08-14T10:29:53.202775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:30:00.849302Z","iopub.execute_input":"2023-08-14T10:30:00.849670Z","iopub.status.idle":"2023-08-14T10:30:02.118724Z","shell.execute_reply.started":"2023-08-14T10:30:00.849642Z","shell.execute_reply":"2023-08-14T10:30:02.117750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset=datas.Dataset.from_pandas(test,preserve_index=False)\nprocessed_dataset=dataset.map(\n    lambda x: tokenizer(x['content'],padding='max_length',truncation=True,max_length=200),\n    remove_columns=['id', 'content', 'lang'],\n    batched=True\n)\ndata_loader=DataLoader(\n    processed_dataset,\n    batch_size=32,\n    num_workers=2,\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:30:42.146301Z","iopub.execute_input":"2023-08-14T10:30:42.146669Z","iopub.status.idle":"2023-08-14T10:31:13.145679Z","shell.execute_reply.started":"2023-08-14T10:30:42.146639Z","shell.execute_reply":"2023-08-14T10:31:13.144770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=RobertaForJigsawMLSentimentAnalysis(config)\nmodel.load_state_dict(torch.load('/kaggle/input/xlmrobertajigsawmlcombinedpytorch/xlm-roberta-sentiment-jigsawml-combined.pt'),strict=False)\nmodel.eval()\nmodel=model.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:31:19.963673Z","iopub.execute_input":"2023-08-14T10:31:19.964037Z","iopub.status.idle":"2023-08-14T10:31:43.920405Z","shell.execute_reply.started":"2023-08-14T10:31:19.964007Z","shell.execute_reply":"2023-08-14T10:31:43.919384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def infer(model, data_loader):\n    results=[]\n    tqdm_ = tqdm(data_loader)\n    \n    with torch.no_grad():\n        for i,x in enumerate(tqdm_):\n            outputs=model(\n                **{\n                    k:torch.stack(x[k]).long().transpose(-1,-2).to(device) for k in ('input_ids','attention_mask')\n                },\n            )\n            results.append(outputs.logits)\n            \n    return results","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:31:43.922503Z","iopub.execute_input":"2023-08-14T10:31:43.922908Z","iopub.status.idle":"2023-08-14T10:31:43.929692Z","shell.execute_reply.started":"2023-08-14T10:31:43.922875Z","shell.execute_reply":"2023-08-14T10:31:43.928375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"infered=infer(model,data_loader)","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:31:53.387999Z","iopub.execute_input":"2023-08-14T10:31:53.389054Z","iopub.status.idle":"2023-08-14T10:41:21.806903Z","shell.execute_reply.started":"2023-08-14T10:31:53.389005Z","shell.execute_reply":"2023-08-14T10:41:21.805835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"infered_pt=torch.cat(infered)\ninfered_pt.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:41:21.985588Z","iopub.execute_input":"2023-08-14T10:41:21.986033Z","iopub.status.idle":"2023-08-14T10:41:21.995263Z","shell.execute_reply.started":"2023-08-14T10:41:21.985994Z","shell.execute_reply":"2023-08-14T10:41:21.994265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['toxic']=torch.nn.functional.sigmoid(infered_pt.flatten()).detach().cpu().numpy()","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:44:11.265958Z","iopub.execute_input":"2023-08-14T10:44:11.266666Z","iopub.status.idle":"2023-08-14T10:44:11.275023Z","shell.execute_reply.started":"2023-08-14T10:44:11.266629Z","shell.execute_reply":"2023-08-14T10:44:11.273993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.groupby('lang').describe()['toxic']","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:46:37.198548Z","iopub.execute_input":"2023-08-14T10:46:37.198933Z","iopub.status.idle":"2023-08-14T10:46:37.283180Z","shell.execute_reply.started":"2023-08-14T10:46:37.198901Z","shell.execute_reply":"2023-08-14T10:46:37.282050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[['id','toxic']].to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-08-14T10:47:21.844892Z","iopub.execute_input":"2023-08-14T10:47:21.845338Z","iopub.status.idle":"2023-08-14T10:47:22.039234Z","shell.execute_reply.started":"2023-08-14T10:47:21.845302Z","shell.execute_reply":"2023-08-14T10:47:22.038232Z"},"trusted":true},"execution_count":null,"outputs":[]}]}