{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install transformers -U","metadata":{"id":"l5ceeO9LhWkU","execution":{"iopub.status.busy":"2022-07-09T13:28:02.525831Z","iopub.execute_input":"2022-07-09T13:28:02.526474Z","iopub.status.idle":"2022-07-09T13:28:20.472816Z","shell.execute_reply.started":"2022-07-09T13:28:02.526347Z","shell.execute_reply":"2022-07-09T13:28:20.471683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score\nimport torch\nfrom transformers import TrainingArguments, Trainer\nfrom transformers import BertTokenizer, BertForSequenceClassification","metadata":{"id":"8HNUE5fXIb-W","execution":{"iopub.status.busy":"2022-07-09T13:28:20.475371Z","iopub.execute_input":"2022-07-09T13:28:20.475762Z","iopub.status.idle":"2022-07-09T13:28:27.956926Z","shell.execute_reply.started":"2022-07-09T13:28:20.475716Z","shell.execute_reply":"2022-07-09T13:28:27.955917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/competitions/jigsaw-toxic-comment-classification-challenge/data\ndata = pd.read_csv(\"../input/nlp-getting-started/train.csv\",error_bad_lines=False, engine=\"python\")\ndata.head()","metadata":{"id":"ZLuKyvHJ47xK","outputId":"ae5dbe57-baca-44f9-c947-f341a8c6206a","execution":{"iopub.status.busy":"2022-07-09T13:28:27.958376Z","iopub.execute_input":"2022-07-09T13:28:27.959332Z","iopub.status.idle":"2022-07-09T13:28:28.263709Z","shell.execute_reply.started":"2022-07-09T13:28:27.959295Z","shell.execute_reply":"2022-07-09T13:28:28.262654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:28:28.266571Z","iopub.execute_input":"2022-07-09T13:28:28.266953Z","iopub.status.idle":"2022-07-09T13:28:28.291985Z","shell.execute_reply.started":"2022-07-09T13:28:28.266913Z","shell.execute_reply":"2022-07-09T13:28:28.291168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:28:28.293405Z","iopub.execute_input":"2022-07-09T13:28:28.293741Z","iopub.status.idle":"2022-07-09T13:28:28.305510Z","shell.execute_reply.started":"2022-07-09T13:28:28.293707Z","shell.execute_reply":"2022-07-09T13:28:28.304370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.fillna(\" \")","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:28:28.307009Z","iopub.execute_input":"2022-07-09T13:28:28.307584Z","iopub.status.idle":"2022-07-09T13:28:28.317168Z","shell.execute_reply.started":"2022-07-09T13:28:28.307546Z","shell.execute_reply":"2022-07-09T13:28:28.316285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['target'].value_counts()","metadata":{"id":"5dbrirLG5D0A","outputId":"c64ef77c-2bb8-4243-8b89-66e0e1a3e6cc","execution":{"iopub.status.busy":"2022-07-09T13:28:28.318811Z","iopub.execute_input":"2022-07-09T13:28:28.319457Z","iopub.status.idle":"2022-07-09T13:28:28.327108Z","shell.execute_reply.started":"2022-07-09T13:28:28.319421Z","shell.execute_reply":"2022-07-09T13:28:28.326072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[\"new_col\"]=data[\"text\"]+data[\"keyword\"]+data[\"location\"]\ndata.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:28:28.329485Z","iopub.execute_input":"2022-07-09T13:28:28.330266Z","iopub.status.idle":"2022-07-09T13:28:28.347759Z","shell.execute_reply.started":"2022-07-09T13:28:28.330149Z","shell.execute_reply":"2022-07-09T13:28:28.346907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.isnull().sum(),data['new_col'].shape","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:28:28.349715Z","iopub.execute_input":"2022-07-09T13:28:28.350703Z","iopub.status.idle":"2022-07-09T13:28:28.362918Z","shell.execute_reply.started":"2022-07-09T13:28:28.350667Z","shell.execute_reply":"2022-07-09T13:28:28.361875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nsns.set_theme(style=\"darkgrid\")\nax = sns.distplot(data.new_col.str.len())","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:28:28.367918Z","iopub.execute_input":"2022-07-09T13:28:28.368560Z","iopub.status.idle":"2022-07-09T13:28:28.750874Z","shell.execute_reply.started":"2022-07-09T13:28:28.368534Z","shell.execute_reply":"2022-07-09T13:28:28.749974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\nmodel = BertForSequenceClassification.from_pretrained('bert-base-uncased',num_labels=2)","metadata":{"id":"Z1JlB3eYiV8a","outputId":"4f606d36-bba0-4880-e3a2-c59eb25f9727","scrolled":true,"execution":{"iopub.status.busy":"2022-07-09T13:28:28.752366Z","iopub.execute_input":"2022-07-09T13:28:28.752703Z","iopub.status.idle":"2022-07-09T13:28:52.529751Z","shell.execute_reply.started":"2022-07-09T13:28:28.752667Z","shell.execute_reply":"2022-07-09T13:28:52.528760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = list(data.new_col)\ny = list(data[\"target\"])","metadata":{"id":"WyWcZBQuid2V","execution":{"iopub.status.busy":"2022-07-09T13:28:52.531222Z","iopub.execute_input":"2022-07-09T13:28:52.531800Z","iopub.status.idle":"2022-07-09T13:28:52.540293Z","shell.execute_reply.started":"2022-07-09T13:28:52.531756Z","shell.execute_reply":"2022-07-09T13:28:52.539442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tokenized = tokenizer(X, padding=True, truncation=True, max_length=200)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:28:52.543177Z","iopub.execute_input":"2022-07-09T13:28:52.546101Z","iopub.status.idle":"2022-07-09T13:29:04.145398Z","shell.execute_reply.started":"2022-07-09T13:28:52.546073Z","shell.execute_reply":"2022-07-09T13:29:04.144328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tokenized.keys()","metadata":{"id":"Ll_YffE1rrKV","outputId":"9112756a-535e-483a-f767-f10a4a904b89","execution":{"iopub.status.busy":"2022-07-09T13:29:04.150349Z","iopub.execute_input":"2022-07-09T13:29:04.152701Z","iopub.status.idle":"2022-07-09T13:29:04.162560Z","shell.execute_reply.started":"2022-07-09T13:29:04.152661Z","shell.execute_reply":"2022-07-09T13:29:04.161603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create torch dataset\nclass Dataset(torch.utils.data.Dataset):\n    def __init__(self, encodings, labels=None):\n        self.encodings = encodings\n        self.labels = labels\n\n    def __getitem__(self, idx):\n        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n        if self.labels:\n            item[\"labels\"] = torch.tensor(self.labels[idx])\n        return item\n\n    def __len__(self):\n        return len(self.encodings[\"input_ids\"])","metadata":{"id":"0Dq__AdmjiEh","execution":{"iopub.status.busy":"2022-07-09T13:29:04.167627Z","iopub.execute_input":"2022-07-09T13:29:04.170199Z","iopub.status.idle":"2022-07-09T13:29:04.180487Z","shell.execute_reply.started":"2022-07-09T13:29:04.170141Z","shell.execute_reply":"2022-07-09T13:29:04.179278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = Dataset(X_tokenized, y)","metadata":{"id":"52bjWWitkMRw","execution":{"iopub.status.busy":"2022-07-09T13:29:04.185660Z","iopub.execute_input":"2022-07-09T13:29:04.188314Z","iopub.status.idle":"2022-07-09T13:29:04.194622Z","shell.execute_reply.started":"2022-07-09T13:29:04.188274Z","shell.execute_reply":"2022-07-09T13:29:04.193358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset[5]","metadata":{"id":"gqrFeE7Trgtd","execution":{"iopub.status.busy":"2022-07-09T13:29:04.199658Z","iopub.execute_input":"2022-07-09T13:29:04.202354Z","iopub.status.idle":"2022-07-09T13:29:04.215647Z","shell.execute_reply.started":"2022-07-09T13:29:04.202317Z","shell.execute_reply":"2022-07-09T13:29:04.214643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_metrics(p):\n    print(type(p))\n    pred, labels = p\n    pred = np.argmax(pred, axis=1)\n\n    accuracy = accuracy_score(y_true=labels, y_pred=pred)\n    recall = recall_score(y_true=labels, y_pred=pred)\n    precision = precision_score(y_true=labels, y_pred=pred)\n    f1 = f1_score(y_true=labels, y_pred=pred)\n\n    return {\"accuracy\": accuracy, \"precision\": precision, \"recall\": recall, \"f1\": f1}","metadata":{"id":"rSYfH_aIkOwf","execution":{"iopub.status.busy":"2022-07-09T13:29:04.219663Z","iopub.execute_input":"2022-07-09T13:29:04.221542Z","iopub.status.idle":"2022-07-09T13:29:04.231831Z","shell.execute_reply.started":"2022-07-09T13:29:04.221499Z","shell.execute_reply":"2022-07-09T13:29:04.230590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define Trainer\nargs = TrainingArguments(\n    output_dir=\"output\",\n    num_train_epochs=1,\n    per_device_train_batch_size=5\n\n)\ntrainer = Trainer(\n    model=model,\n    args=args,\n    train_dataset=train_dataset,\n    compute_metrics=compute_metrics\n)","metadata":{"id":"vFcXgFFNkRDP","scrolled":true,"execution":{"iopub.status.busy":"2022-07-09T13:29:04.236577Z","iopub.execute_input":"2022-07-09T13:29:04.239019Z","iopub.status.idle":"2022-07-09T13:29:09.436015Z","shell.execute_reply.started":"2022-07-09T13:29:04.238980Z","shell.execute_reply":"2022-07-09T13:29:09.435039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train()","metadata":{"id":"topNq2I6kTnX","outputId":"eb8b1262-9301-46b4-f69d-347a47b3539e","scrolled":true,"execution":{"iopub.status.busy":"2022-07-09T13:29:09.437565Z","iopub.execute_input":"2022-07-09T13:29:09.437898Z","iopub.status.idle":"2022-07-09T13:31:45.301309Z","shell.execute_reply.started":"2022-07-09T13:29:09.437860Z","shell.execute_reply":"2022-07-09T13:31:45.300281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"../input/nlp-getting-started/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:31:45.302960Z","iopub.execute_input":"2022-07-09T13:31:45.307439Z","iopub.status.idle":"2022-07-09T13:31:45.336684Z","shell.execute_reply.started":"2022-07-09T13:31:45.307408Z","shell.execute_reply":"2022-07-09T13:31:45.335772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.fillna(\" \")\ntest[\"new_col\"]=test[\"text\"]+test[\"keyword\"]+test[\"location\"]\ntest.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:31:45.338068Z","iopub.execute_input":"2022-07-09T13:31:45.338715Z","iopub.status.idle":"2022-07-09T13:31:45.361077Z","shell.execute_reply.started":"2022-07-09T13:31:45.338678Z","shell.execute_reply":"2022-07-09T13:31:45.360133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds=[]","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:53:34.878725Z","iopub.execute_input":"2022-07-09T13:53:34.879113Z","iopub.status.idle":"2022-07-09T13:53:34.885894Z","shell.execute_reply.started":"2022-07-09T13:53:34.879081Z","shell.execute_reply":"2022-07-09T13:53:34.884909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx,text in enumerate(list(test.new_col)): # text = \"go to hell\"\n    inputs = tokenizer(text, padding = True, truncation = True, max_length=200, return_tensors='pt').to('cuda')\n    outputs = model(**inputs)\n    #print(outputs)\n    predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)\n    #print(predictions)\n    predictions = predictions.cpu().detach().numpy()\n    prediction=np.argmax(predictions, axis=1)\n    preds.append(prediction[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:53:35.293110Z","iopub.execute_input":"2022-07-09T13:53:35.293706Z","iopub.status.idle":"2022-07-09T13:54:12.281496Z","shell.execute_reply.started":"2022-07-09T13:53:35.293671Z","shell.execute_reply":"2022-07-09T13:54:12.280515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.save_model('CustomModel')","metadata":{"id":"6zGTQP62MbAr","outputId":"a4c7049f-127d-4feb-fc32-4a8abcbb80db","execution":{"iopub.status.busy":"2022-07-09T13:41:33.629401Z","iopub.execute_input":"2022-07-09T13:41:33.629760Z","iopub.status.idle":"2022-07-09T13:41:34.429301Z","shell.execute_reply.started":"2022-07-09T13:41:33.629731Z","shell.execute_reply":"2022-07-09T13:41:34.428380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"predictions\"]=preds","metadata":{"execution":{"iopub.status.busy":"2022-07-09T14:00:00.893524Z","iopub.execute_input":"2022-07-09T14:00:00.893874Z","iopub.status.idle":"2022-07-09T14:00:00.907553Z","shell.execute_reply.started":"2022-07-09T14:00:00.893844Z","shell.execute_reply":"2022-07-09T14:00:00.906650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-07-09T14:00:01.128105Z","iopub.execute_input":"2022-07-09T14:00:01.128435Z","iopub.status.idle":"2022-07-09T14:00:01.149904Z","shell.execute_reply.started":"2022-07-09T14:00:01.128406Z","shell.execute_reply":"2022-07-09T14:00:01.149016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub=test[['id','predictions']]","metadata":{"execution":{"iopub.status.busy":"2022-07-09T14:02:16.569683Z","iopub.execute_input":"2022-07-09T14:02:16.570080Z","iopub.status.idle":"2022-07-09T14:02:16.580526Z","shell.execute_reply.started":"2022-07-09T14:02:16.570045Z","shell.execute_reply":"2022-07-09T14:02:16.579015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"execution":{"iopub.status.busy":"2022-07-09T14:03:18.124739Z","iopub.execute_input":"2022-07-09T14:03:18.125883Z","iopub.status.idle":"2022-07-09T14:03:18.140836Z","shell.execute_reply.started":"2022-07-09T14:03:18.125833Z","shell.execute_reply":"2022-07-09T14:03:18.139057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-09T14:03:30.173215Z","iopub.execute_input":"2022-07-09T14:03:30.174112Z","iopub.status.idle":"2022-07-09T14:03:30.191919Z","shell.execute_reply.started":"2022-07-09T14:03:30.174076Z","shell.execute_reply":"2022-07-09T14:03:30.190908Z"},"trusted":true},"execution_count":null,"outputs":[]}]}