{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# https://huggingface.co/transformers/v3.2.0/custom_datasets.html","metadata":{"id":"ImW5xLg5Ezyo"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install transformers -U","metadata":{"id":"l5ceeO9LhWkU","execution":{"iopub.status.busy":"2022-07-09T11:28:34.229956Z","iopub.execute_input":"2022-07-09T11:28:34.230454Z","iopub.status.idle":"2022-07-09T11:28:59.423279Z","shell.execute_reply.started":"2022-07-09T11:28:34.230352Z","shell.execute_reply":"2022-07-09T11:28:59.420713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd","metadata":{"id":"8HNUE5fXIb-W","execution":{"iopub.status.busy":"2022-07-09T11:28:59.428959Z","iopub.execute_input":"2022-07-09T11:28:59.432741Z","iopub.status.idle":"2022-07-09T11:28:59.442189Z","shell.execute_reply.started":"2022-07-09T11:28:59.432687Z","shell.execute_reply":"2022-07-09T11:28:59.439465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/competitions/jigsaw-toxic-comment-classification-challenge/data\ndata = pd.read_csv(\"../input/jigsaw-toxic-comment-classification-challenge/train.csv.zip\",error_bad_lines=False, engine=\"python\")\ndata.head()","metadata":{"id":"ZLuKyvHJ47xK","outputId":"ae5dbe57-baca-44f9-c947-f341a8c6206a","execution":{"iopub.status.busy":"2022-07-09T11:28:59.445731Z","iopub.execute_input":"2022-07-09T11:28:59.446694Z","iopub.status.idle":"2022-07-09T11:29:04.759576Z","shell.execute_reply.started":"2022-07-09T11:28:59.446649Z","shell.execute_reply":"2022-07-09T11:29:04.758494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data.to_csv(\"/content/drive/MyDrive/Youtube Tutorials/datasets/toxic_commnets.csv\",index=False)","metadata":{"id":"zn7OlLJnwZJH","execution":{"iopub.status.busy":"2022-07-09T11:29:04.762498Z","iopub.execute_input":"2022-07-09T11:29:04.763104Z","iopub.status.idle":"2022-07-09T11:29:04.767470Z","shell.execute_reply.started":"2022-07-09T11:29:04.763065Z","shell.execute_reply":"2022-07-09T11:29:04.766390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['toxic'].value_counts()","metadata":{"id":"5dbrirLG5D0A","outputId":"c64ef77c-2bb8-4243-8b89-66e0e1a3e6cc","execution":{"iopub.status.busy":"2022-07-09T11:29:04.770403Z","iopub.execute_input":"2022-07-09T11:29:04.771313Z","iopub.status.idle":"2022-07-09T11:29:04.824714Z","shell.execute_reply.started":"2022-07-09T11:29:04.771209Z","shell.execute_reply":"2022-07-09T11:29:04.823623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data[['comment_text','toxic']]\ndata = data[0:1000]\ndata.head()","metadata":{"id":"U2HyJRvG5trP","outputId":"e01ac8ee-cacd-4fcb-921d-21074d9874dc","execution":{"iopub.status.busy":"2022-07-09T11:29:04.826394Z","iopub.execute_input":"2022-07-09T11:29:04.827001Z","iopub.status.idle":"2022-07-09T11:29:04.851881Z","shell.execute_reply.started":"2022-07-09T11:29:04.826971Z","shell.execute_reply":"2022-07-09T11:29:04.850613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score\nimport torch\nfrom transformers import TrainingArguments, Trainer\nfrom transformers import BertTokenizer, BertForSequenceClassification","metadata":{"id":"s32nl2kn56cH","execution":{"iopub.status.busy":"2022-07-09T11:29:04.853852Z","iopub.execute_input":"2022-07-09T11:29:04.854310Z","iopub.status.idle":"2022-07-09T11:29:14.771066Z","shell.execute_reply.started":"2022-07-09T11:29:04.854271Z","shell.execute_reply":"2022-07-09T11:29:14.769867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import BertTokenizer, BertForSequenceClassification\ntokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\nmodel = BertForSequenceClassification.from_pretrained('bert-base-uncased',num_labels=2)","metadata":{"id":"Z1JlB3eYiV8a","outputId":"4f606d36-bba0-4880-e3a2-c59eb25f9727","scrolled":true,"execution":{"iopub.status.busy":"2022-07-09T11:29:14.776344Z","iopub.execute_input":"2022-07-09T11:29:14.777734Z","iopub.status.idle":"2022-07-09T11:29:54.926353Z","shell.execute_reply.started":"2022-07-09T11:29:14.777694Z","shell.execute_reply":"2022-07-09T11:29:54.925040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"id":"WPKswImmYUjO","execution":{"iopub.status.busy":"2022-07-09T11:29:54.928499Z","iopub.execute_input":"2022-07-09T11:29:54.929008Z","iopub.status.idle":"2022-07-09T11:29:54.943097Z","shell.execute_reply.started":"2022-07-09T11:29:54.928958Z","shell.execute_reply":"2022-07-09T11:29:54.941525Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = model.to('cuda')","metadata":{"id":"nCIO165RiZfC","scrolled":true,"execution":{"iopub.status.busy":"2022-07-09T11:29:54.947652Z","iopub.execute_input":"2022-07-09T11:29:54.948932Z","iopub.status.idle":"2022-07-09T11:30:01.918233Z","shell.execute_reply.started":"2022-07-09T11:29:54.948873Z","shell.execute_reply":"2022-07-09T11:30:01.916968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_data = [\"I am eating\",\"I am playing \"]\ntokenizer(sample_data, padding=True, truncation=True, max_length=512)","metadata":{"id":"tESWTOfVHEqA","outputId":"a0e86b55-9484-4d50-c9be-aa5db6971a00","execution":{"iopub.status.busy":"2022-07-09T11:30:01.924372Z","iopub.execute_input":"2022-07-09T11:30:01.924866Z","iopub.status.idle":"2022-07-09T11:30:01.939872Z","shell.execute_reply.started":"2022-07-09T11:30:01.924820Z","shell.execute_reply":"2022-07-09T11:30:01.938783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = list(data[\"comment_text\"])\ny = list(data[\"toxic\"])\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2,stratify=y)\nX_train_tokenized = tokenizer(X_train, padding=True, truncation=True, max_length=512)\nX_val_tokenized = tokenizer(X_val, padding=True, truncation=True, max_length=512)","metadata":{"id":"WyWcZBQuid2V","execution":{"iopub.status.busy":"2022-07-09T11:31:01.482269Z","iopub.execute_input":"2022-07-09T11:31:01.483014Z","iopub.status.idle":"2022-07-09T11:31:04.678279Z","shell.execute_reply.started":"2022-07-09T11:31:01.482959Z","shell.execute_reply":"2022-07-09T11:31:04.677243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_tokenized.keys()","metadata":{"id":"Ll_YffE1rrKV","outputId":"9112756a-535e-483a-f767-f10a4a904b89","execution":{"iopub.status.busy":"2022-07-09T11:31:05.361083Z","iopub.execute_input":"2022-07-09T11:31:05.361474Z","iopub.status.idle":"2022-07-09T11:31:05.368772Z","shell.execute_reply.started":"2022-07-09T11:31:05.361441Z","shell.execute_reply":"2022-07-09T11:31:05.367615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train_tokenized['attention_mask'][0])","metadata":{"id":"O5TGibGZIAZT","outputId":"a1afaceb-f6c2-480a-f2e1-619a1dc6f1bc","execution":{"iopub.status.busy":"2022-07-09T11:31:13.027380Z","iopub.execute_input":"2022-07-09T11:31:13.028180Z","iopub.status.idle":"2022-07-09T11:31:13.035164Z","shell.execute_reply.started":"2022-07-09T11:31:13.028134Z","shell.execute_reply":"2022-07-09T11:31:13.033631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(X_train),len(X_val)","metadata":{"id":"d2bs25fZish6","outputId":"b6d99fb5-803e-437f-9977-c21990b488a4","execution":{"iopub.status.busy":"2022-07-09T11:31:13.673156Z","iopub.execute_input":"2022-07-09T11:31:13.674057Z","iopub.status.idle":"2022-07-09T11:31:13.680126Z","shell.execute_reply.started":"2022-07-09T11:31:13.674019Z","shell.execute_reply":"2022-07-09T11:31:13.679135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create torch dataset\nclass Dataset(torch.utils.data.Dataset):\n    def __init__(self, encodings, labels=None):\n        self.encodings = encodings\n        self.labels = labels\n\n    def __getitem__(self, idx):\n        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n        if self.labels:\n            item[\"labels\"] = torch.tensor(self.labels[idx])\n        return item\n\n    def __len__(self):\n        return len(self.encodings[\"input_ids\"])","metadata":{"id":"0Dq__AdmjiEh","execution":{"iopub.status.busy":"2022-07-09T11:31:14.546406Z","iopub.execute_input":"2022-07-09T11:31:14.547525Z","iopub.status.idle":"2022-07-09T11:31:14.557870Z","shell.execute_reply.started":"2022-07-09T11:31:14.547463Z","shell.execute_reply":"2022-07-09T11:31:14.555390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = Dataset(X_train_tokenized, y_train)\nval_dataset = Dataset(X_val_tokenized, y_val)","metadata":{"id":"52bjWWitkMRw","execution":{"iopub.status.busy":"2022-07-09T11:31:15.919566Z","iopub.execute_input":"2022-07-09T11:31:15.920481Z","iopub.status.idle":"2022-07-09T11:31:15.926479Z","shell.execute_reply.started":"2022-07-09T11:31:15.920433Z","shell.execute_reply":"2022-07-09T11:31:15.925074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset[5]","metadata":{"id":"gqrFeE7Trgtd","execution":{"iopub.status.busy":"2022-07-09T11:31:16.546025Z","iopub.execute_input":"2022-07-09T11:31:16.546429Z","iopub.status.idle":"2022-07-09T11:31:16.566739Z","shell.execute_reply.started":"2022-07-09T11:31:16.546395Z","shell.execute_reply":"2022-07-09T11:31:16.565735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_metrics(p):\n    print(type(p))\n    pred, labels = p\n    pred = np.argmax(pred, axis=1)\n\n    accuracy = accuracy_score(y_true=labels, y_pred=pred)\n    recall = recall_score(y_true=labels, y_pred=pred)\n    precision = precision_score(y_true=labels, y_pred=pred)\n    f1 = f1_score(y_true=labels, y_pred=pred)\n\n    return {\"accuracy\": accuracy, \"precision\": precision, \"recall\": recall, \"f1\": f1}","metadata":{"id":"rSYfH_aIkOwf","execution":{"iopub.status.busy":"2022-07-09T11:31:17.219331Z","iopub.execute_input":"2022-07-09T11:31:17.220190Z","iopub.status.idle":"2022-07-09T11:31:17.228872Z","shell.execute_reply.started":"2022-07-09T11:31:17.220138Z","shell.execute_reply":"2022-07-09T11:31:17.227604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define Trainer\nargs = TrainingArguments(\n    output_dir=\"output\",\n    num_train_epochs=1,\n    per_device_train_batch_size=5\n\n)\ntrainer = Trainer(\n    model=model,\n    args=args,\n    train_dataset=train_dataset,\n    eval_dataset=val_dataset,\n    compute_metrics=compute_metrics\n)","metadata":{"id":"vFcXgFFNkRDP","scrolled":true,"execution":{"iopub.status.busy":"2022-07-09T11:31:17.841514Z","iopub.execute_input":"2022-07-09T11:31:17.842054Z","iopub.status.idle":"2022-07-09T11:31:17.864599Z","shell.execute_reply.started":"2022-07-09T11:31:17.842003Z","shell.execute_reply":"2022-07-09T11:31:17.863616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train()","metadata":{"id":"topNq2I6kTnX","outputId":"eb8b1262-9301-46b4-f69d-347a47b3539e","scrolled":true,"execution":{"iopub.status.busy":"2022-07-09T11:31:30.640100Z","iopub.execute_input":"2022-07-09T11:31:30.641122Z","iopub.status.idle":"2022-07-09T11:36:35.467862Z","shell.execute_reply.started":"2022-07-09T11:31:30.641080Z","shell.execute_reply":"2022-07-09T11:36:35.466161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.evaluate()","metadata":{"id":"Ia14Y_UmN7qM","outputId":"91ff6e21-fdf0-40bf-c501-7ec880bd72a7","scrolled":true,"execution":{"iopub.status.busy":"2022-07-09T11:36:46.105009Z","iopub.execute_input":"2022-07-09T11:36:46.105389Z","iopub.status.idle":"2022-07-09T11:36:49.786251Z","shell.execute_reply.started":"2022-07-09T11:36:46.105357Z","shell.execute_reply":"2022-07-09T11:36:49.785361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.set_printoptions(suppress=True)","metadata":{"id":"KX1xAXjhqibr","execution":{"iopub.status.busy":"2022-07-09T11:37:06.517408Z","iopub.execute_input":"2022-07-09T11:37:06.518048Z","iopub.status.idle":"2022-07-09T11:37:06.525722Z","shell.execute_reply.started":"2022-07-09T11:37:06.518012Z","shell.execute_reply":"2022-07-09T11:37:06.524080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = \"That was good point\"\n# text = \"go to hell\"\ninputs = tokenizer(text,padding = True, truncation = True, return_tensors='pt').to('cuda')\noutputs = model(**inputs)\nprint(outputs)\npredictions = torch.nn.functional.softmax(outputs.logits, dim=-1)\nprint(predictions)\npredictions = predictions.cpu().detach().numpy()\npredictions","metadata":{"id":"AdVqrPN0kVLY","outputId":"3c63ea34-68f0-4afe-f7e0-235128d8311c","execution":{"iopub.status.busy":"2022-07-09T11:37:07.282436Z","iopub.execute_input":"2022-07-09T11:37:07.283710Z","iopub.status.idle":"2022-07-09T11:37:07.335747Z","shell.execute_reply.started":"2022-07-09T11:37:07.283627Z","shell.execute_reply":"2022-07-09T11:37:07.334809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.save_model('CustomModel')","metadata":{"id":"6zGTQP62MbAr","outputId":"a4c7049f-127d-4feb-fc32-4a8abcbb80db","execution":{"iopub.status.busy":"2022-07-09T11:37:08.914625Z","iopub.execute_input":"2022-07-09T11:37:08.915855Z","iopub.status.idle":"2022-07-09T11:37:09.700829Z","shell.execute_reply.started":"2022-07-09T11:37:08.915807Z","shell.execute_reply":"2022-07-09T11:37:09.699482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trainer.save_model('/content/drive/MyDrive/Youtube Tutorials/toxic')\n# model_2 = BertForSequenceClassification.from_pretrained(\"/content/drive/MyDrive/Youtube Tutorials/toxic\")\n# model_2.to('cuda')","metadata":{"id":"eLnznjr1PYVG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2 = BertForSequenceClassification.from_pretrained(\"CustomModel\")\nmodel_2.to('cuda')","metadata":{"id":"zTu-SoK6MeVJ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# text = \"That was good point\"\ntext = \"go to hell\"\ninputs = tokenizer(text,padding = True, truncation = True, return_tensors='pt').to('cuda')\noutputs = model_2(**inputs)\npredictions = torch.nn.functional.softmax(outputs.logits, dim=-1)\npredictions = predictions.cpu().detach().numpy()\npredictions","metadata":{"id":"v8dqYXZwM12Z","outputId":"606bd583-b60c-4279-c1ea-5640ccdee634"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"owBTtpMkPm0l"},"execution_count":null,"outputs":[]}]}