{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","editable":false,"execution":{"iopub.status.busy":"2021-07-25T05:54:23.982806Z","iopub.execute_input":"2021-07-25T05:54:23.983123Z","iopub.status.idle":"2021-07-25T05:54:23.991949Z","shell.execute_reply.started":"2021-07-25T05:54:23.983095Z","shell.execute_reply":"2021-07-25T05:54:23.990959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport numpy as np\nimport sklearn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils.class_weight import compute_class_weight\nimport torch.nn as nn\nimport os\nos.environ[\"WANDB_DISABLED\"] = \"true\"\n","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-25T05:54:24.468583Z","iopub.execute_input":"2021-07-25T05:54:24.468870Z","iopub.status.idle":"2021-07-25T05:54:26.403083Z","shell.execute_reply.started":"2021-07-25T05:54:24.468843Z","shell.execute_reply":"2021-07-25T05:54:26.402293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv').sample(frac=1, random_state=1)\ntrain,test_data = train_test_split(df,test_size=0.2, random_state=1)\nprint(train['target'].value_counts())","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-25T05:59:27.267976Z","iopub.execute_input":"2021-07-25T05:59:27.268313Z","iopub.status.idle":"2021-07-25T05:59:31.946284Z","shell.execute_reply.started":"2021-07-25T05:59:27.268282Z","shell.execute_reply":"2021-07-25T05:59:31.945327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train.head(len(train)//2)\n# train_data = pd.concat([train.loc[train['target']==1].tail(60000), train.loc[train['target']==0].tail(60000)], axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T05:59:35.776804Z","iopub.execute_input":"2021-07-25T05:59:35.777165Z","iopub.status.idle":"2021-07-25T05:59:35.782428Z","shell.execute_reply.started":"2021-07-25T05:59:35.777134Z","shell.execute_reply":"2021-07-25T05:59:35.781411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = \"distilroberta-base\"\nmax_length = 128","metadata":{"execution":{"iopub.status.busy":"2021-07-25T05:59:42.299486Z","iopub.execute_input":"2021-07-25T05:59:42.299807Z","iopub.status.idle":"2021-07-25T05:59:42.303541Z","shell.execute_reply.started":"2021-07-25T05:59:42.299778Z","shell.execute_reply":"2021-07-25T05:59:42.302563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_weights = compute_class_weight('balanced', np.unique(train_data['target']), train_data['target'])\nprint(\"Class Weights:\",class_weights)\nweights = torch.tensor(class_weights,dtype=torch.float)\nweights = weights.to(\"cuda\")\ncross_entropy  = nn.NLLLoss(weight=weights)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T05:59:49.008267Z","iopub.execute_input":"2021-07-25T05:59:49.008604Z","iopub.status.idle":"2021-07-25T05:59:53.143601Z","shell.execute_reply.started":"2021-07-25T05:59:49.008576Z","shell.execute_reply":"2021-07-25T05:59:53.142741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n# from transformers import BertTokenizerFast, BertForSequenceClassification\nfrom transformers import RobertaTokenizerFast, RobertaForSequenceClassification\n\n# load the tokenizer\ntokenizer = RobertaTokenizerFast.from_pretrained(model_name, do_lower_case=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T05:59:57.571677Z","iopub.execute_input":"2021-07-25T05:59:57.571988Z","iopub.status.idle":"2021-07-25T06:00:01.852690Z","shell.execute_reply.started":"2021-07-25T05:59:57.571958Z","shell.execute_reply":"2021-07-25T06:00:01.851818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_texts, val_texts, train_labels, val_labels = train_test_split(train_data['question_text'].apply(str).tolist(),\n                                                                    train_data['target'].apply(int).tolist(), train_size=0.8)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-25T06:00:04.588106Z","iopub.execute_input":"2021-07-25T06:00:04.588479Z","iopub.status.idle":"2021-07-25T06:00:05.403294Z","shell.execute_reply.started":"2021-07-25T06:00:04.588449Z","shell.execute_reply":"2021-07-25T06:00:05.402387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_encodings = tokenizer(train_texts, truncation=True, padding=True, max_length=max_length)\nval_encodings = tokenizer(val_texts, truncation=True, padding=True, max_length=max_length)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-25T06:00:05.404776Z","iopub.execute_input":"2021-07-25T06:00:05.405125Z","iopub.status.idle":"2021-07-25T06:00:51.612343Z","shell.execute_reply.started":"2021-07-25T06:00:05.405088Z","shell.execute_reply":"2021-07-25T06:00:51.611342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\nclass CustomDataset(torch.utils.data.Dataset):\n    def __init__(self, encodings, labels):\n        self.encodings = encodings\n        self.labels = labels\n\n    def __getitem__(self, idx):\n        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n        item['labels'] = torch.tensor(self.labels[idx])\n        return item\n\n    def __len__(self):\n        return len(self.labels)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-25T06:00:51.614225Z","iopub.execute_input":"2021-07-25T06:00:51.614602Z","iopub.status.idle":"2021-07-25T06:00:51.621247Z","shell.execute_reply.started":"2021-07-25T06:00:51.614561Z","shell.execute_reply":"2021-07-25T06:00:51.620276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert our tokenized data into a torch Dataset\ntrain_dataset = CustomDataset(train_encodings, train_labels)\nvalid_dataset = CustomDataset(val_encodings, val_labels)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-25T06:00:51.623119Z","iopub.execute_input":"2021-07-25T06:00:51.623854Z","iopub.status.idle":"2021-07-25T06:00:51.632078Z","shell.execute_reply.started":"2021-07-25T06:00:51.623810Z","shell.execute_reply":"2021-07-25T06:00:51.631235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = RobertaForSequenceClassification.from_pretrained(model_name, num_labels=2).to(\"cuda\")\n# model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=2).to(\"cuda\")\n# model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=2).to(\"cuda\")","metadata":{"execution":{"iopub.status.busy":"2021-07-25T06:00:51.633626Z","iopub.execute_input":"2021-07-25T06:00:51.634255Z","iopub.status.idle":"2021-07-25T06:01:11.865172Z","shell.execute_reply.started":"2021-07-25T06:00:51.634213Z","shell.execute_reply":"2021-07-25T06:01:11.864221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\ndef compute_metrics(pred):\n  labels = pred.label_ids\n  preds = pred.predictions.argmax(-1)\n  # calculate accuracy using sklearn's function\n  acc = accuracy_score(labels, preds)\n  return {\n      'accuracy': acc,\n  }","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-25T06:01:11.869958Z","iopub.execute_input":"2021-07-25T06:01:11.872253Z","iopub.status.idle":"2021-07-25T06:01:11.879439Z","shell.execute_reply.started":"2021-07-25T06:01:11.872197Z","shell.execute_reply":"2021-07-25T06:01:11.878692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import Trainer, TrainingArguments\n\nclass WeightedTrainer(Trainer):\n    def compute_loss(self, model, inputs, return_outputs=False):\n        labels = inputs.pop(\"labels\")\n        outputs = model(**inputs)\n        logits = outputs.logits\n        # print('labels shape', labels.shape)\n        # print('logits shape', logits.shape)\n        loss_fct = nn.CrossEntropyLoss(weight=torch.tensor(class_weights,device=\"cuda\").float())\n        loss = loss_fct(logits,labels.long())\n        return (loss, outputs) if return_outputs else loss\n\n# training_args = TrainingArguments(\n#     output_dir='./results',          # output directory\n#     num_train_epochs=1,              # total number of training epochs\n#     per_device_train_batch_size=16,  # batch size per device during training\n#     per_device_eval_batch_size=20,   # batch size for evaluation\n#     warmup_steps=1000,                 # number of warmup steps for learning rate scheduler\n#     weight_decay=0.01,               # strength of weight decay\n#     logging_dir='./logs',            # directory for storing logs\n#     load_best_model_at_end=True,     # load the best model when finished training (default metric is loss)\n#     # but you can specify `metric_for_best_model` argument to change to accuracy or other metric\n#     logging_steps=1000,               # log & save weights each logging_steps - used to be 50\n#     evaluation_strategy=\"steps\",     # evaluate each `logging_steps`\n# )\n\ntraining_args = TrainingArguments(\n    output_dir='./results',          # output directory\n    num_train_epochs=2,              # total number of training epochs\n    per_device_train_batch_size=16,  # batch size per device during training\n    per_device_eval_batch_size=20,   # batch size for evaluation\n    warmup_steps=50,                 # number of warmup steps for learning rate scheduler\n    weight_decay=0.01,               # strength of weight decay\n    logging_dir='./logs',            # directory for storing logs\n    load_best_model_at_end=False,     # load the best model when finished training (default metric is loss)\n    # but you can specify `metric_for_best_model` argument to change to accuracy or other metric\n    logging_steps=1000,               # log & save weights each logging_steps\n    save_steps=1000,\n    save_total_limit=5,\n    fp16=True,\n    evaluation_strategy=\"steps\",     # evaluate each `logging_steps`\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T06:01:11.883126Z","iopub.execute_input":"2021-07-25T06:01:11.883464Z","iopub.status.idle":"2021-07-25T06:01:17.437854Z","shell.execute_reply.started":"2021-07-25T06:01:11.883427Z","shell.execute_reply":"2021-07-25T06:01:17.436960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = WeightedTrainer(\n    model=model,                         # the instantiated Transformers model to be trained\n    args=training_args,                  # training arguments, defined above\n    train_dataset=train_dataset,         # training dataset\n    eval_dataset=valid_dataset,          # evaluation dataset\n    compute_metrics=compute_metrics,     # the callback that computes metrics of interest\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T06:01:17.439301Z","iopub.execute_input":"2021-07-25T06:01:17.439659Z","iopub.status.idle":"2021-07-25T06:01:20.275631Z","shell.execute_reply.started":"2021-07-25T06:01:17.439621Z","shell.execute_reply":"2021-07-25T06:01:20.274757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train the model\ntrainer.train()","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-25T06:01:20.276967Z","iopub.execute_input":"2021-07-25T06:01:20.277335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluate the current model after training\ntrainer.evaluate()","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-24T07:30:28.245732Z","iopub.execute_input":"2021-07-24T07:30:28.246088Z","iopub.status.idle":"2021-07-24T07:34:39.2149Z","shell.execute_reply.started":"2021-07-24T07:30:28.246033Z","shell.execute_reply":"2021-07-24T07:34:39.214145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_prediction_proba(text):\n    # prepare our text into tokenized sequence\n    inputs = tokenizer(text, padding=True, truncation=True, return_tensors=\"pt\").to(\"cuda\")\n    # perform inference to our model\n    outputs = model(**inputs)\n    # get output probabilities by doing softmax\n    probs = outputs[0].softmax(1)\n    # executing argmax function to get the candidate label\n    return probs\n\ndef get_prediction(text):\n    return get_prediction_proba(text).argmax().item()","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-24T07:34:39.216217Z","iopub.execute_input":"2021-07-24T07:34:39.216549Z","iopub.status.idle":"2021-07-24T07:34:39.222779Z","shell.execute_reply.started":"2021-07-24T07:34:39.216513Z","shell.execute_reply":"2021-07-24T07:34:39.221497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test_data.sample(4000)\ntest_data['target'].value_counts()\n# test = pd.concat([test_data[test_data['target']==1].head(1000), test_data[test_data['target']==0].head(1000)], axis=0)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-24T07:34:39.224125Z","iopub.execute_input":"2021-07-24T07:34:39.224599Z","iopub.status.idle":"2021-07-24T07:34:39.247704Z","shell.execute_reply.started":"2021-07-24T07:34:39.224561Z","shell.execute_reply":"2021-07-24T07:34:39.246771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['pred'] = test['question_text'].apply(get_prediction)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-24T07:34:39.248848Z","iopub.execute_input":"2021-07-24T07:34:39.249218Z","iopub.status.idle":"2021-07-24T07:35:34.211647Z","shell.execute_reply.started":"2021-07-24T07:34:39.249184Z","shell.execute_reply":"2021-07-24T07:35:34.210734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sklearn.metrics.f1_score(test['target'], test['pred'])","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-24T07:35:34.213079Z","iopub.execute_input":"2021-07-24T07:35:34.21346Z","iopub.status.idle":"2021-07-24T07:35:34.224741Z","shell.execute_reply.started":"2021-07-24T07:35:34.213421Z","shell.execute_reply":"2021-07-24T07:35:34.223644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(test['target'], test['pred'])","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-07-24T07:35:34.226271Z","iopub.execute_input":"2021-07-24T07:35:34.226665Z","iopub.status.idle":"2021-07-24T07:35:34.235018Z","shell.execute_reply.started":"2021-07-24T07:35:34.226624Z","shell.execute_reply":"2021-07-24T07:35:34.233858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-25T05:54:38.629805Z","iopub.execute_input":"2021-07-25T05:54:38.630116Z","iopub.status.idle":"2021-07-25T05:54:39.270032Z","shell.execute_reply.started":"2021-07-25T05:54:38.630087Z","shell.execute_reply":"2021-07-25T05:54:39.269086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final['pred'] = test_final['question_text'].apply(get_prediction)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T05:54:50.964949Z","iopub.execute_input":"2021-07-25T05:54:50.965304Z","iopub.status.idle":"2021-07-25T05:54:50.974115Z","shell.execute_reply.started":"2021-07-25T05:54:50.965271Z","shell.execute_reply":"2021-07-25T05:54:50.973242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = test_final.drop(columns=['question_text'])\nsubmission.to_csv('submission.csv')","metadata":{},"execution_count":null,"outputs":[]}]}