{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\npd.options.display.max_colwidth = None\npd.options.display.max_columns = 10","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-05-27T22:13:49.074777Z","iopub.execute_input":"2021-05-27T22:13:49.075022Z","iopub.status.idle":"2021-05-27T22:13:49.085899Z","shell.execute_reply.started":"2021-05-27T22:13:49.074998Z","shell.execute_reply":"2021-05-27T22:13:49.084845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Quora insincere questions classification\n\n## Objective\n\n* Predict whether a question asked on Quora is sincere or not\n* Binary classification","metadata":{"execution":{"iopub.status.busy":"2021-05-24T01:10:49.742859Z","iopub.execute_input":"2021-05-24T01:10:49.743278Z","iopub.status.idle":"2021-05-24T01:10:49.747603Z","shell.execute_reply.started":"2021-05-24T01:10:49.743246Z","shell.execute_reply":"2021-05-24T01:10:49.746746Z"}}},{"cell_type":"code","source":"train_data = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest_data = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:13:51.58251Z","iopub.execute_input":"2021-05-27T22:13:51.58289Z","iopub.status.idle":"2021-05-27T22:13:56.314481Z","shell.execute_reply.started":"2021-05-27T22:13:51.582858Z","shell.execute_reply":"2021-05-27T22:13:56.313389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hand_picked_positives = [0, 8, 12, 16, 41]\npositive_data = train_data.loc[train_data['target'] == 1].iloc[hand_picked_positives].copy()\npositive_data","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:13:56.319235Z","iopub.execute_input":"2021-05-27T22:13:56.319605Z","iopub.status.idle":"2021-05-27T22:13:56.394487Z","shell.execute_reply.started":"2021-05-27T22:13:56.31957Z","shell.execute_reply":"2021-05-27T22:13:56.393769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hand_picked_negatives = [2, 7, 11, 17, 28]\nnegative_data = train_data.loc[train_data['target'] == 0].iloc[hand_picked_negatives].copy()\nnegative_data","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:13:56.395916Z","iopub.execute_input":"2021-05-27T22:13:56.396377Z","iopub.status.idle":"2021-05-27T22:13:56.508493Z","shell.execute_reply.started":"2021-05-27T22:13:56.396344Z","shell.execute_reply":"2021-05-27T22:13:56.507721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Size of training set: {len(train_data)}')\nprint(f'Size of testing set: {len(test_data)}')\nprint('Distribution of labels in training set:')\nprint(train_data['target'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:13:58.7146Z","iopub.execute_input":"2021-05-27T22:13:58.71494Z","iopub.status.idle":"2021-05-27T22:13:58.732986Z","shell.execute_reply.started":"2021-05-27T22:13:58.714911Z","shell.execute_reply":"2021-05-27T22:13:58.731971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.concat([train_data.loc[train_data['target']==1].tail(2500), train_data.loc[train_data['target']==0].tail(2500)], axis=0)\ntrain_data['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:14:03.186413Z","iopub.execute_input":"2021-05-27T22:14:03.186759Z","iopub.status.idle":"2021-05-27T22:14:03.368504Z","shell.execute_reply.started":"2021-05-27T22:14:03.186728Z","shell.execute_reply":"2021-05-27T22:14:03.367609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Following code is mostly copied from: https://www.thepythoncode.com/article/finetuning-bert-using-huggingface-transformers-python (thank you!)\n\n# the model we gonna train, base uncased BERT\n# check text classification models here: https://huggingface.co/models?filter=text-classification\nmodel_name = \"bert-base-uncased\"\n# max sequence length for each document/sentence sample\nmax_length = 128","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:14:04.556338Z","iopub.execute_input":"2021-05-27T22:14:04.556673Z","iopub.status.idle":"2021-05-27T22:14:04.562794Z","shell.execute_reply.started":"2021-05-27T22:14:04.556641Z","shell.execute_reply":"2021-05-27T22:14:04.562017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import BertTokenizerFast, BertForSequenceClassification\n\n# load the tokenizer\ntokenizer = BertTokenizerFast.from_pretrained(model_name, do_lower_case=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:14:15.562192Z","iopub.execute_input":"2021-05-27T22:14:15.562542Z","iopub.status.idle":"2021-05-27T22:14:20.256481Z","shell.execute_reply.started":"2021-05-27T22:14:15.562496Z","shell.execute_reply":"2021-05-27T22:14:20.255654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_texts, val_texts, train_labels, val_labels = train_test_split(train_data['question_text'].apply(str).tolist(),\n                                                                    train_data['target'].apply(int).tolist(), train_size=0.8)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:14:26.363621Z","iopub.execute_input":"2021-05-27T22:14:26.364312Z","iopub.status.idle":"2021-05-27T22:14:27.228994Z","shell.execute_reply.started":"2021-05-27T22:14:26.36427Z","shell.execute_reply":"2021-05-27T22:14:27.227972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_encodings = tokenizer(train_texts, truncation=True, padding=True, max_length=max_length)\nval_encodings = tokenizer(val_texts, truncation=True, padding=True, max_length=max_length)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:14:27.333492Z","iopub.execute_input":"2021-05-27T22:14:27.333911Z","iopub.status.idle":"2021-05-27T22:14:27.768289Z","shell.execute_reply.started":"2021-05-27T22:14:27.333878Z","shell.execute_reply":"2021-05-27T22:14:27.767432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\nclass CustomDataset(torch.utils.data.Dataset):\n    def __init__(self, encodings, labels):\n        self.encodings = encodings\n        self.labels = labels\n\n    def __getitem__(self, idx):\n        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n        item['labels'] = torch.tensor(self.labels[idx])\n        return item\n\n    def __len__(self):\n        return len(self.labels)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:14:30.342492Z","iopub.execute_input":"2021-05-27T22:14:30.342855Z","iopub.status.idle":"2021-05-27T22:14:30.348852Z","shell.execute_reply.started":"2021-05-27T22:14:30.342826Z","shell.execute_reply":"2021-05-27T22:14:30.34788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert our tokenized data into a torch Dataset\ntrain_dataset = CustomDataset(train_encodings, train_labels)\nvalid_dataset = CustomDataset(val_encodings, val_labels)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:14:30.664328Z","iopub.execute_input":"2021-05-27T22:14:30.664717Z","iopub.status.idle":"2021-05-27T22:14:30.668662Z","shell.execute_reply.started":"2021-05-27T22:14:30.664679Z","shell.execute_reply":"2021-05-27T22:14:30.667764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the model and pass to CUDA\nmodel = BertForSequenceClassification.from_pretrained(model_name, num_labels=2).to(\"cuda\")","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:14:33.233691Z","iopub.execute_input":"2021-05-27T22:14:33.234039Z","iopub.status.idle":"2021-05-27T22:15:03.499915Z","shell.execute_reply.started":"2021-05-27T22:14:33.23401Z","shell.execute_reply":"2021-05-27T22:15:03.499021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\ndef compute_metrics(pred):\n  labels = pred.label_ids\n  preds = pred.predictions.argmax(-1)\n  # calculate accuracy using sklearn's function\n  acc = accuracy_score(labels, preds)\n  return {\n      'accuracy': acc,\n  }","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:15:03.50127Z","iopub.execute_input":"2021-05-27T22:15:03.501616Z","iopub.status.idle":"2021-05-27T22:15:03.509141Z","shell.execute_reply.started":"2021-05-27T22:15:03.50158Z","shell.execute_reply":"2021-05-27T22:15:03.508359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import Trainer, TrainingArguments\n\ntraining_args = TrainingArguments(\n    output_dir='./results',          # output directory\n    num_train_epochs=1,              # total number of training epochs\n    per_device_train_batch_size=16,  # batch size per device during training\n    per_device_eval_batch_size=20,   # batch size for evaluation\n    warmup_steps=50,                 # number of warmup steps for learning rate scheduler\n    weight_decay=0.01,               # strength of weight decay\n    logging_dir='./logs',            # directory for storing logs\n    load_best_model_at_end=True,     # load the best model when finished training (default metric is loss)\n    # but you can specify `metric_for_best_model` argument to change to accuracy or other metric\n    logging_steps=50,               # log & save weights each logging_steps\n    evaluation_strategy=\"steps\",     # evaluate each `logging_steps`\n)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:15:03.51081Z","iopub.execute_input":"2021-05-27T22:15:03.511231Z","iopub.status.idle":"2021-05-27T22:15:07.963065Z","shell.execute_reply.started":"2021-05-27T22:15:03.511198Z","shell.execute_reply":"2021-05-27T22:15:07.962208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = Trainer(\n    model=model,                         # the instantiated Transformers model to be trained\n    args=training_args,                  # training arguments, defined above\n    train_dataset=train_dataset,         # training dataset\n    eval_dataset=valid_dataset,          # evaluation dataset\n    compute_metrics=compute_metrics,     # the callback that computes metrics of interest\n)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:15:07.966213Z","iopub.execute_input":"2021-05-27T22:15:07.966469Z","iopub.status.idle":"2021-05-27T22:15:08.999575Z","shell.execute_reply.started":"2021-05-27T22:15:07.966443Z","shell.execute_reply":"2021-05-27T22:15:08.998636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train the model\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:15:09.000909Z","iopub.execute_input":"2021-05-27T22:15:09.00123Z","iopub.status.idle":"2021-05-27T22:17:43.153682Z","shell.execute_reply.started":"2021-05-27T22:15:09.001196Z","shell.execute_reply":"2021-05-27T22:17:43.152723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluate the current model after training\ntrainer.evaluate()","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:17:43.155651Z","iopub.execute_input":"2021-05-27T22:17:43.156214Z","iopub.status.idle":"2021-05-27T22:17:46.628055Z","shell.execute_reply.started":"2021-05-27T22:17:43.156177Z","shell.execute_reply":"2021-05-27T22:17:46.62728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_prediction_proba(text):\n    # prepare our text into tokenized sequence\n    inputs = tokenizer(text, padding=True, truncation=True, return_tensors=\"pt\").to(\"cuda\")\n    # perform inference to our model\n    outputs = model(**inputs)\n    # get output probabilities by doing softmax\n    probs = outputs[0].softmax(1)\n    # executing argmax function to get the candidate label\n    return probs\n\ndef get_prediction(text):\n    return get_prediction_proba(text).argmax().item()","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:19:11.495107Z","iopub.execute_input":"2021-05-27T22:19:11.495454Z","iopub.status.idle":"2021-05-27T22:19:11.501262Z","shell.execute_reply.started":"2021-05-27T22:19:11.49542Z","shell.execute_reply":"2021-05-27T22:19:11.50038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(get_prediction_proba(\"Is this sincere question?\"))","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:22:49.832744Z","iopub.execute_input":"2021-05-27T22:22:49.83308Z","iopub.status.idle":"2021-05-27T22:22:49.86593Z","shell.execute_reply.started":"2021-05-27T22:22:49.833049Z","shell.execute_reply":"2021-05-27T22:22:49.865078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_data['pred'] = positive_data['question_text'].apply(get_prediction))\nnegative_data['pred'] = negative_data['question_text'].apply(lambda x: get_prediction.argmax().item())","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:19:16.938133Z","iopub.execute_input":"2021-05-27T22:19:16.938614Z","iopub.status.idle":"2021-05-27T22:19:17.127315Z","shell.execute_reply.started":"2021-05-27T22:19:16.938562Z","shell.execute_reply":"2021-05-27T22:19:17.126513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_data","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:19:19.282478Z","iopub.execute_input":"2021-05-27T22:19:19.282931Z","iopub.status.idle":"2021-05-27T22:19:19.30124Z","shell.execute_reply.started":"2021-05-27T22:19:19.282891Z","shell.execute_reply":"2021-05-27T22:19:19.300042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"negative_data","metadata":{"execution":{"iopub.status.busy":"2021-05-27T22:19:22.448958Z","iopub.execute_input":"2021-05-27T22:19:22.449377Z","iopub.status.idle":"2021-05-27T22:19:22.472098Z","shell.execute_reply.started":"2021-05-27T22:19:22.44933Z","shell.execute_reply":"2021-05-27T22:19:22.471159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}