{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom transformers import AutoTokenizer, AutoModelForSequenceClassification\nfrom fastai.text.all import *\nfrom sklearn.model_selection import train_test_split\n\nfrom torch.utils.data import Dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"/kaggle/input/quora-insincere-questions-classification/\"\ntrain_df = pd.read_csv(path + \"train.csv\")\ntest_df = pd.read_csv(path + \"test.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape, test_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"target\"].value_counts()/train_df.shape[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_text = pd.concat([train_df[\"question_text\"], test_df[\"question_text\"]], axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sample_text(n=10):\n    sample = all_text.sample(n)\n    print(\" | \".join(sample))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_text()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"question_text\"].apply(lambda x:len(x.split())).plot(kind=\"hist\");","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class QuestionDataset(Dataset):\n    def __init__(self, X, y, tokenizer):\n        self.text = X.reset_index(drop=True)\n        self.targets = y.reset_index(drop=True)\n        self.tok = tokenizer\n    \n    def __len__(self):\n        return len(self.text)\n    \n    def __getitem__(self, idx):\n        \n        text = self.text[idx]\n        targ = self.targets[idx]\n        \n        return self.tok(text, padding='max_length', \n                        truncation=True,\n                        max_length=30,\n                        return_tensors=\"pt\")[\"input_ids\"][0], tensor(targ)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_df\nX_train, X_valid, y_train, y_valid = train_test_split(df[\"question_text\"], df[\"target\"], \n                                                      stratify=df[\"target\"],  test_size=0.01)\n\ntrain_ds = QuestionDataset(X_train, y_train, tokenizer)\nvalid_ds = QuestionDataset(X_valid, y_valid, tokenizer)\n\ntrain_dl = DataLoader(train_ds, bs=256)\nvalid_dl = DataLoader(valid_ds, bs=512)\ndls = DataLoaders(train_dl, valid_dl).to(\"cuda\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert = AutoModelForSequenceClassification.from_pretrained('bert-base-cased').train()\n\nclassifier = nn.Sequential(\n    nn.Linear(768, 1024),\n    nn.ReLU(),\n    nn.Dropout(0.5),\n    nn.Linear(1024, 2)\n)\n\nbert.classifier = classifier\n\nclass BertClassifier(Module):\n    def __init__(self, bert):\n        self.bert = bert\n    def forward(self, x):\n        x = self.bert(x)\n        return x.logits\n\nmodel = BertClassifier(bert).to(\"cuda\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_0 = (train_df[\"target\"] == 0).sum()\nn_1 = (train_df[\"target\"] == 1).sum()\nn = n_0 + n_1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_weights = tensor([n / (n+n_0), n / (n+n_1)]).to('cuda')\nlearn = Learner(dls, model, \n                loss_func=nn.CrossEntropyLoss(weight=class_weights), \n                metrics=[accuracy, F1Score()]).to_fp16()\nlearn.lr_find()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit_one_cycle(2, lr_max=5e-5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds, targs = learn.get_preds()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thresholds = np.linspace(0.3, 0.7, 50)\nfor threshold in thresholds:\n    f1 = f1_score(targs, F.softmax(preds, dim=1)[:, 1]>threshold)\n    print(f\"threshold:{threshold:.4f} - f1:{f1:.4f}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_tensor = tokenizer(list(test_df[\"question_text\"]),\n                        padding=\"max_length\",\n                        truncation=True,\n                        max_length=30,\n                        return_tensors=\"pt\")[\"input_ids\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDS:\n    def __init__(self, tensors):\n        self.tensors = tensors\n    \n    def __len__(self):\n        return len(self.tensors)\n    \n    def __getitem__(self, idx):\n        t = self.tensors[idx]\n        return t, tensor(0)\n\ntest_dl = DataLoader(TestDS(test_tensor), bs=128)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = learn.get_preds(dl=test_dl)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction = (F.softmax(test_preds[0], dim=1)[:, 1]>0.48).int()\nsub = pd.read_csv(path + \"sample_submission.csv\")\nsub[\"prediction\"] = prediction\nsub.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}