{"cells":[{"metadata":{"_uuid":"c3f5b77bad1c6762871e17d0c0d8d9bb28df1e82"},"cell_type":"markdown","source":"# Quora Insincere Question Classifier\n* Taken from the framework found in FastAI lesson 4 IMDB classifier [notebook](https://nbviewer.jupyter.org/github/fastai/course-v3/blob/master/nbs/dl1/lesson3-imdb.ipynb)\n* Language model does not use embeddings provided in competition and instead the pre-trained models included with FastAI "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, gc\nfrom fastai.text import *\nimport pandas as pd\nfrom fastai import *\nfrom tqdm import tqdm_notebook as tqdm\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05926772238fdcd3bcaa3fc2212fd5123e154fb3"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c85b39d7ce83bb7f4950be6f9141e1118edc36fa"},"cell_type":"code","source":"plt.hist(train.question_text.apply(lambda x: len(x)), density = False, bins = 40)\n#Length of questions asked","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e386af131e61a1681dfdb98e6419aafce2c8a35"},"cell_type":"code","source":"np.random.seed(42)\ntrain_small = train.iloc[train.sample(frac=0.99).index]\ntrain = train_small  #Comment out this cell later, taking smaller dataset in order to run through process\nplt.bar([\"False\",'True'], train.groupby('target').count().qid)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7d90b9b858d2ebc6dbd35674cc1439e301cb3dfd"},"cell_type":"markdown","source":"### Use a 20% sample of data to test the model within the kernel here"},{"metadata":{"trusted":true,"_uuid":"e469046be2a401c53c9fd26855d40395559fae38"},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95e7ac76da50904b95ae0b038cdb324ae048c007"},"cell_type":"code","source":"sample_size = 0.2\ntrain_df =train.sample(frac=(1-sample_size))\nvalid_df = train[~train.index.isin(train_df)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4c987bb8677674612cb4a2bac296a7c3c995b46"},"cell_type":"code","source":"%%time\ndata_lm = TextLMDataBunch.from_df(path = '.',\n                            train_df = train_df,\n                            valid_df = valid_df,\n                            test_df = test,\n                            text_cols = 'question_text',\n                            label_cols = 'target',\n                            max_vocab = 20000)\nprint(len(data_lm.vocab.itos))\ndata_lm.save()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fc6a2e8584fd039fed598400f95a4d3369fe30d"},"cell_type":"code","source":"data_lm.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34349d12bae298ed75bc4b2369ca8edd1db90f63"},"cell_type":"code","source":"data_lm.vocab.itos[100:105]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"50610da46cf0b6268b1967559a35833d29e4d343"},"cell_type":"markdown","source":"Create classification data-bunch. This sets the labels / targets to the actual labels of the data. "},{"metadata":{"trusted":true,"_uuid":"a9aedcba065a99d39f50738022dd3c3f648eec68"},"cell_type":"code","source":"%%time\ndata_class = TextClasDataBunch.from_df(path = '.',\n                                       train_df = train_df,\n                                       valid_df = valid_df,\n                                       test_df = test,\n                                       text_cols = 'question_text',\n                                       label_cols = 'target',\n                                       max_vocab = 20000,\n                                       vocab=data_lm.vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a606f4e42e267c3cfb0f2ef89077a30bc842ab99"},"cell_type":"code","source":"data_class.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f22ff93370f544c60609dfa6872647c75cff568"},"cell_type":"markdown","source":"## Train Language Model, starting from wiki103 \n* Note that this method following here uses pretrained embeddings from the fast ai library NOT the embeddings from the Quora competition. This is probably fine for outside work but would DQ from Quora competition"},{"metadata":{"trusted":true,"_uuid":"c82c9f67b8182a0bc72d80b717257eaf55e175fc"},"cell_type":"code","source":"path = Path(\"../\")\nmodel_path = path/'models'\nmodel_path.mkdir(exist_ok=True)\nurl = 'http://files.fast.ai/models/wt103_v1/'\ndownload_url(f'{url}lstm_wt103.pth', model_path/'lstm_wt103.pth')\ndownload_url(f'{url}itos_wt103.pkl', model_path/'itos_wt103.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0afcd1537f52991dfe892f4875b3a54207118ff"},"cell_type":"code","source":"learn = language_model_learner(data_lm, pretrained_fnames=['lstm_wt103', 'itos_wt103'], drop_mult=0.3, arch = AWD_LSTM, model_dir=model_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e390623f8ad43f8a40602e5bfe4a2dcc6faa9133"},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e32cffe3922780b0c355fc77e9b97a33b1cb95b9"},"cell_type":"code","source":"learn.fit_one_cycle(1, 1e-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e35f6929a57f725965e3aa8b05297a3c5678a59d"},"cell_type":"code","source":"learn.unfreeze()\nlearn.fit_one_cycle(1, 1e-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91317abd9789a58d6a7c747f6f6275b0e9f0ed3d"},"cell_type":"code","source":"learn.save_encoder('ft_enc')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c2b718cbf899c9f45f8df14185a1accdb03ce3d"},"cell_type":"code","source":"data_lm.vocab.itos[10:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9021a7aca70379e84be34c028ec2e6632b8560bd"},"cell_type":"code","source":"learn.predict('Why in the earth', 10)\n#So this looks great, ha","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14968d7d15ebbfa96e41e1390c7c3f33c41087e2"},"cell_type":"markdown","source":"## Train Classifier\nf-beta metric that's used in Quora classifier was found in fastai forum [Comment from wyquek](https://forums.fast.ai/t/f1-score-as-metric/30370/14)"},{"metadata":{"trusted":true,"_uuid":"188de10b423330922fbfafbacb9656c15cbfda78"},"cell_type":"code","source":"def f1_score(y_pred, targets):\n    epsilon = 1e-07\n    \n    y_pred = y_pred.argmax(dim = -1)\n    #targets = targets.argmax(dim=-1)\n\n    tp = (y_pred*targets).float().sum(dim=0)\n    tn = ((1-targets)*(1-y_pred)).float().sum(dim=0)\n    fp = ((1-targets)*y_pred).float().sum(dim=0)\n    fn = (targets*(1-y_pred)).sum(dim=0)\n\n    p = tp / (tp + fp + epsilon)\n    r = tp / (tp + fn + epsilon)\n\n    f1 = 2*p*r / (p+r+epsilon)\n    f1 = torch.where(f1!=f1, torch.zeros_like(f1), f1)\n    return f1.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8990147993e4baa7381982a7865cbfd76c75637e"},"cell_type":"code","source":"%%time\nlearn_class = text_classifier_learner(data_class, drop_mult = 0.5, \n                                      arch = AWD_LSTM, model_dir=model_path, \n                                     metrics = [accuracy, f1_score])\n\nlearn_class.load_encoder('ft_enc')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67b2ff7077446874021f24f0a4066839621b3e61"},"cell_type":"code","source":"learn = None\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a2556a6538c2ecce1a46b796ab9957e141214ed3"},"cell_type":"markdown","source":"Nee dto trouble shoot the lr_find why the graph isn't what we'd expect"},{"metadata":{"trusted":true,"_uuid":"b5631d70eecd99fad4afd9d36d4c9e755d987571"},"cell_type":"code","source":"learn_class.lr_find()\nlearn_class.recorder.plot_lr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"100ba5f2501abe5416ac5b9f2c68126f6ddefe84","scrolled":true},"cell_type":"code","source":"learn_class.fit_one_cycle(3, 1e-3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc7901b9950bc36b8a788d0cf72ea98395c7070b"},"cell_type":"code","source":"learn_class.freeze_to(-2)\nlearn_class.fit_one_cycle(4, slice(1e-3, 1e-1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c4f40f27c756b4ae608d4d1f12638c87a79da54d"},"cell_type":"markdown","source":"## Prediction threshold\n* Currently the f1 score just takes the greater of the two predictions, but since there is some bias in the prediction it's useful to search potential thresholds to determine optimal threshold for F1 scoring\n* Code taken from [fastai test](https://www.kaggle.com/mnpinto/quora-fastai-v1-0-baseline) by mnpinto"},{"metadata":{"trusted":true,"_uuid":"3a41ae09e4487a4a71cd8d5a9a12b83a1bbbe502"},"cell_type":"code","source":"from sklearn.metrics import roc_curve, precision_recall_curve\ndef threshold_search(y_true, y_proba, plot=False):\n    precision, recall, thresholds = precision_recall_curve(y_true, y_proba)\n    thresholds = np.append(thresholds, 1.001) \n    F = 2 / (1/precision + 1/recall)\n    best_score = np.max(F)\n    best_th = thresholds[np.argmax(F)]\n    if plot:\n        plt.plot(thresholds, F, '-b')\n        plt.plot([best_th], [best_score], '*r')\n        plt.show()\n    search_result = {'threshold': best_th , 'f1': best_score}\n    return search_result ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bd46d618492ad6da79bfb177111f9282c1c324b"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0c633fd725a5afd34c2288c395f89c816c9b0a6"},"cell_type":"code","source":"preds = learn_class.get_preds(DatasetType.Valid)\nproba = to_np(preds[0][:,1])\nytrue = to_np(preds[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c79922a3d515f8b7529f5d1f7cb9b6b17b4321ca"},"cell_type":"code","source":"thr = threshold_search(ytrue, proba, plot=True); thr","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fa7e8479f4e98f0a1b1222b0623491359a33b5ed"},"cell_type":"markdown","source":"By updating the threshold for a true flag to 0.19 we can achieve an f1 score of >0.5"},{"metadata":{"trusted":true,"_uuid":"a846b0629cfd21fbc22ce96d28a8dbc420a9f448"},"cell_type":"code","source":"probs, _ = learn_class.get_preds(DatasetType.Test)\npreds = np.argmax(probs, axis=1)\n\nsubmission = pd.DataFrame(test['qid'])\nsubmission['prediction'] = preds \nsubmission.to_csv('submission.csv',index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}