{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-24T01:32:39.46982Z","iopub.execute_input":"2021-07-24T01:32:39.470187Z","iopub.status.idle":"2021-07-24T01:32:39.489723Z","shell.execute_reply.started":"2021-07-24T01:32:39.470119Z","shell.execute_reply":"2021-07-24T01:32:39.488752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Main idea\npre-trained language model (wikitext 103) --> refined language model (trained on quora data) --> classifier","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"# Must install specific version such that it's compatible with acc41 language model\n!pip uninstall spacy --yes\n!pip install spacy==2.2.4\n!pip install fastai --upgrade","metadata":{"execution":{"iopub.status.busy":"2021-07-24T01:32:46.742637Z","iopub.execute_input":"2021-07-24T01:32:46.742968Z","iopub.status.idle":"2021-07-24T01:33:02.899223Z","shell.execute_reply.started":"2021-07-24T01:32:46.742937Z","shell.execute_reply":"2021-07-24T01:33:02.898247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"########## Additional Import ##########\nimport fastai\nfrom fastai.losses import *\nfrom fastai.text.all import *\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nimport multiprocessing as mp\nfrom math import floor\nimport pickle\nimport spacy\nfrom matplotlib import pyplot\n#######################################","metadata":{"execution":{"iopub.status.busy":"2021-07-24T01:33:02.901372Z","iopub.execute_input":"2021-07-24T01:33:02.901737Z","iopub.status.idle":"2021-07-24T01:33:04.616912Z","shell.execute_reply.started":"2021-07-24T01:33:02.901697Z","shell.execute_reply":"2021-07-24T01:33:04.616056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read training csv\ntrain_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\n# Optional (to gain a sample)\ntrain_df = train_df.sample(frac = 1)\nprint(train_df.shape)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-07-24T01:33:04.618703Z","iopub.execute_input":"2021-07-24T01:33:04.619113Z","iopub.status.idle":"2021-07-24T01:33:07.880872Z","shell.execute_reply.started":"2021-07-24T01:33:04.619073Z","shell.execute_reply":"2021-07-24T01:33:07.879741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Proportion of sincere ot insincere questions\ntrain_df.target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-24T01:33:07.882571Z","iopub.execute_input":"2021-07-24T01:33:07.883023Z","iopub.status.idle":"2021-07-24T01:33:07.904255Z","shell.execute_reply.started":"2021-07-24T01:33:07.882981Z","shell.execute_reply":"2021-07-24T01:33:07.903154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Create is_train column that defines which rows are in the training set\nseed = 38\nnp.random.seed(seed)\ntrain_proportion = 0.8\nproportion = floor(train_df.shape[0] * train_proportion)\nidx = np.random.choice(train_df.shape[0], (proportion), replace=False)\n\ntrain_df[\"is_train\"] = train_df.index.isin(idx)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-07-24T01:33:07.905525Z","iopub.execute_input":"2021-07-24T01:33:07.905884Z","iopub.status.idle":"2021-07-24T01:33:08.301146Z","shell.execute_reply.started":"2021-07-24T01:33:07.90585Z","shell.execute_reply":"2021-07-24T01:33:08.300163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Proportion of sincere to insincere if using custom validation set\ntrain_df.loc[train_df.is_train].target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-24T01:33:46.02145Z","iopub.execute_input":"2021-07-24T01:33:46.021804Z","iopub.status.idle":"2021-07-24T01:33:46.182039Z","shell.execute_reply.started":"2021-07-24T01:33:46.021773Z","shell.execute_reply":"2021-07-24T01:33:46.181028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Proportion of sincere to insincere if using custom validation set (dev set proportions)\ntrain_df.loc[~train_df.is_train].target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-24T01:33:48.04641Z","iopub.execute_input":"2021-07-24T01:33:48.046738Z","iopub.status.idle":"2021-07-24T01:33:48.111541Z","shell.execute_reply.started":"2021-07-24T01:33:48.046709Z","shell.execute_reply":"2021-07-24T01:33:48.110563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Language Model","metadata":{}},{"cell_type":"markdown","source":"Datablock utilises parallelisation to tokenize and numericalize when textblock is passed to datablock.","metadata":{}},{"cell_type":"code","source":"# Create DataBlock (dev_set = 20% of train data i.e. entire data set --> no need to create a test seta as that isn't the objective)\ndev_set_size = 0.2\nquora_db = DataBlock(blocks=TextBlock.from_df('question_text', is_lm=True),\n                       get_x=ColReader('text'), splitter=RandomSplitter(dev_set_size))\ndls = quora_db.dataloaders(train_df, bs=128)\ndls.show_batch(max_n=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* xxbos to indicate the beginning of a text\n* xxmaj to indicate the next word was capitalized","metadata":{}},{"cell_type":"code","source":"\"\"\"\nCreates a pre-trained langauge model (pretrained model: Wikitext 103) using AWD_LSTM architecture.\n- Metrics: accuracy, perplexity (exponential of the loss)\n- drop_mult is a parameter that controls the magnitude of all dropouts in that model \n(i.e. randomly drops input based on given probability for all layers, in this case we provide a value which scales the default probabilities)\n- to_fp16 asserts that predictions are float16 values (helps speed up training on Nvidia GPUs with tensor cores)\n\"\"\"\n\n#dls = pickle.load(open(\"/kaggle/input/acc41-language-model/savelm.p\", \"rb\"))\n    \nlearn = language_model_learner(\n    dls, AWD_LSTM, drop_mult=0.3, \n    metrics=[accuracy, Perplexity()]).to_fp16()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Search for an appropriate learning rate \n# Template: start_lr = ?, end_lr = ?\nlearn.lr_find()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"By default, a pretrained Learner is in a frozen state, meaning that only the head of the model will train while the body stays frozen. Below, fit one epoch then save in case we want to re-load the model and retrain once we fit more epochs.","metadata":{}},{"cell_type":"code","source":"# Trains only the last layer\nlearn.fit_one_cycle(1, 0.03)\nlearn.save(\"1epoch\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find new lr\nlearn.lr_find()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Begin fine-tuning model, by un-freezing, then training for multiple epochs.\n\nNote: Encoder == saves the entire model except the final layer that converts activations to probabilities of picking each token in the vocabulary","metadata":{}},{"cell_type":"code","source":"# unfreeze all layers of the model\nlearn.unfreeze()\n\n# Begin Training\n# epochs = 10\n\n# SaveModelCallback keeps the model with the best val_loss\n# ReduceLROnPlateau reduces LR after 2 succesive epochs with no decrease greater than 0.1\ncallbacks = [SaveModelCallback(fname=\"languageModel\"), \n             ReduceLROnPlateau(monitor='valid_loss', min_delta=0.1, patience=2)]\n\nlearn.fit_one_cycle(10, 1e-3, cbs = callbacks)\n\n# Save model except final layer (this is what's used for the classifier)\nlearn.save_encoder(\"finetuned\")\n\n# Plot loss\nlearn.recorder.plot_loss()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save dls so that we can load it for the classifier\npickle.dump(dls, open(\"savelm.p\", \"wb\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test the langauge model\nlearn.load(\"/kaggle/input/acc41-language-model/languageModel\")\n#learn.load(\"languageModel\")\nTEXT = input(\"Enter the start of a question: \")\n#TEXT = \"What do I do if\"\nN_WORDS = 10\nlearn.predict(TEXT, N_WORDS, temperature=0.3) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Language Classifier","metadata":{}},{"cell_type":"code","source":"# Create DataBlock for classifier using pretrained vocab from language model (to ensure that they use the same token to index mapping) \n# Datablock will also handle any padding (for each batch pad by the size of the largest text [libraries cannot apply all batches with the same padding])\n# Batches text with similar lengths\n\n# If available pre-load language model dataloader from binary\ntry: \n    dls = pickle.load(open(\"savelm.p\", \"rb\"))\nexcept FileNotFoundError:\n    dls = pickle.load(open(\"/kaggle/input/acc41-language-model/savelm.p\", \"rb\"))\n    \n# If available pre-load language classifier from binary\ntry: \n    if seed != 42: raise Exception(\"New random state, re-intialise data loader\")\n    dls_classifier = pickle.load(open(\"../input/9417-v1/dls_classifier.p\", \"rb\"))\n    dls_classifier.show_batch(max_n=2)\nexcept: \n    # Create dataloader (purpose: preprocessing and batching)\n    dev_set_size = 0.2\n    dls_classifier = DataBlock(\n        blocks=(TextBlock.from_df('question_text', vocab=dls.vocab), CategoryBlock),\n        get_x=ColReader('text'), \n        get_y=ColReader(\"target\"), \n        splitter=RandomSplitter(dev_set_size)\n    ).dataloaders(train_df.loc[train_df.is_train], bs=512)\n    dls_classifier.show_batch(max_n=2)\n\n    pickle.dump(dls_classifier, open(\"dls_classifier.p\", \"wb\"))","metadata":{"execution":{"iopub.status.busy":"2021-07-24T01:35:16.496375Z","iopub.execute_input":"2021-07-24T01:35:16.496748Z","iopub.status.idle":"2021-07-24T02:00:22.08587Z","shell.execute_reply.started":"2021-07-24T01:35:16.496714Z","shell.execute_reply":"2021-07-24T02:00:22.084906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialise testing data loader\ntest_x = train_df.loc[~train_df.is_train].rename(columns={\"question_text\":\"text\"})\ntest_x_dl = dls_classifier.test_dl(test_x, with_labels=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-24T02:00:22.087503Z","iopub.execute_input":"2021-07-24T02:00:22.087846Z","iopub.status.idle":"2021-07-24T02:06:14.257375Z","shell.execute_reply.started":"2021-07-24T02:00:22.08781Z","shell.execute_reply":"2021-07-24T02:06:14.25638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dump test data loader binaries for later use if necessary\npickle.dump(test_x_dl, open(\"test_x_dl.p\", \"wb\"))","metadata":{"execution":{"iopub.status.busy":"2021-07-24T02:06:14.259217Z","iopub.execute_input":"2021-07-24T02:06:14.259588Z","iopub.status.idle":"2021-07-24T02:06:15.080276Z","shell.execute_reply.started":"2021-07-24T02:06:14.259546Z","shell.execute_reply":"2021-07-24T02:06:15.079373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialise RNN and load pre-trained language model\n# Metrics: F1-Score, RocAucBinary\nlearn = text_classifier_learner(dls_classifier, \n                                AWD_LSTM, \n                                drop_mult=0.5, \n                                loss_func=FocalLossFlat(),\n                                metrics=[F1Score()]).to_fp16()\ntry: \n    # load encoding if available in kaggle working directory\n    learn = learn.load_encoder(\"finetuned\")\nexcept:\n    # Else load from kaggle data set\n    learn = learn.load_encoder(\"/kaggle/input/acc41-language-model/finetuned\")","metadata":{"execution":{"iopub.status.busy":"2021-07-24T02:06:15.081699Z","iopub.execute_input":"2021-07-24T02:06:15.082046Z","iopub.status.idle":"2021-07-24T02:06:20.923858Z","shell.execute_reply.started":"2021-07-24T02:06:15.082006Z","shell.execute_reply":"2021-07-24T02:06:20.922881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fine-tune by gradually unfreezing layers","metadata":{}},{"cell_type":"code","source":"# Find lr on un-refined model\nlearn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2021-07-24T02:06:20.925421Z","iopub.execute_input":"2021-07-24T02:06:20.925804Z","iopub.status.idle":"2021-07-24T02:07:21.521916Z","shell.execute_reply.started":"2021-07-24T02:06:20.925765Z","shell.execute_reply":"2021-07-24T02:07:21.520974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit one cycle with appropriate lr from above.\nlearn.fit_one_cycle(1, 0.001)","metadata":{"execution":{"iopub.status.busy":"2021-07-24T02:07:28.802516Z","iopub.execute_input":"2021-07-24T02:07:28.802936Z","iopub.status.idle":"2021-07-24T02:26:27.934297Z","shell.execute_reply.started":"2021-07-24T02:07:28.802902Z","shell.execute_reply":"2021-07-24T02:26:27.933251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save first cycle\nlearn.save(\"classifier_1\")","metadata":{"execution":{"iopub.status.busy":"2021-07-24T02:26:27.936387Z","iopub.execute_input":"2021-07-24T02:26:27.936818Z","iopub.status.idle":"2021-07-24T02:26:28.382133Z","shell.execute_reply.started":"2021-07-24T02:26:27.93676Z","shell.execute_reply":"2021-07-24T02:26:28.381214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find new lr\nlearn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2021-07-24T02:26:28.383987Z","iopub.execute_input":"2021-07-24T02:26:28.384306Z","iopub.status.idle":"2021-07-24T02:27:27.466994Z","shell.execute_reply.started":"2021-07-24T02:26:28.384275Z","shell.execute_reply":"2021-07-24T02:27:27.466186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Refine last 2 layers\n# Use discriminative layer training\nlearn.freeze_to(-2)\nlearn.fit_one_cycle(1, slice(1e-3/(2.6**4),1e-3))","metadata":{"execution":{"iopub.status.busy":"2021-07-24T02:37:23.519455Z","iopub.execute_input":"2021-07-24T02:37:23.51981Z","iopub.status.idle":"2021-07-24T02:56:33.006531Z","shell.execute_reply.started":"2021-07-24T02:37:23.519776Z","shell.execute_reply":"2021-07-24T02:56:33.005604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save 2nd cycle\nlearn.save(\"classifier_2\")","metadata":{"execution":{"iopub.status.busy":"2021-07-24T02:56:33.008346Z","iopub.execute_input":"2021-07-24T02:56:33.008708Z","iopub.status.idle":"2021-07-24T02:56:33.460009Z","shell.execute_reply.started":"2021-07-24T02:56:33.008666Z","shell.execute_reply":"2021-07-24T02:56:33.45915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find new lr \nlearn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2021-07-24T02:56:33.461938Z","iopub.execute_input":"2021-07-24T02:56:33.46237Z","iopub.status.idle":"2021-07-24T02:57:33.48158Z","shell.execute_reply.started":"2021-07-24T02:56:33.462322Z","shell.execute_reply":"2021-07-24T02:57:33.480632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Refine last 3 layers\n# Use discriminative layer training\nlearn.freeze_to(-3)\nlearn.fit_one_cycle(1, slice(1e-3/(2.6**4), 1e-3))","metadata":{"execution":{"iopub.status.busy":"2021-07-24T02:58:06.538161Z","iopub.execute_input":"2021-07-24T02:58:06.538526Z","iopub.status.idle":"2021-07-24T03:17:35.65261Z","shell.execute_reply.started":"2021-07-24T02:58:06.538497Z","shell.execute_reply":"2021-07-24T03:17:35.651701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save 3rd cycle\nlearn.save(\"classifier_3\")","metadata":{"execution":{"iopub.status.busy":"2021-07-24T03:17:35.654525Z","iopub.execute_input":"2021-07-24T03:17:35.654899Z","iopub.status.idle":"2021-07-24T03:17:36.242757Z","shell.execute_reply.started":"2021-07-24T03:17:35.654855Z","shell.execute_reply":"2021-07-24T03:17:36.241286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find new lr \nlearn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2021-07-24T03:17:36.247938Z","iopub.execute_input":"2021-07-24T03:17:36.24858Z","iopub.status.idle":"2021-07-24T03:18:28.073255Z","shell.execute_reply.started":"2021-07-24T03:17:36.248509Z","shell.execute_reply":"2021-07-24T03:18:28.072081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Refine entire model\nlearn.unfreeze()\ncallbacks = [SaveModelCallback(fname=\"final_classifier\"),\n            ReduceLROnPlateau(monitor='valid_loss', min_delta=0.1, patience=2)]\nlearn.fit_one_cycle(4, slice(0.001/(2.6**4),0.001), cbs = callbacks)\n\n# Plot loss\nlearn.recorder.plot_loss()","metadata":{"execution":{"iopub.status.busy":"2021-07-24T03:21:24.366559Z","iopub.execute_input":"2021-07-24T03:21:24.3669Z","iopub.status.idle":"2021-07-24T04:39:30.261131Z","shell.execute_reply.started":"2021-07-24T03:21:24.366866Z","shell.execute_reply":"2021-07-24T04:39:30.260247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions","metadata":{}},{"cell_type":"code","source":"# Reload model if necessary\ndls_classifier = pickle.load(open(\"/kaggle/input/d/juliangarratt/9417-v1/dls_classifier.p\", \"rb\"))\nlearn = text_classifier_learner(dls_classifier, \n                                AWD_LSTM, \n                                drop_mult=0.5, \n                                loss_func=FocalLossFlat(),\n                                metrics=[F1Score()],\n                                model_dir=\"../input/d/juliangarratt/9417-v1/models\").to_fp16()\nlearn.load(\"final_classifier\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test on validation set (loss, f1-score)\nlearn.validate()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get Preds\ntest_x_df = train_df.loc[~train_df.is_train]\ntry:\n    test_dl = pickle.load(open(\"./test_x_dl.p\", \"rb\"))\nexcept: \n    # Initialise dls classifier if not available in locally\n    test_dl = dls_classifier.test_dl(test_x_df.question_text.tolist())\npreds = learn.get_preds(dl=test_dl)","metadata":{"execution":{"iopub.status.busy":"2021-07-24T04:53:15.868686Z","iopub.execute_input":"2021-07-24T04:53:15.869033Z","iopub.status.idle":"2021-07-24T04:58:56.158897Z","shell.execute_reply.started":"2021-07-24T04:53:15.868993Z","shell.execute_reply":"2021-07-24T04:58:56.157942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate Predictions from proba & get true preds\ny_true = test_x_df.target.to_numpy()\ny_preds = np.argmax(np.array(preds[0].tolist()), axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-24T04:58:56.160673Z","iopub.execute_input":"2021-07-24T04:58:56.161022Z","iopub.status.idle":"2021-07-24T04:58:56.680038Z","shell.execute_reply.started":"2021-07-24T04:58:56.160983Z","shell.execute_reply":"2021-07-24T04:58:56.679055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# F-Score (out of sample)\nprint(\"Out of sample f-score:\", metrics.f1_score(y_true, y_preds))","metadata":{"execution":{"iopub.status.busy":"2021-07-24T04:58:56.682295Z","iopub.execute_input":"2021-07-24T04:58:56.682802Z","iopub.status.idle":"2021-07-24T04:58:56.774808Z","shell.execute_reply.started":"2021-07-24T04:58:56.682759Z","shell.execute_reply":"2021-07-24T04:58:56.773618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Precision-Recall Curve\npyplot.figure(figsize=(8, 6), dpi=80)\nprecision, recall, thresholds = metrics.precision_recall_curve(y_true, np.array(preds[0].tolist())[:, 1])\nno_skill = len(y_true[y_true==1]) / len(y_true)\npyplot.plot([0, 1], [no_skill, no_skill], linestyle='--', label='No Skill')\npyplot.plot(recall, precision, marker='.', label='Model')\npyplot.xlabel('Recall')\npyplot.ylabel('Precision')\npyplot.title(\"Precision-Recall Curve\")\npyplot.legend()\npyplot.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-24T04:58:56.776511Z","iopub.execute_input":"2021-07-24T04:58:56.776873Z","iopub.status.idle":"2021-07-24T04:58:57.484505Z","shell.execute_reply.started":"2021-07-24T04:58:56.776833Z","shell.execute_reply":"2021-07-24T04:58:57.483668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate optimal threshold & plot curve from custom package\nfrom f_score_thresholding_utility import *\nthreshold_finder(np.array(preds[0].tolist())[:, 0].tolist(), y_true)","metadata":{"execution":{"iopub.status.busy":"2021-07-24T04:58:57.485729Z","iopub.execute_input":"2021-07-24T04:58:57.486055Z","iopub.status.idle":"2021-07-24T04:59:39.516007Z","shell.execute_reply.started":"2021-07-24T04:58:57.486017Z","shell.execute_reply":"2021-07-24T04:59:39.514575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Notes\n\n## Observations\n* Sincere qs\n    * Often contain math\n    * Often highly specialised e.g. scientific jargon\n* Insincere qs\n    * Highly political\n    * Contain swear words\n\n## Resources\n* https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing\n* https://stanford-cs221.github.io/autumn2019-extra/posters/162.pdf\n* https://towardsdatascience.com/quora-insincere-questions-classification-d5a655370c47\n* https://medium.com/@ph_singer/1st-place-in-kaggle-quora-insincere-questions-classification-competition-520616d39938\n\n## Improvements\n* Loss function --> focal loss (no improvements)\n* Embeddinngs\n\n## Previous results\n* f-score 0.55 with dev_size=0.1 and sample=0.2\n* f-score 0.63 with dev_size=0.2 and sample=0.8\n* f-score 0.64 with dev_size=0.2 and sample=0.8\n* f-score 0.64 (focal loss)","metadata":{}}]}