{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-23T03:01:20.004629Z","iopub.execute_input":"2021-05-23T03:01:20.005157Z","iopub.status.idle":"2021-05-23T03:01:20.015506Z","shell.execute_reply.started":"2021-05-23T03:01:20.005117Z","shell.execute_reply":"2021-05-23T03:01:20.014558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport random\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:20.01706Z","iopub.execute_input":"2021-05-23T03:01:20.017661Z","iopub.status.idle":"2021-05-23T03:01:20.034952Z","shell.execute_reply.started":"2021-05-23T03:01:20.017628Z","shell.execute_reply":"2021-05-23T03:01:20.034201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:20.036237Z","iopub.execute_input":"2021-05-23T03:01:20.036639Z","iopub.status.idle":"2021-05-23T03:01:22.795875Z","shell.execute_reply.started":"2021-05-23T03:01:20.036609Z","shell.execute_reply":"2021-05-23T03:01:22.794839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:22.797227Z","iopub.execute_input":"2021-05-23T03:01:22.797528Z","iopub.status.idle":"2021-05-23T03:01:22.807457Z","shell.execute_reply.started":"2021-05-23T03:01:22.797498Z","shell.execute_reply":"2021-05-23T03:01:22.806621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:22.810305Z","iopub.execute_input":"2021-05-23T03:01:22.810616Z","iopub.status.idle":"2021-05-23T03:01:22.94643Z","shell.execute_reply.started":"2021-05-23T03:01:22.810585Z","shell.execute_reply":"2021-05-23T03:01:22.945474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:22.948775Z","iopub.execute_input":"2021-05-23T03:01:22.949032Z","iopub.status.idle":"2021-05-23T03:01:22.954239Z","shell.execute_reply.started":"2021-05-23T03:01:22.949005Z","shell.execute_reply":"2021-05-23T03:01:22.953394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['question_text'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:22.955272Z","iopub.execute_input":"2021-05-23T03:01:22.955521Z","iopub.status.idle":"2021-05-23T03:01:23.023308Z","shell.execute_reply.started":"2021-05-23T03:01:22.955497Z","shell.execute_reply":"2021-05-23T03:01:23.022322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['question_text'].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:23.026175Z","iopub.execute_input":"2021-05-23T03:01:23.026466Z","iopub.status.idle":"2021-05-23T03:01:23.090533Z","shell.execute_reply.started":"2021-05-23T03:01:23.026419Z","shell.execute_reply":"2021-05-23T03:01:23.089765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.rename({'question_text': 'question'}, axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:23.093664Z","iopub.execute_input":"2021-05-23T03:01:23.093934Z","iopub.status.idle":"2021-05-23T03:01:23.180386Z","shell.execute_reply.started":"2021-05-23T03:01:23.093907Z","shell.execute_reply":"2021-05-23T03:01:23.179563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.columns","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:23.182495Z","iopub.execute_input":"2021-05-23T03:01:23.182916Z","iopub.status.idle":"2021-05-23T03:01:23.189318Z","shell.execute_reply.started":"2021-05-23T03:01:23.182872Z","shell.execute_reply":"2021-05-23T03:01:23.188301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:23.191201Z","iopub.execute_input":"2021-05-23T03:01:23.191594Z","iopub.status.idle":"2021-05-23T03:01:23.217113Z","shell.execute_reply.started":"2021-05-23T03:01:23.191554Z","shell.execute_reply":"2021-05-23T03:01:23.216201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.ticker as ticker\n\nncount = train_df.shape[0]\n\nplt.figure(figsize=(7, 5))\n\nax = sns.countplot(data=train_df, x='target')\nplt.title('Distribution of Questions')\nplt.xlabel('Number of Axles')\n\n# Make twin axis\nax2=ax.twinx()\n\n# Switch so count axis is on right, frequency on left\nax2.yaxis.tick_left()\nax.yaxis.tick_right()\n\n# Also switch the labels over\nax.yaxis.set_label_position('right')\nax2.yaxis.set_label_position('left')\n\nax2.set_ylabel('Frequency [%]')\n\nfor p in ax.patches:\n    x=p.get_bbox().get_points()[:,0]\n    y=p.get_bbox().get_points()[1,1]\n    ax.annotate('{:.1f}%'.format(100.*y/ncount), (x.mean(), y), \n            ha='center', va='bottom') # set the alignment of the text\n\n# Use a LinearLocator to ensure the correct number of ticks\nax.yaxis.set_major_locator(ticker.LinearLocator(11))\n\n# Fix the frequency range to 0-100\nax2.set_ylim(0,100)\nax.set_ylim(0,ncount)\n\n# And use a MultipleLocator to ensure a tick spacing of 10\nax2.yaxis.set_major_locator(ticker.MultipleLocator(10))\n\n# Need to turn the grid on ax2 off, otherwise the gridlines end up on top of the bars\nax2.grid(None)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:23.218313Z","iopub.execute_input":"2021-05-23T03:01:23.218626Z","iopub.status.idle":"2021-05-23T03:01:23.530622Z","shell.execute_reply.started":"2021-05-23T03:01:23.218593Z","shell.execute_reply":"2021-05-23T03:01:23.52957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\ninsincere_qes = train_df[train_df['target'] == 1]\nprint(insincere_qes[:5].question)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:23.532049Z","iopub.execute_input":"2021-05-23T03:01:23.532327Z","iopub.status.idle":"2021-05-23T03:01:23.569248Z","shell.execute_reply.started":"2021-05-23T03:01:23.532298Z","shell.execute_reply":"2021-05-23T03:01:23.568263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\nsincere_qes = train_df[train_df['target'] == 0]\nprint(sincere_qes[-5:].question)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:23.57042Z","iopub.execute_input":"2021-05-23T03:01:23.570691Z","iopub.status.idle":"2021-05-23T03:01:23.782562Z","shell.execute_reply.started":"2021-05-23T03:01:23.570664Z","shell.execute_reply":"2021-05-23T03:01:23.781271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['question_length'] = train_df['question'].apply(len)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:23.784025Z","iopub.execute_input":"2021-05-23T03:01:23.784411Z","iopub.status.idle":"2021-05-23T03:01:24.141398Z","shell.execute_reply.started":"2021-05-23T03:01:23.784368Z","shell.execute_reply":"2021-05-23T03:01:24.140363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:24.142699Z","iopub.execute_input":"2021-05-23T03:01:24.142962Z","iopub.status.idle":"2021-05-23T03:01:24.153504Z","shell.execute_reply.started":"2021-05-23T03:01:24.142938Z","shell.execute_reply":"2021-05-23T03:01:24.152473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.stem import SnowballStemmer\n\nstop_words = stopwords.words('english')\nstop_words.remove('not')\nstop_words.append('would')\nlemmatizer = WordNetLemmatizer()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:24.154938Z","iopub.execute_input":"2021-05-23T03:01:24.155277Z","iopub.status.idle":"2021-05-23T03:01:24.169807Z","shell.execute_reply.started":"2021-05-23T03:01:24.155247Z","shell.execute_reply":"2021-05-23T03:01:24.168688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"contraction_dict = {\"dont\": \"do not\", \"aint\": \"is not\", \"isnt\": \"is not\", \"doesnt\": \"does not\", \"cant\": \"cannot\", \"mustnt\": \"must not\", \"hasnt\": \"has not\", \"havent\": \"have not\", \"arent\": \"are not\", \"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"‘cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\", \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\", \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"Iam\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\", \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\", \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\"}\n\ndef replace_contractions(question):\n    return [contraction_dict.get(token) if (contraction_dict.get(token) != None) else token for token in question]","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:24.171134Z","iopub.execute_input":"2021-05-23T03:01:24.171425Z","iopub.status.idle":"2021-05-23T03:01:24.185516Z","shell.execute_reply.started":"2021-05-23T03:01:24.171398Z","shell.execute_reply":"2021-05-23T03:01:24.184619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\n\nnlp = spacy.load(\"en_core_web_lg\")","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:24.186753Z","iopub.execute_input":"2021-05-23T03:01:24.187031Z","iopub.status.idle":"2021-05-23T03:01:30.461525Z","shell.execute_reply.started":"2021-05-23T03:01:24.187006Z","shell.execute_reply":"2021-05-23T03:01:30.460718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\nimport gensim\n\narchive = zipfile.ZipFile('../input/quora-insincere-questions-classification/embeddings.zip', 'r')\n#print(archive.namelist()) # print all zip content\n\nembeddings = gensim.models.KeyedVectors.load_word2vec_format(archive.open('GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'), binary=True,limit=100000)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:30.462931Z","iopub.execute_input":"2021-05-23T03:01:30.463453Z","iopub.status.idle":"2021-05-23T03:01:32.165683Z","shell.execute_reply.started":"2021-05-23T03:01:30.463399Z","shell.execute_reply":"2021-05-23T03:01:32.164666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_contain_digit(word):\n    for i in word:\n        if i.isdigit():\n            return True\n    return False","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:32.166872Z","iopub.execute_input":"2021-05-23T03:01:32.167139Z","iopub.status.idle":"2021-05-23T03:01:32.170917Z","shell.execute_reply.started":"2021-05-23T03:01:32.167113Z","shell.execute_reply":"2021-05-23T03:01:32.170195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def qes_preprocessing(qes):\n    \n    # Data cleaning:\n    qes = re.sub(re.compile('<.*?>'), '', qes)\n    qes = re.sub('[^A-Za-z0-9]+', ' ', qes)\n\n    # Lowercase:\n    qes = qes.lower()\n    \n    # Tokenization:\n    tokens = word_tokenize(qes)\n\n    # Contractions replacement:\n    tokens = [contraction_dict.get(token) if (contraction_dict.get(token) != None) else token for token in tokens]\n\n    # Stop words removal:\n    tokens = [w for w in tokens if w not in stop_words]\n    \n    # Number removal:\n    tokens = [w for w in tokens if any(i.isdigit() for i in w) == False]\n    \n    # Lemmatization:\n    tokens = [lemmatizer.lemmatize(w) for w in tokens]\n\n    return ' '.join(tokens)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:32.171868Z","iopub.execute_input":"2021-05-23T03:01:32.17233Z","iopub.status.idle":"2021-05-23T03:01:32.187878Z","shell.execute_reply.started":"2021-05-23T03:01:32.172296Z","shell.execute_reply":"2021-05-23T03:01:32.187058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"found_synonyms = {}\n\ndef find_synonyms(word):\n    if word in found_synonyms:\n        replacements = found_synonyms[word]\n    else:\n        replacements = embeddings.most_similar(word)\n        found_synonyms[word] = replacements\n    rep_index = int(len(replacements) * random.random())\n    return replacements[rep_index][0].lower()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:32.188941Z","iopub.execute_input":"2021-05-23T03:01:32.189316Z","iopub.status.idle":"2021-05-23T03:01:32.262333Z","shell.execute_reply.started":"2021-05-23T03:01:32.189289Z","shell.execute_reply":"2021-05-23T03:01:32.261446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def qes_preprocessing_oversampling(qes): \n\n    # Data cleaning:\n    qes = re.sub(re.compile('<.*?>'), '', qes)\n    qes = re.sub('[^A-Za-z0-9]+', ' ', qes)\n\n    doc_ents = set([i.text.lower() for i in nlp(qes).ents])\n    \n    # Lowercase:\n    qes = qes.lower()\n    \n    # Tokenization:\n    tokens = word_tokenize(qes)\n\n    # Contractions replacement:\n    tokens = [contraction_dict.get(token) if (contraction_dict.get(token) != None) else token for token in tokens]\n\n    # Stop words removal:\n    tokens = [w for w in tokens if w not in stop_words]\n    \n    # Number removal:\n    tokens = [w for w in tokens if check_contain_digit(w) == False]\n    \n    # Lemmatization:\n    tokens = [lemmatizer.lemmatize(w) for w in tokens]\n    \n    final_ques = [' '.join(tokens)]\n    \n    for i in range(5):\n        if len(tokens) > 1:\n            # Number of replaced words\n            no_words = int(len(tokens) * random.random())\n            # Perform replacements\n            for x in range(no_words):\n                token_index = int(len(tokens) * random.random())\n                try:\n                    token = tokens[token_index]\n                    # Search for token in synonyms dict\n                    if token not in doc_ents:\n                        synonym = find_synonyms(token)  \n                        tokens[token_index] = synonym\n                        final_ques.append(' '.join(tokens))\n                except:\n                     pass\n        elif len(tokens) == 1:\n            try:\n                synonym = find_synonyms(token) \n                final_ques.append(synonym) \n            except:\n                pass\n        else:\n            break\n\n    return list(set(final_ques))","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:32.263305Z","iopub.execute_input":"2021-05-23T03:01:32.263676Z","iopub.status.idle":"2021-05-23T03:01:32.279705Z","shell.execute_reply.started":"2021-05-23T03:01:32.263646Z","shell.execute_reply":"2021-05-23T03:01:32.27883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embeddings.most_similar(\"adopt\")","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:32.281136Z","iopub.execute_input":"2021-05-23T03:01:32.281424Z","iopub.status.idle":"2021-05-23T03:01:32.400343Z","shell.execute_reply.started":"2021-05-23T03:01:32.281393Z","shell.execute_reply":"2021-05-23T03:01:32.399051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ntest = train_df['question'][:10].progress_apply(qes_preprocessing_oversampling)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:32.401882Z","iopub.execute_input":"2021-05-23T03:01:32.402255Z","iopub.status.idle":"2021-05-23T03:01:33.444806Z","shell.execute_reply.started":"2021-05-23T03:01:32.402213Z","shell.execute_reply":"2021-05-23T03:01:33.443801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[5]","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:33.446344Z","iopub.execute_input":"2021-05-23T03:01:33.447008Z","iopub.status.idle":"2021-05-23T03:01:33.453575Z","shell.execute_reply.started":"2021-05-23T03:01:33.446963Z","shell.execute_reply":"2021-05-23T03:01:33.452393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['question'].iloc[1]","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:33.455428Z","iopub.execute_input":"2021-05-23T03:01:33.456166Z","iopub.status.idle":"2021-05-23T03:01:33.469418Z","shell.execute_reply.started":"2021-05-23T03:01:33.456121Z","shell.execute_reply":"2021-05-23T03:01:33.468192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ninsincere_questions = train_df[train_df['target'] == 0]['question'].progress_apply(qes_preprocessing)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:01:33.471366Z","iopub.execute_input":"2021-05-23T03:01:33.472367Z","iopub.status.idle":"2021-05-23T03:05:12.396205Z","shell.execute_reply.started":"2021-05-23T03:01:33.472312Z","shell.execute_reply":"2021-05-23T03:05:12.395342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X1 = pd.DataFrame({\n    'target': np.zeros(insincere_questions.shape, dtype=int),\n    'question': insincere_questions\n})","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:05:12.397462Z","iopub.execute_input":"2021-05-23T03:05:12.397713Z","iopub.status.idle":"2021-05-23T03:05:12.423752Z","shell.execute_reply.started":"2021-05-23T03:05:12.397689Z","shell.execute_reply":"2021-05-23T03:05:12.422787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X1","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:05:12.424862Z","iopub.execute_input":"2021-05-23T03:05:12.425136Z","iopub.status.idle":"2021-05-23T03:05:12.437195Z","shell.execute_reply.started":"2021-05-23T03:05:12.42511Z","shell.execute_reply":"2021-05-23T03:05:12.436017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_questions.shape","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:05:12.441535Z","iopub.execute_input":"2021-05-23T03:05:12.441822Z","iopub.status.idle":"2021-05-23T03:05:12.452766Z","shell.execute_reply.started":"2021-05-23T03:05:12.441794Z","shell.execute_reply":"2021-05-23T03:05:12.451859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\nsincere_questions = train_df[train_df['target'] == 1]['question'].progress_apply(qes_preprocessing_oversampling)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:05:12.454603Z","iopub.execute_input":"2021-05-23T03:05:12.454902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_questions = sincere_questions.explode('question')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_questions.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_questions.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X2 = pd.DataFrame({\n    'target': np.ones(sincere_questions.shape, dtype=int),\n    'question': sincere_questions\n})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset = pd.concat([X1, X2])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset = trainset.dropna(how='any')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.linear_model import LogisticRegression\n\npipeline = Pipeline([(\"cv\", CountVectorizer(analyzer=\"word\", ngram_range=(1,4), max_df=0.9)),\n                     (\"clf\", LogisticRegression(solver=\"saga\", class_weight=\"balanced\", C=0.45, max_iter=600, verbose=1, n_jobs=-1))])\n\nX_train, X_test, y_train, y_test = train_test_split(trainset['question'], trainset.target, test_size=0.2, stratify = trainset.target.values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_model = pipeline.fit(X_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_fscore_matrix(fitted_clf, model_name):\n    print(model_name, ' :')\n    \n    # get classes predictions for the classification report \n    y_train_pred, y_pred = fitted_clf.predict(X_train), fitted_clf.predict(X_test)\n    print(classification_report(y_test, y_pred), '\\n') # target_names=y\n    \n    # computes probabilities keep the ones for the positive outcome only      \n    print(f'F1-score = {f1_score(y_test, y_pred):.2f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score, confusion_matrix, classification_report\n\nget_fscore_matrix(lr_model, 'Linear Regression')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['preprocessed'] = test_df['question_text'].apply(qes_preprocessing)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = lr_model.predict(test_df['preprocessed'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(predictions))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['prediction'] = predictions\nresults = test_df[['qid', 'prediction']]\nresults.to_csv('submission.csv', index=False)\nresults.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}