{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Importing all the required modules.","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport re\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nimport string\nfrom nltk.util import ngrams\nfrom collections import Counter\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings(\"ignore\")\npd.set_option('display.max_colwidth', 100)\n\n\nfrom nltk.stem import WordNetLemmatizer\n\nlemmatizer = WordNetLemmatizer()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> We will be drawing interactive charts so cufflink is one of my favourite liberary which is built on top of the matplotlib. Here I have selected offline mode, means it's scope will be bound to this notebook. You can even publish these graphs also. For more information visit https://plotly.com/","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import cufflinks as cf\nimport plotly.express as px\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\ninit_notebook_mode(connected=True)\n\ninit_notebook_mode(connected=True)\ncf.go_offline()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> > Let's start loading all csv files using pandas.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/test.csv\")\nsample_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = train_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> By looking at the count of records, it seems more than sufficient for neural network. Congratulation!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape, test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_dist = train_df.target.value_counts()\ntarget_dist = target_dist.reset_index().rename(columns={'index':\"Target Label\", 'target':'Count'})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Data class is very imbalanced as you can see in the below graph. Later will try to handle this class imbalance issue. For now let's move to the next destination. Yeah, but will be back to you soon.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.pie(target_dist, names='Target Label', values='Count', title=\"Target Label Distribution\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# seperating out label 0 and 1 data\nlabel1 = train_df[train_df.target==1]\nlabel0 = train_df[train_df.target==0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label0.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_stopwords(text):\n    count = 0\n    text = text.split(\" \")\n    text = [word for word in text if word in stopwords.words('english')]\n    # print(text)\n    return len(text)\n\ndef count_punct(text):\n    count = 0\n    text = [c for c in text if c in list(string.punctuation)]\n    # print(text)\n    return len(text)\n\ndef get_ngrams(text, n=2):\n    ngram = list(ngrams(text.split(), n=n))\n    df = pd.DataFrame.from_dict(data = dict(Counter(ngram)), orient='index')\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.reindex(np.random.permutation(train_df.index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_subset = train_df.iloc[:100000, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_subset['length_of_sent'] = train_df_subset.question_text.apply(len)\ntrain_df_subset[\"word_count\"] = train_df_subset.question_text.apply(lambda x: len(x.split(\" \")))\ntrain_df_subset['count_stopwords'] = train_df_subset.question_text.apply(lambda x : count_stopwords(x))\ntrain_df_subset['count_punctuation'] = train_df_subset.question_text.apply(lambda x : count_punct(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_subset.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_subset = train_df_subset.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"full_text = \"\"\nfor i in range(train_df_subset.shape[0]):\n    full_text = full_text + \" \" + train_df_subset.question_text[i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bigram_df = get_ngrams(full_text, n=2)\ntrigram_df = get_ngrams(full_text, n=3)\nquadgram_df = get_ngrams(full_text, n=4)\n\ndel full_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bigram_df = bigram_df.sort_values(by=0, ascending=False)\ntrigram_df = trigram_df.sort_values(by=0, ascending=False)\nquadgram_df = quadgram_df.sort_values(by=0, ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label1 = train_df_subset[train_df_subset.target==1]\nlabel0 = train_df_subset[train_df_subset.target==0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig1 = px.histogram(train_df_subset, x=\"length_of_sent\", color = \"target\",\n                   title = \"Length of Sentence Distribution\")\nfig1.show()\n\nfig2 = px.histogram(train_df_subset, x=\"word_count\", color = \"target\",\n                   title = \"Count of Word Distribution\")\nfig2.show()\n\nfig3 = px.histogram(train_df_subset, x=\"count_punctuation\", color = \"target\",\n                   title = \"Count of Punctuation Distribution\")\nfig3.show()\n\nfig4 = px.histogram(train_df_subset, x=\"count_stopwords\", color = \"target\",\n                   title = \"Count of Stopwords Distribution\")\nfig4.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label1_sort = label1.sort_values(by=\"length_of_sent\")\nlabel0_sort = label0.sort_values(by=\"length_of_sent\")\n\nfig1 = px.bar(label1_sort.head(20).reset_index(drop=True), y='length_of_sent', title=\"Short Lengths - 0\")\nfig2 = px.bar(label0_sort.head(20).reset_index(drop=True), y='length_of_sent', title=\"Short Lengths - 1\")\nfig1.show()\nfig2.show()\n\nfig3 = px.bar(label1_sort.tail(20).reset_index(drop=True), y='length_of_sent', title=\"Long Lengths - 1\")\nfig4 = px.bar(label0_sort.tail(20).reset_index(drop=True), y='length_of_sent', title=\"Long Lengths - 0\")\nfig3.show()\nfig4.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# bigram_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bigram_df = bigram_df.reset_index().rename(columns={'index':'Bigram', 0:'count'})\ntrigram_df = trigram_df.reset_index().rename(columns={'index':'Trigram', 0:'count'})\nquadgram_df = quadgram_df.reset_index().rename(columns={'index':'Quadgram', 0:'count'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bigram_top50= bigram_df.head(50)\nbigram_top50.Bigram = bigram_top50.Bigram.apply(lambda x: \" \".join(x))\n\ntrigram_top50= trigram_df.head(50)\ntrigram_top50.Trigram = trigram_top50.Trigram.apply(lambda x: \" \".join(x))\n\nquadgram_top50= quadgram_df.head(50)\nquadgram_top50.Quadgram = quadgram_top50.Quadgram.apply(lambda x: \" \".join(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig1 = bigram_top50.iplot(kind='barh', x='Bigram', y='count', title='Most Frequent top 20 Bigrams')\nfig2 = trigram_top50.iplot(kind='barh', x='Trigram', y='count', title='Most Frequent top 20 Trigrams')\nfig3 = quadgram_top50.iplot(kind='barh', x='Quadgram', y='count', title='Most Frequent top 20 Quadgram')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_original = df.question_text\nY_Original = df.target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Undersampling\n\ndf = df.sample(frac=1)\n\ndf_label0 = df[df.target==0].iloc[:80810, :]\ndf_label1 = df[df.target==1]\n\nundersampled_df = pd.concat([df_label0, df_label1])\nnew_df = undersampled_df.sample(frac=1, random_state=21)\n\nnew_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"undersampled_target_dist = new_df.target.value_counts().reset_index().rename(columns={'index':'Target Label', 'target':'Count'})\nfig = px.pie(undersampled_target_dist,  names='Target Label', values='Count', title=\"Target Label Distribution\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del bigram_df\ndel trigram_df\ndel quadgram_df\ndel bigram_top50\ndel trigram_top50\ndel quadgram_top50\ndel label0\ndel label0_sort\ndel label1\ndel label1_sort","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\",\n                       \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\",\n                       \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\n                       \"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \n                       \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\",\n                       \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\",\n                       \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\",\n                       \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\",\n                       \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\",\n                       \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\n                       \"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\",\n                       \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\n                       \"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\",\n                       \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\",\n                       \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\",\n                       \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\",\n                       \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\",\n                       \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\n                       \"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\",\n                       \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\",\n                       \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\",\n                       \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\",\n                       \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\",\n                       \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\",\n                       \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\",\n                       \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\",\n                       \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\n                       \"you'd\": \"you would\", \"you'd've\": \"you would have\",\n                       \"you'll\": \"you will\", \"you'll've\": \"you will have\",\n                       \"you're\": \"you are\", \"you've\": \"you have\" }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ! pip install autocorrect\n# ! pip install contractions\n# from autocorrect import Speller\n# import contractions\n# spell = Speller('en')\n\ndef clean_data(text):\n    text = \" \".join([contraction_mapping[word].lower() if word in contraction_mapping.keys() else word.lower() for word in text.split(' ')])\n    text = text.replace('http.*.com', '')\n    text = re.sub('<[^<]+?>','', text)\n    text = ''.join([c for c in text if c not in list(string.punctuation)])\n    text = ''.join(c for c in text if not c.isdigit())\n    text = ' '.join([lemmatizer.lemmatize(word) for word in text.split(\" \")])\n    # text = ' '.join([spell(w) for w in (nltk.word_tokenize(text))])\n    # text = contractions.fix(text)\n    # text = ' '.join([word for word in text.split(' ') if word not in stopwords.words('english')])\n    text = text.replace(\"’\", ' ')\n    text = text.replace('“', ' ')\n    text = re.sub(' +', ' ', text)\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# new_df.question_text = new_df.question_text.apply(lambda text: text.replace('http.*.com', ''))# ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# new_df.question_text = new_df.question_text.apply(lambda text: re.sub('<[^<]+?>','', text))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# new_df.question_text = new_df.question_text.apply(lambda text: ''.join([c for c in text if c not in list(string.punctuation)]))#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# new_df.question_text = new_df.question_text.apply(lambda text: ''.join(c for c in text if not c.isdigit()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# new_df.question_text = new_df.question_text.apply(lambda text: ' '.join([lemmatizer.lemmatize(word) for word in text.split(\" \")]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# new_df.question_text = new_df.question_text.apply(lambda text: contractions.fix(text))# ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# new_df.question_text = new_df.question_text.apply(lambda text: ' '.join([word for word in text.split(' ') if word not in stopwords.words('english')]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# new_df.question_text = new_df.question_text.apply(lambda text: text.replace(\"’\", ' '))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# new_df.question_text = new_df.question_text.apply(lambda text: text.replace('“', ' '))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# new_df.question_text = new_df.question_text.apply(lambda text: re.sub(' +', ' ', text))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_df.question_text = new_df.question_text.apply(lambda x : clean_data(x))\ntest_df.question_text = test_df.question_text.apply(lambda x : clean_data(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# new_df.to_pickle(\"training_undersampled_df.pkl\")\n# test_df.to_pickle(\"test_df.pkl\")\n\n# new_df = pd.read_pickle('training_undersampled_df.pkl')\n# test_df = pd.read_pickle('test_df.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_df.question_text = new_df.question_text.apply(lambda x: x.lower())\ntest_df.question_text = test_df.question_text.apply(lambda x: x.lower())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\narchive = zipfile.ZipFile('/kaggle/input/quora-insincere-questions-classification/embeddings.zip', 'r')\nnews_path=archive.open('GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', 'r')\n\n# from gensim.models import KeyedVectors\n# embeddings_index = KeyedVectors.load_word2vec_format(news_path, binary=True)\n\n# del embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(text):\n    text = text.split(\" \")\n    for word in text:\n        if word not in vocab:\n            vocab[word] = 1\n        else:\n            vocab[word] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator\ndef check_coverage(vocab, embeddings_index):\n    known_words = {}\n    unknown_words = {}\n    nb_known_words = 0\n    nb_unknown_words = 0\n    for word in vocab.keys():\n        try:\n            known_words[word] = embeddings_index[word]\n            nb_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n            pass\n\n    print('Found embeddings for {:.3%} of vocab'.format(len(known_words) / len(vocab)))\n    print('Found embeddings for  {:.3%} of all text'.format(nb_known_words / (nb_known_words + nb_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n    \n    return unknown_words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = {}\ntemp = new_df.question_text.apply(lambda x: build_vocab(x))\ndel temp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# oov = check_coverage(vocab, embeddings_index)\n# del oov","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del oov","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import io\nfrom tqdm import tqdm\nembeddings_glove={}\n\nwith zipfile.ZipFile(\"/kaggle/input/quora-insincere-questions-classification/embeddings.zip\") as zf:\n    with io.TextIOWrapper(zf.open(\"glove.840B.300d/glove.840B.300d.txt\"), encoding=\"utf-8\") as f:\n        for line in tqdm(f):\n            values=line.split(' ') # \".split(' ')\" only for glove-840b-300d; for all other files, \".split()\" works\n            word=values[0]\n            vectors=np.asarray(values[1:],'float32')\n            embeddings_glove[word]=vectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov_glove = check_coverage(vocab, embeddings_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov_glove","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# count = 0\n# for key, val in oov_glove:\n#     try:\n#         embeddings_glove[key] = embeddings_index[key]\n#     except:\n#         count += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# oov_glove = check_coverage(vocab, embeddings_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# len(oov_glove), len(vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nlength_of_vocab = 80000\nmax_len = 150\ndef tokenize(train, test):\n    tokenizer = Tokenizer(num_words = length_of_vocab)\n    tokenizer.fit_on_texts(train)\n    \n    X = tokenizer.texts_to_sequences(train)\n    X = pad_sequences(X, maxlen = max_len, padding =\"post\")\n    \n    X_test = tokenizer.texts_to_sequences(test)\n    X_test = pad_sequences(X_test, maxlen = max_len, padding =\"post\")\n    \n    return X, X_test, tokenizer.word_index, tokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X, X_test, word_index, tokenizer = tokenize(new_df.question_text, test_df.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, Y_train, Y_val = train_test_split(X, new_df.target, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_embeded_matrix(embeded_glove, word_index, length_of_vocab):\n    embed = np.stack(embeded_glove.values())\n    embed_mean, embed_std = embed.mean(), embed.std()\n    embed_size = embed.shape[1]\n    embeded_matrix = np.random.normal(embed_mean, embed_std, (length_of_vocab, embed_size))\n    \n    for word, index in word_index.items():\n        if index >= length_of_vocab:\n            continue\n        embeded_vector = embeded_glove.get(word)\n        if embeded_vector is not None:\n            embeded_matrix[index] = embeded_vector\n    return embeded_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_matrix = create_embeded_matrix(embeddings_glove, word_index, length_of_vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from tensorflow.python.compiler.tensorrt import trt_convert as trt\nfrom keras.models import Model\nfrom keras.layers import Dense,LSTM, Dropout,Conv1D, Embedding, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, Input, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom tensorflow.keras import regularizers\nfrom keras.regularizers import l2, l1, l1_l2\n# slim = tf.contrib.slim\n\ninp    = Input(shape=(max_len,))\nx      = Embedding(length_of_vocab, 300, weights=[embed_matrix], trainable=False)(inp)\nsdrop = SpatialDropout1D(rate=0.4)(x)\nb1 = Bidirectional(LSTM(128, return_sequences=True, kernel_regularizer=l1_l2(0.01), activity_regularizer=l1_l2(0.01)))(sdrop)\nconv1 = Conv1D(filters=64, kernel_size=3, activation='relu')(b1)\ngmax1_p = GlobalAveragePooling1D()(conv1)\nd1 = Dense(128, activation=\"relu\")(gmax1_p)\ndrop = Dropout(0.5)(d1)\nd2 = Dense(1, activation=\"sigmoid\")(drop)\n# b1 = keras.layers.BatchNormalization()\nmodel  = Model(inputs=inp, outputs=d2)\n\nmodel.compile(loss=\"binary_crossentropy\", optimizer=Adam(lr=0.01), metrics=['accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(X_train, Y_train, batch_size=256, epochs=5, verbose=1, validation_split=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.evaluate(x = X_val, y = Y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict = np.where(predict < 0.5, 0, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = pd.DataFrame()\nresult['qid'] = test_df.qid\nresult['prediction'] = predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}