{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n#from wordcloud import WordCloud, STOPWORDS\nimport pyprind \nimport warnings\nwarnings.filterwarnings('ignore')\nimport gc\nimport re\nimport gensim\nfrom gensim.models import KeyedVectors\nimport operator\nimport string\nimport tensorflow as tf\nimport json\nfrom tqdm import tqdm, tqdm_notebook\ntqdm_notebook().pandas()\nfrom zipfile import ZipFile\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.layers import Add,Conv1D, GlobalAveragePooling1D, GlobalMaxPooling1D, Flatten,Dense, Embedding, Concatenate, Input, Dropout,Bidirectional, GRU, GlobalAveragePooling1D, GlobalMaxPooling1D, LeakyReLU, Activation, LSTM, SpatialDropout1D, BatchNormalization\n#Dense, Embedding, Bidirectional, CuDNNGRU, GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, Input, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau, BaseLogger\nfrom tensorflow.keras import backend as K\nfrom sklearn.model_selection import train_test_split\n#from sklearn.metrics import precision_recall_fscore_support\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/google-quest-challenge/train.csv', header=0,encoding='utf-8')\ntest = pd.read_csv('../input/google-quest-challenge/test.csv', header=0,encoding='utf-8')\ndf = pd.concat([train,test],axis=0,ignore_index=True)\nprint(f'''Train Shape: {train.shape}\nTest Shape: {test.shape}\nDf Shape:{df.shape}''')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df['treated_question_title'] = df['question_title'].progress_apply(lambda x: x.lower().replace('\\n',' ').strip())\n# df['treated_question_body'] = df['question_body'].progress_apply(lambda x: x.lower().replace('\\n',' ').strip())\n# df['treated_answer'] = df['answer'].progress_apply(lambda x: x.lower().replace('\\n',' ').strip())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def separate_puncts(sentence, category):\n    if category=='STACKOVERFLOW' or  category=='TECHNOLOGY':\n        sentence = re.sub(r\"(<.*?>)|(/*.*?\\*/)|(for.*?\\+\\+)|(lt;.*?}})|(gt\\s.*?}})|({\\s.*?})|([&\\s]lt;.*?gt[;\\s])\",' programming code ',sentence)\n        sentence = re.sub(r\"(programming code[\\s\\W]*)\", ' programming code ',sentence)\n    sentence = re.sub(r'http\\S+', 'url', sentence)\n    emojis=re.findall(r'\\s(?::|;|=|\\^)(?:-|\\s)?(?:\\)|\\\\|\\/|\\(|D|P|\\^|\\|)\\s',sentence)\n    for i in emojis:\n        sentence = sentence.replace(i,'')\n    sentence = re.sub(r\"(\\$\\$.*?\\$\\$)\", ' mathematical formula ', sentence)\n    sentence = re.sub(r\"((?<=\\w)[^\\s\\w'?\\-](?![^\\s\\w]))|([^\\s\\w'?\\-](?![^\\s\\w])(?=\\w))\",' ',sentence)\n    sentence = re.sub(r\"((?<=\\w)[.?\\/\\-])|([.?\\/\\-](?=\\w))\",' \\g<1>\\g<2> ',sentence)\n    return ' '.join([sentence]+emojis)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(sentences, vocab={}, verbose=True):\n    sentences = sentences.apply(lambda x: x.split()).values\n    #vocab={}\n    for sentence in pyprind.prog_bar(sentences):\n        for word in sentence:\n            try:\n                vocab[word] +=1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def load_embed(file):\n#     def get_coefs(word, *arr):\n#         return word, np.array(arr, dtype='float16')\n    \n#     if file == '../input/quora-insincere-questions-classification/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':\n#         embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(file,encoding='utf-8') if len(o)>100)\n    \n#     elif file == '../input/quora-insincere-questions-classification/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin':\n#         embeddings_index = KeyedVectors.load_word2vec_format(file, binary=True)\n    \n#     elif file == '../input/quora-insincere-questions-classification/embeddings.zip/paragram_300_sl999/paragram_300_sl999.txt':\n#         embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin') if len(o)>100)\n        \n#     elif file == '../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt':\n#         embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n#     return embeddings_index\n\n\n\n# glove = '../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'\n# paragram =  '..//input/quora-insincere-questions-classification/embeddings.zip/paragram_300_sl999/paragram_300_sl999.txt'\n# wiki_news = '../input/quora-insincere-questions-classification/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n# google_embed = '../input/quora-insincere-questions-classification/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_embed(file, name):\n    def get_coefs(word, *arr):\n        return word, np.array(arr, dtype='float16')\n    \n    if name == 'wiki':\n        embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(file,encoding='utf-8') if len(o)>100)\n    \n    elif name == 'word2vec':\n        embeddings_index = KeyedVectors.load_word2vec_format(file, binary=True)\n    \n    elif name == 'paragram':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin') if len(o)>100)\n        \n    elif name == 'glove':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# with ZipFile('../input/quora-insincere-questions-classification/embeddings.zip','r') as f:\n#     f.printdir()\n#     f.extract('paragram_300_sl999/paragram_300_sl999.txt')\n#     print('Done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(\"Extracting GloVe embedding\")\n# embed_glove = load_embed(glove)\nprint(\"Extracting Paragram embedding\")\nembed_paragram = load_embed('../input/paragram-300-sl999/paragram_300_sl999/paragram_300_sl999/paragram_300_sl999.txt','paragram')\n\n# print(\"Extracting Wiki embedding\")\n# embed_wiki = load_embed(wiki_news)\n# print(\"Extracting Google embedding\")\n# embed_google = load_embed(google_embed)\nprint('Done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(embed_paragram)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.remove('/kaggle/working/paragram_300_sl999/paragram_300_sl999.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_coverage(vocab,embeddings_index):\n    known_words={}\n    unknown_words={}\n    nb_known_words=0\n    nb_unknown_words=0\n    for word in vocab.keys():\n        if word in embeddings_index:\n            known_words[word]=embeddings_index[word]\n            nb_known_words += vocab[word]\n        elif word.capitalize() in embeddings_index:\n            known_words[word] = embeddings_index[word.capitalize()]\n            nb_known_words += vocab[word]\n        elif word.lower() in embeddings_index:\n            known_words[word] = embeddings_index[word.lower()]\n            nb_known_words += vocab[word]\n        elif word.upper() in embeddings_index:\n            known_words[word] = embeddings_index[word.upper()]\n            nb_known_words += vocab[word]\n        else:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n    print(f'Found embeddings for {round((len(known_words)/len(vocab))*100,5)}% of the vocab\\nFound embeddings for {round((nb_known_words/(nb_known_words+nb_unknown_words))*100,5)}% of all text')\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1),reverse=True)#[::-1]\n    return unknown_words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# len(vocab),sorted(vocab.items(),key = operator.itemgetter(1),reverse=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"contraction_mapping = {\"\\'\":\"'\", #,\"..\":'',\n    \"Trump's\" : 'trump is',\"'cause\": 'because',\"’\": \"'\",\",cause\": 'because',\";cause\": 'because',\"ain't\": 'am not',\"ain,t\": 'am not',\n    \"ain;t\": 'am not',\"ain't\": 'am not',\"ain’t\": 'am not',\"aren't\": 'are not',\"–\": '-',\"“\":'\"',\n    \"aren,t\": 'are not',\"aren;t\": 'are not',\"aren't\": 'are not',\"aren’t\": 'are not',\"can't\": 'cannot',\"can't've\": 'cannot have',\"can,t\": 'cannot',\"can,t,ve\": 'cannot have',\n    \"can;t\": 'cannot',\"can;t;ve\": 'cannot have',\n    \"can't\": 'cannot',\"can't´ve\": 'cannot have',\"can’t\": 'cannot',\"can’t’ve\": 'cannot have',\n    \"could've\": 'could have',\"could,ve\": 'could have',\"could;ve\": 'could have',\"couldn't\": 'could not',\"couldn't've\": 'could not have',\"couldn,t\": 'could not',\"couldn,t,ve\": 'could not have',\"couldn;t\": 'could not',\n    \"couldn;t;ve\": 'could not have',\"couldn't\": 'could not',\n    \"couldn't´ve\": 'could not have',\"couldn’t\": 'could not',\"couldn’t’ve\": 'could not have',\"could´ve\": 'could have',\n    \"could’ve\": 'could have',\"didn't\": 'did not',\"didn,t\": 'did not',\"didn;t\": 'did not',\"didn't\": 'did not',\n    \"didn’t\": 'did not',\"doesn't\": 'does not',\"doesn,t\": 'does not',\"doesn;t\": 'does not',\"doesn't\": 'does not',\n    \"doesn’t\": 'does not',\"don't\": 'do not',\"don,t\": 'do not',\"don;t\": 'do not',\"don't\": 'do not',\"don’t\": 'do not',\n    \"hadn't\": 'had not',\"hadn't've\": 'had not have',\"hadn,t\": 'had not',\"hadn,t,ve\": 'had not have',\"hadn;t\": 'had not',\n    \"hadn;t;ve\": 'had not have',\"hadn't\": 'had not',\"hadn't´ve\": 'had not have',\"hadn’t\": 'had not',\"hadn’t’ve\": 'had not have',\"hasn't\": 'has not',\"hasn,t\": 'has not',\"hasn;t\": 'has not',\"hasn't\": 'has not',\"hasn’t\": 'has not',\n    \"haven't\": 'have not',\"haven,t\": 'have not',\"haven;t\": 'have not',\"haven't\": 'have not',\"haven’t\": 'have not',\"he'd\": 'he would',\n    \"he'd've\": 'he would have',\"he'll\": 'he will',\n    \"he's\": 'he is',\"he,d\": 'he would',\"he,d,ve\": 'he would have',\"he,ll\": 'he will',\"he,s\": 'he is',\"he;d\": 'he would',\n    \"he;d;ve\": 'he would have',\"he;ll\": 'he will',\"he;s\": 'he is',\"he'd\": 'he would',\"he'd've\": 'he would have',\"he´ll\": 'he will',\n    \"he´s\": 'he is',\"he’d\": 'he would',\"he’d’ve\": 'he would have',\"he’ll\": 'he will',\"he’s\": 'he is',\"how'd\": 'how did',\"how'll\": 'how will',\n    \"how's\": 'how is',\"how,d\": 'how did',\"how,ll\": 'how will',\"how,s\": 'how is',\"how;d\": 'how did',\"how;ll\": 'how will',\n    \"how;s\": 'how is',\"how´d\": 'how did',\"how´ll\": 'how will',\"how´s\": 'how is',\"how’d\": 'how did',\"how’ll\": 'how will',\n    \"how’s\": 'how is',\"i'd\": 'i would',\"i'll\": 'i will',\"i'm\": 'i am',\"i've\": 'i have',\"i,d\": 'i would',\"i,ll\": 'i will',\n    \"i,m\": 'i am',\"i,ve\": 'i have',\"i;d\": 'i would',\"i;ll\": 'i will',\"i;m\": 'i am',\"i;ve\": 'i have',\"isn't\": 'is not',\n    \"isn,t\": 'is not',\"isn;t\": 'is not',\"isn't\": 'is not',\"isn’t\": 'is not',\"it'd\": 'it would',\"it'll\": 'it will',\"It's\":'it is',\n    \"it's\": 'it is',\"it,d\": 'it would',\"it,ll\": 'it will',\"it,s\": 'it is',\"it;d\": 'it would',\"it;ll\": 'it will',\"it;s\": 'it is',\"it´d\": 'it would',\"it´ll\": 'it will',\"it´s\": 'it is',\n    \"it’d\": 'it would',\"it’ll\": 'it will',\"it’s\": 'it is',\n    \"i´d\": 'i would',\"i´ll\": 'i will',\"i´m\": 'i am',\"i´ve\": 'i have',\"i’d\": 'i would',\"i’ll\": 'i will',\"i’m\": 'i am',\n    \"i’ve\": 'i have',\"let's\": 'let us',\"let,s\": 'let us',\"let;s\": 'let us',\"let´s\": 'let us',\n    \"let’s\": 'let us',\"ma'am\": 'madam',\"ma,am\": 'madam',\"ma;am\": 'madam',\"mayn't\": 'may not',\"mayn,t\": 'may not',\"mayn;t\": 'may not',\n    \"mayn't\": 'may not',\"mayn’t\": 'may not',\"ma´am\": 'madam',\"ma’am\": 'madam',\"might've\": 'might have',\"might,ve\": 'might have',\"might;ve\": 'might have',\"mightn't\": 'might not',\"mightn,t\": 'might not',\"mightn;t\": 'might not',\"mightn't\": 'might not',\n    \"mightn’t\": 'might not',\"might´ve\": 'might have',\"might’ve\": 'might have',\"must've\": 'must have',\"must,ve\": 'must have',\"must;ve\": 'must have',\n    \"mustn't\": 'must not',\"mustn,t\": 'must not',\"mustn;t\": 'must not',\"mustn't\": 'must not',\"mustn’t\": 'must not',\"must´ve\": 'must have',\n    \"must’ve\": 'must have',\"needn't\": 'need not',\"needn,t\": 'need not',\"needn;t\": 'need not',\"needn't\": 'need not',\"needn’t\": 'need not',\"oughtn't\": 'ought not',\"oughtn,t\": 'ought not',\"oughtn;t\": 'ought not',\n    \"oughtn't\": 'ought not',\"oughtn’t\": 'ought not',\"sha'n't\": 'shall not',\"sha,n,t\": 'shall not',\"sha;n;t\": 'shall not',\"shan't\": 'shall not',\n    \"shan,t\": 'shall not',\"shan;t\": 'shall not',\"shan't\": 'shall not',\"shan’t\": 'shall not',\"sha´n't\": 'shall not',\"sha’n’t\": 'shall not',\n    \"she'd\": 'she would',\"she'll\": 'she will',\"she's\": 'she is',\"she,d\": 'she would',\"she,ll\": 'she will',\n    \"she,s\": 'she is',\"she;d\": 'she would',\"she;ll\": 'she will',\"she;s\": 'she is',\"she´d\": 'she would',\"she´ll\": 'she will',\n    \"she´s\": 'she is',\"she’d\": 'she would',\"she’ll\": 'she will',\"she’s\": 'she is',\"should've\": 'should have',\"should,ve\": 'should have',\"should;ve\": 'should have',\n    \"shouldn't\": 'should not',\"shouldn,t\": 'should not',\"shouldn;t\": 'should not',\"shouldn't\": 'should not',\"shouldn’t\": 'should not',\"should´ve\": 'should have',\n    \"should’ve\": 'should have',\"that'd\": 'that would',\"that's\": 'that is',\"that,d\": 'that would',\"that,s\": 'that is',\"that;d\": 'that would',\n    \"that;s\": 'that is',\"that´d\": 'that would',\"that´s\": 'that is',\"that’d\": 'that would',\"that’s\": 'that is',\"there'd\": 'there had',\n    \"there's\": 'there is',\"there,d\": 'there had',\"there,s\": 'there is',\"there;d\": 'there had',\"there;s\": 'there is',\n    \"there´d\": 'there had',\"there´s\": 'there is',\"there’d\": 'there had',\"there’s\": 'there is',\n    \"they'd\": 'they would',\"they'll\": 'they will',\"they're\": 'they are',\"they've\": 'they have',\n    \"they,d\": 'they would',\"they,ll\": 'they will',\"they,re\": 'they are',\"they,ve\": 'they have',\"they;d\": 'they would',\"they;ll\": 'they will',\"they;re\": 'they are',\n    \"they;ve\": 'they have',\"they´d\": 'they would',\"they´ll\": 'they will',\"they´re\": 'they are',\"they´ve\": 'they have',\"they’d\": 'they would',\"they’ll\": 'they will',\n    \"they’re\": 'they are',\"they’ve\": 'they have',\"wasn't\": 'was not',\"wasn,t\": 'was not',\"wasn;t\": 'was not',\"wasn't\": 'was not',\n    \"wasn’t\": 'was not',\"we'd\": 'we would',\"we'll\": 'we will',\"we're\": 'we are',\"we've\": 'we have',\"we,d\": 'we would',\"we,ll\": 'we will',\n    \"we,re\": 'we are',\"we,ve\": 'we have',\"we;d\": 'we would',\"we;ll\": 'we will',\"we;re\": 'we are',\"we;ve\": 'we have',\n    \"weren't\": 'were not',\"weren,t\": 'were not',\"weren;t\": 'were not',\"weren't\": 'were not',\"weren’t\": 'were not',\"we´d\": 'we would',\"we´ll\": 'we will',\n    \"we´re\": 'we are',\"we´ve\": 'we have',\"we’d\": 'we would',\"we’ll\": 'we will',\"we’re\": 'we are',\"we’ve\": 'we have',\"what'll\": 'what will',\"what're\": 'what are',\"what's\": 'what is',\n    \"what've\": 'what have',\"what,ll\": 'what will',\"what,re\": 'what are',\"what,s\": 'what is',\"what,ve\": 'what have',\"what;ll\": 'what will',\"what;re\": 'what are',\n    \"what;s\": 'what is',\"what;ve\": 'what have',\"what´ll\": 'what will',\n    \"what´re\": 'what are',\"what´s\": 'what is',\"what´ve\": 'what have',\"what’ll\": 'what will',\"what’re\": 'what are',\"what’s\": 'what is',\n    \"what’ve\": 'what have',\"where'd\": 'where did',\"where's\": 'where is',\"where,d\": 'where did',\"where,s\": 'where is',\"where;d\": 'where did',\n    \"where;s\": 'where is',\"where´d\": 'where did',\"where´s\": 'where is',\"where’d\": 'where did',\"where’s\": 'where is',\n    \"who'll\": 'who will',\"who's\": 'who is',\"who,ll\": 'who will',\"who,s\": 'who is',\"who;ll\": 'who will',\"who;s\": 'who is',\n    \"who´ll\": 'who will',\"who´s\": 'who is',\"who’ll\": 'who will',\"who’s\": 'who is',\"won't\": 'will not',\"won,t\": 'will not',\"won;t\": 'will not',\n    \"won't\": 'will not',\"won't\": 'will not',\"wouldn't\": 'would not',\"wouldn,t\": 'would not',\"wouldn;t\": 'would not',\"wouldn't\": 'would not',\n    \"wouldn't\": 'would not',\"you'd\": 'you would',\"you'll\": 'you will',\"you're\": 'you are',\"you,d\": 'you would',\"you,ll\": 'you will',\n    \"you,re\": 'you are',\"you;d\": 'you would',\"you;ll\": 'you will',\n    \"you;re\": 'you are',\"you´d\": 'you would',\"you´ll\": 'you will',\"you´re\": 'you are',\"you’d\": 'you would',\"you’ll\": 'you will',\"you’re\": 'you are',\n    \"´cause\": 'because',\"’cause\": 'because',\"you've\": \"you have\",\"could'nt\": 'could not',\n    \"havn't\": 'have not',\"here’s\": \"here is\",\"i'm\": 'i am' ,\"i'l\": \"i will\",\"i'v\": 'i have',\"wan't\": 'want',\"was'nt\": \"was not\",\"who'd\": \"who would\",\n    \"who're\": \"who are\",\"who've\": \"who have\",\"why'd\": \"why would\",\"would've\": \"would have\",\"y'all\": \"you all\",\"y'know\": \"you know\",\"you.i\": \"you i\",\n    \"your'e\": \"you are\",\"arn't\": \"are not\",\"agains't\": \"against\",\"c'mon\": \"common\",\"doens't\": \"does not\",\"don't\": \"do not\",\"dosen't\": \"does not\",\n    \"dosn't\": \"does not\",\"shoudn't\": \"should not\",\"that'll\": \"that will\",\"there'll\": \"there will\",\"there're\": \"there are\",\n    \"this'll\": \"this all\",\"u're\": \"you are\", \"ya'll\": \"you all\",\"you'r\": \"you are\",\"you’ve\": \"you have\",\"d'int\": \"did not\",\"did'nt\": \"did not\",\"din't\": \"did not\",\"dont't\": \"do not\",\"gov't\": \"government\",\n    \"i'ma\": \"i am\",\"is'nt\": \"is not\",\"‘I\":'I',\"ᴀɴᴅ\":'and',\"ᴛʜᴇ\":'the',\"ʜᴏᴍᴇ\":'home',\"ᴜᴘ\":'up',\"ʙʏ\":'by',\"ᴀᴛ\":'at',\"…and\":'and',\"civilbeat\":'civil beat',\"TrumpCare\":'Trump care',\"Trumpcare\":'Trump care', \"OBAMAcare\":'Obama care',\"ᴄʜᴇᴄᴋ\":'check',\"ғᴏʀ\":'for',\"ᴛʜɪs\":'this',\"ᴄᴏᴍᴘᴜᴛᴇʀ\":'computer',\"ᴍᴏɴᴛʜ\":'month',\"ᴡᴏʀᴋɪɴɢ\":'working',\"ᴊᴏʙ\":'job',\"ғʀᴏᴍ\":'from',\"Sᴛᴀʀᴛ\":'start',\"gubmit\":'submit',\"CO₂\":'carbon dioxide',\"ғɪʀsᴛ\":'first',\"ᴇɴᴅ\":'end',\"ᴄᴀɴ\":'can',\"ʜᴀᴠᴇ\":'have',\"ᴛᴏ\":'to',\"ʟɪɴᴋ\":'link',\"ᴏғ\":'of',\"ʜᴏᴜʀʟʏ\":'hourly',\"ᴡᴇᴇᴋ\":'week',\"ᴇɴᴅ\":'end',\"ᴇxᴛʀᴀ\":'extra',\"Gʀᴇᴀᴛ\":'great',\"sᴛᴜᴅᴇɴᴛs\":'student',\"sᴛᴀʏ\":'stay',\"ᴍᴏᴍs\":'mother',\"ᴏʀ\":'or',\"ᴀɴʏᴏɴᴇ\":'anyone',\"ɴᴇᴇᴅɪɴɢ\":'needing',\"ᴀɴ\":'an',\"ɪɴᴄᴏᴍᴇ\":'income',\n    \"ʀᴇʟɪᴀʙʟᴇ\":'reliable',\"ғɪʀsᴛ\":'first',\"ʏᴏᴜʀ\":'your',\"sɪɢɴɪɴɢ\":'signing',\"ʙᴏᴛᴛᴏᴍ\":'bottom',\"ғᴏʟʟᴏᴡɪɴɢ\":'following',\"Mᴀᴋᴇ\":'make',\n    \"ᴄᴏɴɴᴇᴄᴛɪᴏɴ\":'connection',\"ɪɴᴛᴇʀɴᴇᴛ\":'internet',\"financialpost\":'financial post', \"ʜaᴠᴇ\":' have ', \"ᴄaɴ\":' can ', \"Maᴋᴇ\":' make ', \"ʀᴇʟɪaʙʟᴇ\":' reliable ', \"ɴᴇᴇᴅ\":' need ',\n    \"ᴏɴʟʏ\":' only ', \"ᴇxᴛʀa\":' extra ', \"aɴ\":' an ', \"aɴʏᴏɴᴇ\":' anyone ', \"sᴛaʏ\":' stay ', \"Sᴛaʀᴛ\":' start', \"SHOPO\":'shop',\n    \" :-/ \":'Perplexed smilee',\" :/ \":'Perplexed smilee',\" ;-/ \":'Perplexed',\" ;/ \":'Perplexed', \" ;/ \":'Perplexed', \" :/ \":'Perplexed',\n    \" =( \":'Sad',\" :-( \":'Sad',\" :( \":'Sad', \"=gt\":'=>', \"=gt\":'=>', \"n't\":\" not\", \"'s\": \" is\", 'amp;amp':'ampersand', '..':'', '////':'', 'nbsp;':'',\n    '\\\\\\\\':'', \"usepackage\" : \"use package\",'instrumentsettingsid':'instrumental settings id','rippleshaderProgram' : 'ripple shader program','shaderprogramconstants':'shader program constants','storedElements':'stored elements','stackSize' : 'stack size'                   \n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#bar = pyprind.ProgBar(df.shape[0], bar_char='█')\ndef clean_contractions(text, mapping):\n    #text = text.split()\n    #text_val = text.lower()\n    #map_val = \n    for word in mapping.keys():\n        word_val = word.lower()\n        if word in text:\n            text = text.replace(word, mapping[word])\n        elif word_val in text:\n            text = text.replace(word_val, mapping[word])\n#         elif word.lower() in text:\n#             text = text.replace(word, mapping[word])\n#         elif word.capitalize() in text:\n#             text = text.replace(word, mapping[word])\n#         elif word.upper() in text:\n#             text = text.replace(word, mapping[word])\n\n#     bar.update()\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df['treated_question_title'] = df['treated_question_title'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))\n# df['treated_question_body'] = df['treated_question_body'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))\n# df['treated_answer'] = df['treated_answer'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"punct = \"-?!.,#$%\\*+<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'\npunct_mapping = {\"‘\": \"'\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': ' divided by ', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(embed_paragram['smilee'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def unknown_punct(embed, punct):\n    unknown_punct=''\n    for val in punct:\n        if val not in embed:\n            unknown_punct += val+' '\n    return unknown_punct\n# print(f'Glove:\\n{unknown_punct(embed_glove, punct)}')\n# print(f'Paragram:\\n{unknown_punct(embed_paragram, punct)}')\n# print(f'Wiki:\\n{unknown_punct(embed_wiki, punct)}')\n# print(f'Google:\\n{unknown_punct(embed_google, punct)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_puncts(text, mapping, punct):\n    for p in punct_mapping.keys():\n        text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n#     for word in text:\n#         for p in '!\"#$%&\\()*+,-./:;<=>?@[\\\\]^_`{|}~':\n#             word = word.replace(p, f' {p} ')\n    \n#     specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}\n#     for p in specials.keys():\n#         text = text.replace(p, specials[p])\n#    bar.update()    \n    return text\n\n# df['treated_question_title'] = df['treated_question_title'].progress_apply(lambda x: clean_puncts(x, punct_mapping, punct))\n# df['treated_question_body'] = df['treated_question_body'].progress_apply(lambda x: clean_puncts(x, punct_mapping, punct))\n# df['treated_answer'] = df['treated_answer'].progress_apply(lambda x: clean_puncts(x, punct_mapping, punct))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['treated_question_title'] = df['question_title'].apply(lambda x: x.lower().replace('\\n',' ').strip())\ndf['treated_question_body'] = df['question_body'].apply(lambda x: x.lower().replace('\\n',' ').strip())\ndf['treated_answer'] = df['answer'].apply(lambda x: x.lower().replace('\\n',' ').strip())\nprint('Stripping Done')\n\ndf['treated_question_title'] = df[['treated_question_title','category']].progress_apply(lambda x: separate_puncts(x['treated_question_title'], x['category']),axis=1)\ndf['treated_question_body'] = df[['treated_question_body','category']].progress_apply(lambda x: separate_puncts(x['treated_question_body'], x['category']),axis=1)\ndf['treated_answer'] = df[['treated_answer','category']].progress_apply(lambda x: separate_puncts(x['treated_answer'], x['category']),axis=1)\nprint('Cleaning Data Done')\n\ndf['treated_question_title'] = df['treated_question_title'].apply(lambda x: clean_contractions(x, contraction_mapping))\ndf['treated_question_body'] = df['treated_question_body'].apply(lambda x: clean_contractions(x, contraction_mapping))\ndf['treated_answer'] = df['treated_answer'].apply(lambda x: clean_contractions(x, contraction_mapping))\nprint('Contraction Mapping Done')\n\ndf['treated_question_title'] = df['treated_question_title'].apply(lambda x: clean_puncts(x, punct_mapping, punct))\ndf['treated_question_body'] = df['treated_question_body'].apply(lambda x: clean_puncts(x, punct_mapping, punct))\ndf['treated_answer'] = df['treated_answer'].apply(lambda x: clean_puncts(x, punct_mapping, punct))\nprint('Clean Punctuation Done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = build_vocab(df['treated_question_title'])\nprint(len(vocab))\nvocab = build_vocab(df['treated_question_body'], vocab=vocab)\nprint(len(vocab))\nvocab = build_vocab(df['treated_answer'], vocab=vocab)\nprint(len(vocab))\n# print(\"Glove : \")\n# oov_glove = check_coverage(vocab, embed_glove)\nprint(\"Paragram : \")\noov_paragram = check_coverage(vocab, embed_paragram)\n\n# print(\"Wiki : \")\n# oov_glove = check_coverage(vocab, embed_wiki)\n# print(\"Google : \")\n# oov_paragram = check_coverage(vocab, embed_google)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import traceback\ntraceback.extract_stack()[-2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"find = re.compile(r\"^[^.]*\")\ndf['web_category'] = df['url'].apply(lambda x: re.findall(find, x)[0].replace('http://',''))\ndf[['url', 'web_category']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.groupby(['category','web_category']).size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in df['category'].unique():\n    print(i, len(df[df['category']==i]['web_category'].unique()))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dums = pd.get_dummies(df[['category','web_category']])\ndf = pd.concat([df,dums], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df.loc[500, ['category','web_category','category_STACKOVERFLOW','web_category_stackoverflow']]\n\ndf.drop(columns=['question_title','question_body','answer', 'category','web_category'],inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = df[df['answer_helpful'].notna()]\nX_test = df[df['answer_helpful'].isna()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"title_max, title_mean = df['treated_question_title'].apply(lambda x: x.split()).map(len).max(), df['treated_question_title'].apply(lambda x: x.split()).map(len).mean()\nbody_max, body_mean = df['treated_question_body'].apply(lambda x: x.split()).map(len).max(), df['treated_question_body'].apply(lambda x: x.split()).map(len).mean() \nans_max, ans_mean = df['treated_answer'].apply(lambda x: x.split()).map(len).max(), df['treated_answer'].apply(lambda x: x.split()).map(len).mean() \nprint(f'Df Max Title Size is:\\t{title_max}\\twith average size being:\\t{title_mean}')\nprint(f'Df Max Body Size is:\\t{body_max}\\twith average size being:\\t{body_mean}')\nprint(f'Df Max Ans Size is:\\t{ans_max}\\twith average size being:\\t{ans_mean}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"title_max, title_mean = X_train['treated_question_title'].apply(lambda x: x.split()).map(len).max(), X_train['treated_question_title'].apply(lambda x: x.split()).map(len).mean()\nbody_max, body_mean = X_train['treated_question_body'].apply(lambda x: x.split()).map(len).max(), X_train['treated_question_body'].apply(lambda x: x.split()).map(len).mean() \nans_max, ans_mean = X_train['treated_answer'].apply(lambda x: x.split()).map(len).max(), X_train['treated_answer'].apply(lambda x: x.split()).map(len).mean() \nprint(f'X_train Max Title Size is:\\t{title_max}\\twith average size being:\\t{title_mean}')\nprint(f'X_train Max Body Size is:\\t{body_max}\\twith average size being:\\t{body_mean}')\nprint(f'X_train Max Ans Size is:\\t{ans_max}\\twith average size being:\\t{ans_mean}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"title_max, title_mean = X_test['treated_question_title'].apply(lambda x: x.split()).map(len).max(), X_test['treated_question_title'].apply(lambda x: x.split()).map(len).mean()\nbody_max, body_mean = X_test['treated_question_body'].apply(lambda x: x.split()).map(len).max(), X_test['treated_question_body'].apply(lambda x: x.split()).map(len).mean() \nans_max, ans_mean = X_test['treated_answer'].apply(lambda x: x.split()).map(len).max(), X_test['treated_answer'].apply(lambda x: x.split()).map(len).mean() \nprint(f'X_test Max Title Size is:\\t{title_max}\\twith average size being:\\t{title_mean}')\nprint(f'X_test Max Body Size is:\\t{body_max}\\twith average size being:\\t{body_mean}')\nprint(f'X_test Max Answ Size is:\\t{ans_max}\\twith average size being:\\t{ans_mean}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"maxlen = 250\nlength_to_keep = 1000000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def length_maker(text, maxlen=maxlen):\n    text = text.split()\n    if len(text)>220:\n        text = text[:125]+text[-125:]\n    return ' '.join(text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['treated_question_title'] = df['treated_question_title'].progress_apply(lambda x: length_maker(x))\ndf['treated_question_body'] = df['treated_question_body'].progress_apply(lambda x: length_maker(x))\ndf['treated_answer'] = df['treated_answer'].progress_apply(lambda x: length_maker(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train['treated_question_title'] = X_train['treated_question_title'].progress_apply(lambda x: length_maker(x))\nX_train['treated_question_body'] = X_train['treated_question_body'].progress_apply(lambda x: length_maker(x))\nX_train['treated_answer'] = X_train['treated_answer'].progress_apply(lambda x: length_maker(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test['treated_question_title'] = X_test['treated_question_title'].apply(lambda x: length_maker(x))\nX_test['treated_question_body'] = X_test['treated_question_body'].apply(lambda x: length_maker(x))\nX_test['treated_answer'] = X_test['treated_answer'].apply(lambda x: length_maker(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"title_max, title_mean = X_test['treated_question_title'].apply(lambda x: x.split()).map(len).max(), X_test['treated_question_title'].apply(lambda x: x.split()).map(len).mean()\nbody_max, body_mean = X_test['treated_question_body'].apply(lambda x: x.split()).map(len).max(), X_test['treated_question_body'].apply(lambda x: x.split()).map(len).mean() \nans_max, ans_mean = X_test['treated_answer'].apply(lambda x: x.split()).map(len).max(), X_test['treated_answer'].apply(lambda x: x.split()).map(len).mean() \nprint(f'X_test Max Title Size is:\\t{title_max}\\twith average size being:\\t{title_mean}')\nprint(f'X_test Max Body Size is:\\t{body_max}\\twith average size being:\\t{body_mean}')\nprint(f'X_test Max Answ Size is:\\t{ans_max}\\twith average size being:\\t{ans_mean}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnt=0\ncat_cols=[]\nfor i in X_train.columns:\n    \n    if i.startswith('category') or i.startswith('web_category'):\n        cnt+=1\n        cat_cols.append(i)\nprint(cat_cols, len(cat_cols))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets = [\n        'question_asker_intent_understanding',\n        'question_body_critical',\n        'question_conversational',\n        'question_expect_short_answer',\n        'question_fact_seeking',\n        'question_has_commonly_accepted_answer',\n        'question_interestingness_others',\n        'question_interestingness_self',\n        'question_multi_intent',\n        'question_not_really_a_question',\n        'question_opinion_seeking',\n        'question_type_choice',\n        'question_type_compare',\n        'question_type_consequence',\n        'question_type_definition',\n        'question_type_entity',\n        'question_type_instructions',\n        'question_type_procedure',\n        'question_type_reason_explanation',\n        'question_type_spelling',\n        'question_well_written',\n        'answer_helpful',\n        'answer_level_of_information',\n        'answer_plausible',\n        'answer_relevance',\n        'answer_satisfaction',\n        'answer_type_instructions',\n        'answer_type_procedure',\n        'answer_type_reason_explanation',\n        'answer_well_written'    \n    ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = X_train[targets]\nX_train.drop(columns=targets+['answer_user_name', 'answer_user_page', 'host', 'qa_id', 'question_user_name', 'question_user_page', 'url'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ny_test = X_test[targets]\nX_test.drop(columns=targets+['answer_user_name', 'answer_user_page', 'host', 'qa_id', 'question_user_name', 'question_user_page', 'url'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.shape, X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_text = pd.concat([X_train['treated_question_body'],X_train['treated_answer'],X_test['treated_question_body'],X_test['treated_answer'],X_train[\"treated_question_title\"],X_test[\"treated_question_title\"]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_tr, X_val, y_tr, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=42, stratify=train['category'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def tokenize_treated_data(text, text_test, maxlen=maxlen):\n#     t=Tokenizer(num_words=length_to_keep, filters=' ')\n#     t.fit_on_texts(text)\n#     text = t.texts_to_sequences(text)\n#     text_test = t.texts_to_sequences(text_test)\n#     text = pad_sequences(text, maxlen=maxlen)\n#     text_test = pad_sequences(text_test, maxlen=maxlen)\n#     return text, t.word_index, text_test\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t=Tokenizer(num_words=length_to_keep, filters=' ')\nt.fit_on_texts(all_text)\ndef tokenize_treated_data(text, text_test, maxlen=maxlen, t=t):\n#     t=Tokenizer(num_words=length_to_keep, filters=' ')\n#     t.fit_on_texts(text)\n    text = t.texts_to_sequences(text)\n    text_test = t.texts_to_sequences(text_test)\n    text = pad_sequences(text, maxlen=maxlen)\n    text_test = pad_sequences(text_test, maxlen=maxlen)\n    return text, t.word_index, text_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_question_body, word_index, test_question_body = tokenize_treated_data(X_tr['treated_question_body'],\n                                                                                          X_test['treated_question_body']\n                                                                                         )\n_, _, val_question_body = tokenize_treated_data(X_tr['treated_question_body'],\n                                                                                          X_val['treated_question_body']\n                                                                                         )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_answer, _, test_answer = tokenize_treated_data(X_tr['treated_answer'],\n                                                                                          X_test['treated_answer']\n                                                                                         )\n\n_, _, val_answer = tokenize_treated_data(X_train['treated_answer'],\n                                                                                          X_val['treated_answer']\n                                                                                         )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_question_title, _, test_question_title = tokenize_treated_data(X_tr['treated_question_title'],\n                                                                                          X_test['treated_question_title'], maxlen=X_tr['treated_question_title'].apply(lambda x: x.split()).map(len).max()\n                                                                                         )\n\n_, _, val_question_title = tokenize_treated_data(X_tr['treated_question_title'],\n                                                                                          X_val['treated_question_title'], maxlen=X_tr['treated_question_title'].apply(lambda x: x.split()).map(len).max()\n                                                                                         )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def embed_matrix(embed_paragram, word_index, length_to_keep = length_to_keep):\n    embeddings = np.stack(embed_paragram.values())\n    \n#    embeddings_mean, embeddings_std = embeddings.mean(), embeddings.std(ddof=1)\n#    print(embeddings_mean)\n     \n    embeddings_shape = embeddings.shape[1]\n    embedding_matrix = np.zeros((length_to_keep, 300))\n    \n    for word, i in word_index.items():\n        if i >= length_to_keep:\n            continue\n        embeddings_vector = embed_paragram.get(word)\n        if embeddings_vector is not None:\n            embedding_matrix[i] = embeddings_vector\n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# embedding_question_body = embed_matrix(embed_paragram, question_body_word_index, length_to_keep)\n# embedding_question_title = embed_matrix(embed_paragram, question_title_word_index, length_to_keep)\n# embedding_answer = embed_matrix(embed_paragram, answer_word_index, length_to_keep)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_mat = embed_matrix(embed_paragram, word_index, length_to_keep)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del embed_paragram\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # del vocab, embed_paragram, oov_paragram, df\n# del df\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# embedding_matrices = {'answer':embedding_answer, 'body':embedding_question_body, 'title':embedding_question_title}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#gbdgdfbdbdxvbdfhdf\n\n\n#fghfffffff","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def make_model(embedding_matrices=embedding_matrices, embed_size=300, loss='binary_crossentropy'):\n#     input_title = Input(shape=(X_tr['treated_question_title'].apply(lambda x: x.split()).map(len).max(),), name='title')\n#     input_body = Input(shape=(maxlen,), name='body')\n#     input_answer = Input(shape=(maxlen,), name='answer')\n#     input_category = Input(shape=(len(cat_cols),),name='categorical')\n#     GRU_nums = 64\n#     filter_size=[3,4,5]\n#     num_filters=128\n#     ###  Answer  ##########\n#     x = Embedding(length_to_keep, embed_size, weights=[embedding_matrices['answer']],trainable=False)(input_answer)\n#     ans_lstm = Bidirectional(LSTM(GRU_nums, return_sequences=True), merge_mode='sum')(x)\n# #     x = Bidirectional(LSTM(GRU_nums, return_sequences=True))(x)\n#     x = Concatenate(axis=2)([ans_lstm, x])\n# #     x = Bidirectional(LSTM(GRU_nums, return_sequences=True),merge_mode='sum')(x)\n#     conv2 = Conv1D(kernel_size=filter_size[0], strides=1, filters=num_filters, padding='same', name='ans_conv1')(x)\n#     conv2 = LeakyReLU()(conv2)\n# #     conv2 = BatchNormalization()(conv2)\n#     conv3 = Conv1D(kernel_size=filter_size[1], strides=1, filters=num_filters, padding='same')(x)\n#     conv3 = LeakyReLU()(conv3)\n# #     conv3 = BatchNormalization()(conv3)\n# #     conv4 = Conv1D(kernel_size=filter_size[2], strides=1, filters=num_filters, padding='same')(x)\n# #     conv4 = LeakyReLU()(conv4)\n\n#     conv2_max = GlobalMaxPooling1D()(conv2)\n#     conv3_max = GlobalMaxPooling1D()(conv3)\n    \n#     conv2_avg = GlobalAveragePooling1D()(conv2)\n#     conv3_avg = GlobalAveragePooling1D()(conv3)\n    \n# #     conv4 = GlobalMaxPooling1D()(conv4)\n#     conc_answer_maxp = Concatenate(axis=1)([conv2_max, conv3_max])#, conv4])\n#     conc_answer_avgp = Concatenate(axis=1)([conv2_avg, conv3_avg])\n    \n#     conc_answer = Concatenate(axis=1)([conc_answer_maxp, conc_answer_avgp])#, conv4])\n    \n#     Answer = Model(inputs=input_answer, outputs = conc_answer)\n    \n#     ###  Body  ##########\n#     y = Embedding(length_to_keep, embed_size, weights=[embedding_matrices['body']],trainable=False)(input_body)\n#     body_lstm = Bidirectional(LSTM(GRU_nums, return_sequences=True), merge_mode='sum')(y)\n# #     y = Bidirectional(LSTM(GRU_nums, return_sequences=True))(y)\n# #     y = Bidirectional(LSTM(GRU_nums, return_sequences=True),merge_mode='sum')(y)\n    \n#     y = Concatenate(axis=2)([body_lstm, y])\n    \n#     conv2_body = Conv1D(kernel_size=filter_size[0], strides=1, filters=num_filters, padding='same', name='body_conv1')(y)\n#     conv2_body = LeakyReLU()(conv2_body)\n# #     conv2_body = BatchNormalization()(conv2_body)\n#     conv3_body = Conv1D(kernel_size=filter_size[1], strides=1, filters=num_filters, padding='same')(y)\n#     conv3_body = LeakyReLU()(conv3_body)\n# #     conv3_body = BatchNormalization()(conv3_body)\n# #     conv4_body = Conv1D(kernel_size=filter_size[2], strides=1, filters=num_filters, padding='same')(y)\n# #     conv4_body = LeakyReLU()(conv4_body)\n\n#     conv2_body_max = GlobalMaxPooling1D()(conv2_body)\n#     conv3_body_max = GlobalMaxPooling1D()(conv3_body)\n    \n#     conv2_body_avg = GlobalAveragePooling1D()(conv2_body)\n#     conv3_body_avg = GlobalAveragePooling1D()(conv3_body)\n# #     conv4_body = GlobalMaxPooling1D()(conv4_body)\n    \n#     conc_body_maxp = Concatenate(axis=1)([conv2_body_max, conv3_body_max])#, conv4_body])\n#     conc_body_avgp = Concatenate(axis=1)([conv2_body_avg, conv3_body_avg])\n#     conc_body = Concatenate(axis=1)([conc_body_maxp, conc_body_avgp])\n#     Body = Model(inputs=input_body, outputs = conc_body)\n    \n    \n#     ### Title  ###########\n#     z = Embedding(length_to_keep, embed_size, weights=[embedding_matrices['title']],trainable=False)(input_title)\n#     title_lstm = Bidirectional(LSTM(GRU_nums, return_sequences=True), merge_mode='sum')(z)\n    \n#     z = Concatenate(axis=2)([title_lstm, z])\n    \n#     conv2_title = Conv1D(kernel_size=filter_size[0], strides=1, filters=num_filters, padding='same')(z)\n#     conv2_title = LeakyReLU()(conv2_title)\n# #     conv2_title = BatchNormalization()(conv2_title)\n# #     conv3_title = Conv1D(kernel_size=filter_size[1], strides=1, filters=num_filters, padding='same')(z)\n# #     conv3_title = LeakyReLU()(conv3_title)\n# #     conv4_title = Conv1D(kernel_size=filter_size[2], strides=1, filters=num_filters, padding='same')(z)\n# #     conv4_title = LeakyReLU()(conv4_title)\n    \n#     conv2_title_maxp = GlobalMaxPooling1D()(conv2_title)\n#     conv2_title_avgp = GlobalAveragePooling1D()(conv2_title)\n# #     conv3_title = GlobalMaxPooling1D()(conv3_title)\n# #     conv4_title = GlobalMaxPooling1D()(conv4_title)\n    \n#     conc_title = Concatenate(axis=1)([conv2_title_maxp, conv2_title_avgp])\n    \n#     Title = Model(inputs=input_title, outputs = conc_title)\n    \n#     ######################################\n#     combined = Concatenate(axis=1)([Title.output, Body.output, Answer.output, input_category])\n    \n# #     dense1 = Dense(1024)(combined)\n# #     dense1 = Activation('relu')(dense1)\n#     dense2 = Dense(128)(combined)\n#     dense2 = LeakyReLU()(dense2)\n#     output = Dense(30)(dense2)\n#     output = Activation('sigmoid')(output)\n    \n#     model = Model(inputs=[Title.input, Body.input, Answer.input, input_category], outputs=output)\n#     model.compile(loss=loss, optimizer = Adam(lr=1e-3), metrics=['accuracy'])\n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_model(embedding_matrices=embedding_mat, embed_size=300, loss='binary_crossentropy'):\n    input_title = Input(shape=(X_tr['treated_question_title'].apply(lambda x: x.split()).map(len).max(),), name='title')\n    input_body = Input(shape=(maxlen,), name='body')\n    input_answer = Input(shape=(maxlen,), name='answer')\n    input_category = Input(shape=(len(cat_cols),),name='categorical')\n    GRU_nums = 256\n    filter_size=[3,4,5]\n    num_filters=512\n    \n    allwords = Concatenate(axis=1)([input_title,input_body, input_category, input_answer])\n    allwords = Embedding(length_to_keep, embed_size, weights=[embedding_matrices],trainable=False)(allwords)\n    \n    lstm1 = Bidirectional(LSTM(GRU_nums, return_sequences=True), merge_mode='sum')(allwords)\n    lstm2 = Bidirectional(LSTM(GRU_nums, return_sequences=True), merge_mode='sum')(lstm1)\n#     lstm3 = Bidirectional(LSTM(GRU_nums, return_sequences=True), merge_mode='sum')(lstm2)\n    \n    conc_in = Concatenate(axis=2)([lstm2, allwords])\n    conv1 = Conv1D(kernel_size=filter_size[0], strides=1, filters=num_filters, padding='same', name='conv1')(conc_in)\n    conv1 = LeakyReLU()(conv1)\n    conv2 = Conv1D(kernel_size=filter_size[1], strides=1, filters=num_filters, padding='same', name='conv2')(conv1)\n    conv2 = LeakyReLU()(conv2)\n    \n    conv_max = GlobalMaxPooling1D()(conv2)\n    conv_avg = GlobalAveragePooling1D()(conv2)\n    \n    conc_out = Concatenate(axis=1)([conv_max, conv_avg])\n    \n    dense1 = Add()([Dense(num_filters*2)(conc_out),conc_out])\n    dense1 = LeakyReLU()(dense1)\n    dense2 = Add()([Dense(num_filters*2)(dense1),conc_out])\n    dense2 = LeakyReLU()(dense2)\n    output = Dense(30)(dense2)\n    output = Activation('sigmoid')(output)\n    \n    ######################################\n        \n    model = Model(inputs=[input_title, input_body, input_answer, input_category], outputs=output)\n    model.compile(loss=loss, optimizer = Adam(lr=1e-3), metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = make_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainingMonitor(BaseLogger):\n    def __init__(self, figPath=None, jsonPath=None, startAt=0):\n        super().__init__()\n        self.figPath = figPath\n        self.jsonPath = jsonPath\n        self.startAt = startAt\n        \n    def on_train_begin(self,logs={}):\n        self.H = {}\n        \n        if self.jsonPath is not None:\n            if os.path.exists(self.jsonPath):\n                self.H = json.loads(open(self.jsonPath).read())\n                \n                if self.startAt>0:\n                    for k in self.H:\n                        H[k] = self.H[k][:self.startAt]\n    \n    def on_epoch_end(self, epoch, logs={}):\n        for k,v in logs.items():\n            orig_list = self.H.get(k,[])\n            orig_list.append(float(v))\n            self.H[k] = orig_list\n            \n        if self.jsonPath is not None:\n            with open(self.jsonPath, 'w') as f:\n                f.write(json.dumps(self.H))\n                \n        \n        if len(self.H)>1:\n            num = np.arange(0, len(self.H['loss']))\n            fig = plt.figure(figsize=(12,8))\n            sns.lineplot(x=num, y=self.H['loss'], label = 'Train Loss')\n            sns.lineplot(x=num, y=self.H['accuracy'], label = 'Train Accuracy')\n            sns.lineplot(x=num, y=self.H['val_loss'], label = 'Val Loss')\n            sns.lineplot(x=num, y=self.H['val_accuracy'], label = 'Val Accuracy')\n            plt.title('Training Loss and Accuracy [Epoch {}]')\n            plt.xlabel('Epochs')\n            plt.ylabel('Accuracy/Loss')\n            plt.legend(loc='best')\n            plt.savefig(self.figPath)\n#             plt.show()\n            plt.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"earlystopping = EarlyStopping(monitor='val_loss', patience=8, mode='min', verbose=1, min_delta=0.01, restore_best_weights=True)\nmodelcheckpoint = ModelCheckpoint(filepath='/kaggle/working/best_model.hdf5', monitor='val_loss', mode='min',\n                                  verbose=1, min_delta=0.04, save_best_only=True)\nreduceLR = ReduceLROnPlateau(monitor='val_loss', patience=2, verbose=1,factor=0.1, mode='min', min_lr=1e-6)\ntrainingmonitor = TrainingMonitor(figPath='/kaggle/working/training.png', jsonPath='/kaggle/working/training.json')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"H = model.fit([train_question_title, train_question_body, train_answer, X_tr[cat_cols]],\n              y_tr, \n              validation_data=([val_question_title, val_question_body, val_answer, X_val[cat_cols]],y_val),\n              epochs=30,\n              batch_size=128,\n              callbacks = [trainingmonitor, modelcheckpoint, reduceLR, earlystopping],\n              verbose=1\n             )\n#fgfvcvc cccc cccccccccccccvcvc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = model.predict([test_question_title, test_question_body, test_answer, X_test[cat_cols]],\n                      batch_size=128, verbose=1\n                     )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds, preds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm = test = pd.read_csv('../input/google-quest-challenge/sample_submission.csv', header=0,encoding='utf-8')\nprint(subm.shape)\nsubm.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm[targets] = preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.to_csv(\"submission.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_tr['treated_question_title'].apply(lambda x: x.split()).map(len).max()#, X_train['treated_question_title'].apply(lambda x: x.split()).map(len).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del model\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.remove('/kaggle/working/best_model.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}