{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nimport numpy as np \nimport pandas as pd \nfrom numpy import array,asarray\nimport re\nfrom tqdm import tqdm\nfrom keras.regularizers import l1,l2\nimport os, gc\nimport random\nfrom nltk.corpus import brown\nimport nltk\nimport numpy as np\nimport pandas as pd\nimport spacy\nfrom spacy.lang.en import English\nfrom spacy.lang.en.stop_words import STOP_WORDS\nfrom string import punctuation\nfrom keras_tqdm import TQDMNotebookCallback\nfrom sklearn.decomposition import LatentDirichletAllocation\nfrom sklearn.cluster import DBSCAN\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom nltk.tokenize import TweetTokenizer, MWETokenizer\nimport time\nimport pyLDAvis\nfrom pyLDAvis.sklearn import prepare\nfrom sklearn.model_selection import train_test_split\nimport keras\nimport keras.backend as K\nfrom keras.callbacks import *\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Input, Conv1D, Dropout, Embedding, SpatialDropout1D, Bidirectional, CuDNNGRU,CuDNNLSTM, GRU, Dense, Concatenate, Reshape, Dot, Softmax, Activation, RepeatVector, Lambda, Conv1D, GlobalMaxPool1D, GlobalAvgPool1D, MaxPool1D, BatchNormalization, LSTM\nfrom keras.layers.merge import concatenate\nfrom gensim.models import KeyedVectors\nimport keras\nfrom keras.callbacks import EarlyStopping,ModelCheckpoint\nfrom sklearn.metrics import f1_score\nfrom keras.layers import GaussianDropout","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab3250666db4a181e05ef22d242b37c6c169b59a"},"cell_type":"markdown","source":"# Cleaning"},{"metadata":{"_uuid":"e084a957083d0ac36cb2d6c2b2891cd4d96d687a"},"cell_type":"markdown","source":"## Cleaning 1"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf.columns\n\nparser = English()\ndef cleaner(doc):\n    return ' '.join([token.lemma_ for token in parser(doc.lower()) if token.lemma_ not in STOP_WORDS and token.lemma_ != \"-PRON-\" and token.lemma_ not in punctuation])\n\ntqdm.pandas()\ndf['clean'] = df['question_text'].progress_apply(cleaner)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"933ce15265261330badada02eadfdc5b7f788ee2"},"cell_type":"markdown","source":"## Cleaning 2"},{"metadata":{"trusted":true,"_uuid":"bc4200b4521dcb5027b398ef849a9e134f2dc067"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef cleaner_2(doc):\n    doc = str(doc)\n    # cleaning numbers\n    doc = re.sub('[0-9]{5,}', '#####', doc)\n    doc = re.sub('[0-9]{4}', '####', doc)\n    doc = re.sub('[0-9]{3}', '###', doc)\n    doc = re.sub('[0-9]{2}', '##', doc)\n    \n    # spacing out punctuations\n    for punct in puncts:\n        doc = doc.replace(punct, f' {punct} ')\n    return doc\n\ntqdm.pandas()\ndf['clean_2'] = df['question_text'].progress_apply(cleaner_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85465247611280a5151cf7edf9babda09ddb7ecf"},"cell_type":"code","source":"def _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\n\nmispell_dict = {'colour':'color',\n                'centre':'center',\n                'didnt':'did not',\n                'cant': 'cannot',\n                'couldnt': 'could not',\n                'shant':'shall not',\n                'shouldnt': 'should not',\n                'wont': 'will not',\n                'wouldnt': 'would not',\n                'havent': 'have not',\n                'hadnt':'had not',\n                'doesnt':'does not',\n                'isnt':'is not',\n                'shouldnt':'should not',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium'\n\n                }\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2db485df77ecd9624c67bec2e71b8f616b39fb34"},"cell_type":"code","source":"tqdm.pandas()\ndf[\"clean_2\"] = df[\"clean_2\"].progress_apply(lambda x: replace_typical_misspell(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bed82bb23d92188fc21812d0456a32fa8be05c89"},"cell_type":"markdown","source":"# Generate vocab for determining spelling mistakes"},{"metadata":{"trusted":true,"_uuid":"8207089baf19dc65e10744b78c35668ece775b64"},"cell_type":"code","source":"tick = time.time()\nbrown_vocab_lower = brown.words(categories=brown.categories())\nbrown_vocab_lower = set(map(lambda x:x.lower(),brown_vocab_lower))\nprint(\"brown vocab generated in {} seconds.\".format(str(time.time()-tick)))\n\ntick = time.time()\nEMBEDDING_FILE = \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"\ndef get_coefs(word,*arr): return word\nglove_vocab = set(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding='utf-8'))\nglove_vocab = set(map(lambda x:x.lower(),glove_vocab))\nprint(\"glove vocab generated in {} seconds.\".format(str(time.time()-tick)))\n\n\ntick = time.time()\nEMBEDDING_FILE = \"../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec\"\ndef get_coefs(word,*arr): return word\nwiki_vocab = set(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding='utf-8'))\nwiki_vocab = set(map(lambda x:x.lower(),wiki_vocab))\nprint(\"wiki vocab generated in {} seconds.\".format(str(time.time()-tick)))\n\n\ntick = time.time()\nEMBEDDING_FILE = \"../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt\"\ndef get_coefs(word,*arr): return word\nparagram_vocab = set(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding='utf-8', errors='ignore')if len(o)>100)\nparagram_vocab = set(map(lambda x:x.lower(),paragram_vocab))\nprint(\"paragram vocab generated in {} seconds.\".format(str(time.time()-tick)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2713abc59b3cb45ace5cad5db837b471baeae434"},"cell_type":"code","source":"final_vocab = set(list(glove_vocab) + list(wiki_vocab) + list(brown_vocab_lower) + list(paragram_vocab))\ndel glove_vocab,wiki_vocab,brown_vocab_lower,paragram_vocab, EMBEDDING_FILE\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0094de58fa443a90f6135317027b9bc6998e5a93"},"cell_type":"markdown","source":"# Generating linguistic features"},{"metadata":{"trusted":true,"_uuid":"a880bb4faa687196ead833cacb5297b91a280677"},"cell_type":"code","source":"def get_feats(doc):\n    \n    token_length = len(doc.split())\n    i1 = len(re.findall(r'(?=\\bI\\b|\\bwe\\b|\\bme\\b|\\bus\\b|\\bmy\\b|\\bour\\b|\\bmine\\b|\\bours\\b)', doc))/token_length # FP\n    i2 = len(re.findall(r'(?=\\byou\\b|\\byour\\b|\\byours\\b)', doc)) /token_length # sP\n    i3 = len(re.findall(r'(?=\\bhe\\b|\\bshe\\b|\\bit\\b|\\bhim\\b|\\bher\\b|\\bhis\\b|\\bhers\\b|\\bits\\b)', doc))/token_length # tps\n    i4 = len(re.findall(r'(?=\\bthey\\b|\\bthem\\b|\\btheir\\b|\\btheirs\\b)', doc))/token_length # tpp\n    i5 = len(re.findall(r'(?=\\bwhat\\b|\\bwhatever\\b|\\bwhich\\b|\\bwhichever\\b|\\bwho\\b|\\bwhoever\\b|\\bwhom\\b|\\bwhomever\\b|\\bwhoses\\b)', doc)) # interrogative\n    i6 = len(re.findall(r'(?=\\bmyself\\b|\\byourself\\b|\\bherself\\b|\\bhimself\\b|\\bitself\\b|\\bourselves\\b|\\byourselves\\b|\\bthemselves\\b)', doc)) # intensive\n    i7 = len(re.findall(r'(?=\\bfor\\b|\\band\\b|\\bnor\\b|\\bbut\\b|\\byet\\b|\\bso\\b|\\bbefore\\b|\\bonce\\b|\\bsince\\b|\\bthough\\b|\\bwhile\\b|\\bas\\b|\\bbecause\\b|\\bafter\\b)', doc))/token_length # conjunctions\n    \n    i8 = (len(doc) - len( re.findall('[a-zA-Z]', doc)) - doc.count(' ') - len(re.findall('[0-9]', doc)) - doc.count(',') - doc.count('?') - doc.count('.'))\n    i9 = doc.count(',')\n    i10 = doc.count('?')\n    i11 = token_length\n    # negs\n    i12 = len(re.findall(r'(?=\\bcan\\'t\\b|\\bcouldn\\'t\\b|\\bshan\\'t\\b|\\bshouldn\\'t\\b|\\bwouldn\\'t\\b|\\bhaven\\'t\\b|\\bdidn\\'t\\b|\\bnot\\b|\\bnever\\b|\\bwon\\'t\\b|\\bdon\\'t\\b|\\bhadn\\'t\\b|\\bcant\\b|\\bcouldnt\\b|\\bshant\\b|\\bshouldnt\\b|\\bwouldnt\\b|\\bhavent\\b|\\bdidnt\\b|\\bwont\\b|\\bdont\\b|\\bhadnt\\b)', doc))\n    \n    #spelling mistakes\n    temp_doc = (\" \").join(re.findall(r\"[a-zA-Z0-9\\']+\", doc))\n    i13 = len(set(temp_doc.split())-final_vocab)\n    \n    return (i1,i2,i3,i4,i5,i6,i7,i8,i9,i10,i11,i12,i13)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30b52d1a02bfae5597fc9d849694a08b6fbe29fe"},"cell_type":"code","source":"tqdm.pandas()\ndf['feat_vect'] = df['question_text'].progress_apply(get_feats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"304391f813568ca7cb5abae4ed81de4618c5240a"},"cell_type":"code","source":"# tqdm.pandas()\n# df[\"num_words_upper\"] = df[\"question_text\"].progress_apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n\n# ## Number of title case words in the text\n# tqdm.pandas()\n# df[\"num_words_title\"] = df[\"question_text\"].progress_apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\n\n# ## Average length of the words in the text\n# tqdm.pandas()\n# df[\"mean_word_len\"] = df[\"question_text\"].progress_apply(lambda x: np.mean([len(w) for w in str(x).split()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e8a168cd18466a38c9205fa671b47473ae1be6b"},"cell_type":"code","source":"df['fp'] = df['feat_vect'].apply(lambda x: x[0])\ndf['sp'] = df['feat_vect'].apply(lambda x: x[1])\ndf['tps'] = df['feat_vect'].apply(lambda x: x[2])\ndf['tpp'] = df['feat_vect'].apply(lambda x: x[3])\n# df['interrogative'] = df['feat_vect'].apply(lambda x: x[4])\n# df['intensive'] = df['feat_vect'].apply(lambda x: x[5])\ndf['conjunction'] = df['feat_vect'].apply(lambda x: x[6])\ndf['special_chars'] = df['feat_vect'].apply(lambda x: x[7])\ndf['commas'] = df['feat_vect'].apply(lambda x: x[8])\ndf['qm'] = df['feat_vect'].apply(lambda x: x[9])\ndf['len'] = df['feat_vect'].apply(lambda x: x[10])\n# df['negs'] = df['feat_vect'].apply(lambda x: x[11])\ndf['sm'] = df['feat_vect'].apply(lambda x: x[12])\nsincere = df[df['target']==0]\ninsincere = df[df['target']==1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"431b8960b3e6bb68ebdc54552ae3582a43d57fe3"},"cell_type":"code","source":"sincere.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5e8c3fda859d5e70440da5c18f6e9283e97cb87"},"cell_type":"code","source":"insincere.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f54f8c3152f559b8256f9c7e3088330260c7d18"},"cell_type":"markdown","source":"# Prepare train & test data"},{"metadata":{"trusted":true,"_uuid":"86988ca640973be9745523d85a8ed24b348458c6"},"cell_type":"code","source":"# Prepare test data\ntest = pd.read_csv('../input/test.csv')\ntest['clean'] = test['question_text'].apply(cleaner)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53d1ac80619e8db3371de14f6992c0fedf69d2c6"},"cell_type":"code","source":"test['clean_2'] = test['question_text'].apply(cleaner_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"684edff27eb89b218dd885b9d29b7a49185dfbb4"},"cell_type":"code","source":"# tqdm.pandas()\n# test[\"num_words_upper\"] = test[\"question_text\"].progress_apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n\n# ## Number of title case words in the text\n# tqdm.pandas()\n# test[\"num_words_title\"] = test[\"question_text\"].progress_apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\n\n# ## Average length of the words in the text\n# tqdm.pandas()\n# test[\"'mean_word_len'\"] = test[\"question_text\"].progress_apply(lambda x: np.mean([len(w) for w in str(x).split()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00537d7cc3ccdc961592fd8bd734958b69b37736"},"cell_type":"code","source":"tqdm.pandas()\ntest['feat_vect'] = test['question_text'].progress_apply(get_feats)\n\ntest['fp'] = test['feat_vect'].apply(lambda x: x[0])\ntest['sp'] = test['feat_vect'].apply(lambda x: x[1])\ntest['tps'] = test['feat_vect'].apply(lambda x: x[2])\ntest['tpp'] = test['feat_vect'].apply(lambda x: x[3])\n# test['interrogative'] = test['feat_vect'].apply(lambda x: x[4])\n# test['intensive'] = test['feat_vect'].apply(lambda x: x[5])\ntest['conjunction'] = test['feat_vect'].apply(lambda x: x[6])\ntest['special_chars'] = test['feat_vect'].apply(lambda x: x[7])\ntest['commas'] = test['feat_vect'].apply(lambda x: x[8])\ntest['qm'] = test['feat_vect'].apply(lambda x: x[9])\ntest['len'] = test['feat_vect'].apply(lambda x: x[10])\n# test['negs'] = test['feat_vect'].apply(lambda x: x[11])\ntest['sm'] = test['feat_vect'].apply(lambda x: x[12])\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92ec0c36d23d7c2f28359cf374a681b017ac9cc3"},"cell_type":"code","source":"del final_vocab\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd502631f33a1cf5c0fc703a797ec8d786197e25"},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e48d527ea332a03340d8f17167beab312aa26e3"},"cell_type":"code","source":"### LDA v2 - only bigrams & trigrams\n\ndef my_tweet_tokenizer(doc):\n    t1 = TweetTokenizer()\n    return t1.tokenize(doc)\nc2 = CountVectorizer( tokenizer=my_tweet_tokenizer, ngram_range=(2,3), min_df=6)\n\ntick = time.time()\nc2.fit(insincere['clean'])\nbow2 = c2.transform(insincere['clean'])\nprint('Time taken to fit CV: ' + str(time.time()-tick) + ' seconds.')\n\nlda2 = LatentDirichletAllocation(n_components=8, learning_method='batch')\ntick = time.time()\nlda2.fit(bow2)\nprint('Time taken to fit lda: ' + str(time.time()-tick) + ' seconds.')\n\n# pyLDAvis.enable_notebook()\n# prepare(lda2,bow2,c2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ef48b70aeb24629cd130b754ad44b53bca3b322"},"cell_type":"code","source":"# Prepare training data: 0.8 of sincere, 1 of sincere\ntick = time.time()\nnew_train_df = pd.concat([sincere.sample(frac=0.8, random_state=42),insincere])\nnew_train_df = new_train_df.sample(frac=1, random_state=113)\n\n# Train test split\nX_train, X_dev, y_train, y_dev = train_test_split(new_train_df, new_train_df['target'], test_size=0.2, random_state=42, stratify=new_train_df['target'])\nt1 = Tokenizer()\nt1.fit_on_texts(pd.concat([df['clean_2'], test['clean_2']]))\n\nt2 = Tokenizer(filters=None, char_level=True)\nt2.fit_on_texts(pd.concat([df['clean_2'], test['clean_2']]))\n\nprint('Time taken to fit keras tokenizers : ' + str(time.time()-tick) + ' seconds.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81fa5994c0f61931e21c2feeaf2a2fc533a9fbf3"},"cell_type":"code","source":"tick = time.time()\nX_train_tts = t1.texts_to_sequences(X_train['clean_2'])\nX_dev_tts = t1.texts_to_sequences(X_dev['clean_2'])\ntest_tts = t1.texts_to_sequences(test['clean_2'])\n\nX_train_tts = pad_sequences(X_train_tts, maxlen=90, padding='post', truncating='post')\nX_dev_tts = pad_sequences(X_dev_tts, maxlen=90, padding='post', truncating='post')\ntest_tts = pad_sequences(test_tts, maxlen=90, padding='post', truncating='post')\n\nX_train_cts = t2.texts_to_sequences(X_train['clean_2'])\nX_dev_cts = t2.texts_to_sequences(X_dev['clean_2'])\ntest_cts = t2.texts_to_sequences(test['clean_2'])\n\nX_train_cts = pad_sequences(X_train_cts, maxlen=180, padding='post', truncating='post')\nX_dev_cts = pad_sequences(X_dev_cts, maxlen=180, padding='post', truncating='post')\ntest_cts = pad_sequences(test_cts, maxlen=180, padding='post', truncating='post')\n\nprint('Time taken for tts conversion : ' + str(time.time()-tick) + ' seconds.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3233004c8f2d7a27f42a522795625c390085b3e"},"cell_type":"code","source":"tick = time.time()\nbow_train = c2.transform(X_train['clean'])\nldavec_train = lda2.transform(bow_train)\n\nbow_dev = c2.transform(X_dev['clean'])\nldavec_dev = lda2.transform(bow_dev)\n\nbow_test = c2.transform(test['clean'])\nldavec_test = lda2.transform(bow_test)\nprint('Time taken to generate lda vectors for train, dev & test : ' + str(time.time()-tick) + ' seconds.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5fd81cb227cd039bbb34c7e6d6a123bd3bd55ac6"},"cell_type":"code","source":"feat_train = X_train[['fp', 'sp', 'tps', 'tpp',  'conjunction', 'special_chars', \n                      'commas', 'qm', 'len',  'sm']].values\nfeat_dev = X_dev[['fp', 'sp', 'tps', 'tpp', 'conjunction', 'special_chars', \n                      'commas', 'qm', 'len', 'sm']].values\nfeat_test = test[['fp', 'sp', 'tps', 'tpp', 'conjunction', 'special_chars', \n                      'commas', 'qm', 'len',  'sm']].values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f384dc20386b38133c62a2fdd3dcd93e490579d"},"cell_type":"markdown","source":"# Prepare Embedding matrix"},{"metadata":{"trusted":true,"_uuid":"dcaf2a7d5338a7bbc8a727de4464cbbcf5ac44c0"},"cell_type":"code","source":"tick = time.time()\nembedding_dim = 300\nvocab_size = len(t1.word_index)+1\n# vocab_size = 100000\nEMBEDDING_FILE = \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_glove = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding='utf-8'))\n\nembedding_matrix = np.random.randn(vocab_size,embedding_dim).astype(np.float32) * np.sqrt(2.0/vocab_size)\n\nfor word, i in t1.word_index.items():\n    embedding_vector = embeddings_glove.get(word)\n    if embedding_vector is not None: \n        embedding_matrix[i] = embedding_vector\n\ndel embeddings_glove, EMBEDDING_FILE\ngc.collect()\nprint(\"Embedding Matrix generated in {} seconds.\".format(str(time.time()-tick)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4cbb2a463f0bb0660ce4683df2036545f8e4d9a"},"cell_type":"code","source":"tick = time.time()\nEMBEDDING_FILE = \"../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec\"\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_wiki = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding='utf-8'))\n\nembedding_matrix_wiki = np.random.randn(vocab_size,embedding_dim).astype(np.float32) * np.sqrt(2.0/vocab_size)\n\nfor word, i in t1.word_index.items():\n    embedding_vector = embeddings_wiki.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_wiki[i] = embedding_vector\n\ndel embeddings_wiki, EMBEDDING_FILE\ngc.collect()\nprint(\"Embedding Matrix WIKI generated in {} seconds.\".format(str(time.time()-tick)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab321e0659a1598cb725ac8dfe8d76addac2bc89"},"cell_type":"code","source":"tick = time.time()\nfrom gensim.models import KeyedVectors\nembeddings_google = KeyedVectors.load_word2vec_format('../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)\nprint(\"Load time {} seconds.\".format(str(time.time()-tick)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8eddfa342a1484d4fdde4044779741be038030b1"},"cell_type":"code","source":"embedding_matrix_google = np.random.randn(vocab_size,embedding_dim).astype(np.float32) * np.sqrt(2.0/vocab_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92edcd00e5a1a16d66e87483db2af85afd8db2b4"},"cell_type":"code","source":"tick = time.time()\nfor word, i in t1.word_index.items():\n    try:\n        embedding_vector = 'init'\n        embedding_vector = embeddings_google[word]\n        embedding_matrix_google[i] = embedding_vector\n    except:\n        if embedding_vector=='init':\n             pass\nprint(\"Embedding Matrix google generated in {} seconds.\".format(str(time.time()-tick)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37c05a195cc036ac7585b13bbd1f550eff83b79b"},"cell_type":"code","source":"del embeddings_google\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87ea95dac59e743a8547f58db73e37d02a69fdc0"},"cell_type":"code","source":"mean_emb = np.mean([embedding_matrix, embedding_matrix_wiki, embedding_matrix_google], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34b79e979354252613cf37f9845a279d9e89ab08"},"cell_type":"code","source":"mean_emb.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4631d7327bd6d2d2c1104ee1263231b4f3dd22a8"},"cell_type":"code","source":"char_embedding_matrix = np.random.randn(len(t2.word_index)+1,128).astype(np.float32) * np.sqrt(2.0/len(t2.word_index)+1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2e573f4614251699719a5f33ae6ef9c2bf4d66c9"},"cell_type":"markdown","source":"# Modelling"},{"metadata":{"trusted":true,"_uuid":"033ad084762b10416f717390d0c924c563c24e9d"},"cell_type":"code","source":"class CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72743d1381f190db470d52d3aa56bc9b79a63db5"},"cell_type":"code","source":"def precision(y_true, y_pred):\n    \"\"\"Precision metric.\n\n    Only computes a batch-wise average of precision.\n\n    Computes the precision, a metric for multi-label classification of\n    how many selected items are relevant.\n    \"\"\"\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef recall(y_true, y_pred):\n    \"\"\"Recall metric.\n\n    Only computes a batch-wise average of recall.\n\n    Computes the recall, a metric for multi-label classification of\n    how many relevant items are selected.\n    \"\"\"\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"925abdad8e006cb8ad23ce0ad5cf102bbaab11d5"},"cell_type":"code","source":"class Configuration:\n    \n    def get_config_object():\n        config = Configuration()\n        config.max_seq_size = 90\n        config.word_vocab_size = vocab_size\n        config.embedding_dim = 300\n        config.units1 =64 \n        config.units2 = 32\n        config.units3 = 32\n        \n        return config","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bf0c6ef2a115496deb7baca7ea9587ec5c5905d"},"cell_type":"code","source":"class Model:\n    \n    def __init__(self, config):\n        self.config = config\n        \n    def get_model(self):\n        \n        input_layer = Input(batch_shape=(None,self.config.max_seq_size), name='input_layer')\n        input_layer_2 = Input(batch_shape=(None,180), name='input_layer_cts')\n        input_layer_3 = Input(batch_shape=(None,10), name='input_layer_feat')\n        lda_input = Input(batch_shape=(None, 8), name='lda_input_layer')\n        \n        embed_op = Embedding(input_dim= self.config.word_vocab_size, input_length=self.config.max_seq_size, output_dim=self.config.embedding_dim, \n                              weights=[mean_emb], trainable=True, name='word_embeddings')(input_layer)\n#         \n        spdropout = SpatialDropout1D(0.2, name='sp_dropout')(embed_op)\n        bigru1 = Bidirectional(CuDNNLSTM(units=self.config.units1, return_sequences=True, name='bigru1'))(spdropout)\n#         spdropout_lstm = SpatialDropout1D(0.1, name='sp_dropout_lstm')(bigru1)\n        \n        embed_op_2 = Embedding(input_dim=len(t2.word_index)+1, input_length=180, output_dim=128, name='char_embeddings')(input_layer_2)\n        spdropout_2 = SpatialDropout1D(0.2, name='sp_dropout_2')(embed_op_2)\n        conv1 = Conv1D(filters=32, kernel_size=2, name='conv1')(spdropout_2)\n        maxpool1 = GlobalMaxPool1D()(conv1)\n#         avgpool1 = GlobalAvgPool1D()(conv1)\n        \n        conv2 = Conv1D(filters=32, kernel_size=4, name='conv2')(spdropout_2)\n        maxpool2 = GlobalMaxPool1D()(conv2)\n#         avgpool2 = GlobalAvgPool1D()(conv2)\n        \n        conv3 = Conv1D(filters=32, kernel_size=8, name='conv3')(spdropout_2)\n        maxpool3 = GlobalMaxPool1D()(conv3)\n#         avgpool3 = GlobalAvgPool1D()(conv3)\n\n        # Attention\n        # Inputs: [bs, TX, units2], Output = [bs,TX, dense_units]\n        tile_layer = RepeatVector(n=90, name='tile_layer')\n        tiled_op = tile_layer(lda_input)\n        concat_1 = Concatenate(axis=2, name='concat_1')([bigru1,tiled_op])\n        \n        att_dense_layer = Dense(units=1, name='attention_dense_1', activation='tanh')\n        att_dense_op_1 = att_dense_layer(concat_1) #Output = [bs,TX, 1]\n        softmax_layer = Softmax(axis=1, name='softmax_over_time')\n        alpha_1 = softmax_layer(att_dense_op_1) #Output = [bs,TX, 1]\n        dot_layer = Dot(axes=1, name='dot_layer')\n        context_1 = dot_layer([alpha_1, concat_1])\n        context_reshape_1 = Reshape(target_shape=(136,), name='reshape_context')(context_1)#133 136 150\n       \n        \n        selector = Lambda(lambda x: x[:, -1], name='slicer')\n        lt_bigru1 = selector(bigru1)\n        \n      \n        concat = concatenate([lt_bigru1,context_reshape_1,maxpool1,maxpool2, maxpool3,lda_input, input_layer_3], name='concatenate')\n        \n        dense1 = Dense(units=self.config.units3, activation='tanh', name='dense1')(concat)\n#         drop1 = Dropout(0.1, name='Dropout_1_c3')(dense1)\n        drop1 = GaussianDropout(0.1, name='Dropout_1_c3')(dense1)\n        bn = BatchNormalization()(drop1)\n        dense2 = Dense(units=16, activation='tanh', name='dense2')(bn)\n        dense3 = Dense(units=1, activation='sigmoid', name='output')(dense2)\n        classifier = keras.models.Model(inputs=[input_layer,input_layer_2,input_layer_3,lda_input], outputs=dense3)\n        \n        return classifier\n    \n        \n    def get_model_2(self):\n        \n        input_layer = Input(batch_shape=(None,self.config.max_seq_size), name='input_layer')\n        input_layer_2 = Input(batch_shape=(None,180), name='input_layer_cts')\n        input_layer_3 = Input(batch_shape=(None,10), name='input_layer_feat')\n        lda_input = Input(batch_shape=(None, 8), name='lda_input_layer')\n        \n        \n        embed_op1 = Embedding(input_dim= self.config.word_vocab_size, input_length=90, output_dim=self.config.embedding_dim, \n                              weights=[embedding_matrix], trainable=True, name='word_embeddings')(input_layer)  \n        spdropout1 = SpatialDropout1D(0.2, name='sp_dropout1')(embed_op1)\n        conv1_word = Conv1D(filters=32, kernel_size=2, name='conv1_word')(spdropout1)\n        maxpool1_word = GlobalMaxPool1D()(conv1_word)\n        avgpool1_word = GlobalAvgPool1D()(conv1_word)\n        \n        conv2_word = Conv1D(filters=32, kernel_size=4, name='conv2_word')(spdropout1)\n        maxpool2_word = GlobalMaxPool1D()(conv2_word)\n        avgpool2_word = GlobalAvgPool1D()(conv2_word)\n        \n        conv3_word = Conv1D(filters=32, kernel_size=8, name='conv3_word')(spdropout1)\n        maxpool3_word = GlobalMaxPool1D()(conv3_word)\n        avgpool3_word = GlobalAvgPool1D()(conv3_word)\n        \n        embed_op_2 = Embedding(input_dim=len(t2.word_index)+1, input_length=180, output_dim=128, name='char_embeddings')(input_layer_2)\n        spdropout_2 = SpatialDropout1D(0.2, name='sp_dropout_2')(embed_op_2)\n        conv1 = Conv1D(filters=32, kernel_size=2, name='conv1')(spdropout_2)\n        maxpool1 = GlobalMaxPool1D()(conv1)\n        avgpool1 = GlobalAvgPool1D()(conv1)\n        \n        conv2 = Conv1D(filters=32, kernel_size=4, name='conv2')(spdropout_2)\n        maxpool2 = GlobalMaxPool1D()(conv2)\n        avgpool2 = GlobalAvgPool1D()(conv2)\n        \n        conv3 = Conv1D(filters=32, kernel_size=8, name='conv3')(spdropout_2)\n        maxpool3 = GlobalMaxPool1D()(conv3)\n        avgpool3 = GlobalAvgPool1D()(conv3)\n\n\n        \n        concat = concatenate([maxpool1_word,maxpool2_word, maxpool3_word,avgpool1_word, avgpool2_word, avgpool3_word,maxpool1,maxpool2, maxpool3,avgpool1, avgpool2, avgpool3,lda_input, input_layer_3], name='concatenate')\n        \n        dense1 = Dense(units=self.config.units3, activation='tanh', name='dense1')(concat)\n#         drop1 = Dropout(0.1, name='Dropout_1_c3')(dense1)\n        drop1 = GaussianDropout(0.1, name='Dropout_1_c3')(dense1)\n        bn = BatchNormalization()(drop1)\n        dense2 = Dense(units=16, activation='tanh', name='dense2')(bn)\n        dense3 = Dense(units=1, activation='sigmoid', name='output')(dense2)\n        classifier = keras.models.Model(inputs=[input_layer,input_layer_2,input_layer_3,lda_input], outputs=dense3)\n        \n        return classifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b1924f99e2d34392e531094dda2bcb60ac8872a"},"cell_type":"code","source":"config = Configuration.get_config_object()\nmodel = Model(config)\nopt = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6aaacfec4dbe070084726cf668e1d524322b575b"},"cell_type":"code","source":"classifier = model.get_model()\nclassifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', precision, recall, f1])\nclassifier.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dcc55723acf8bff680ba670ca7571833ef9c12a"},"cell_type":"code","source":"earlystopping = EarlyStopping(monitor='val_f1', patience=1, mode='max')\nmodelcheckpoint = ModelCheckpoint(filepath='model.h5', monitor='val_f1', verbose=1, save_best_only=True, mode='max')\npbar = TQDMNotebookCallback(leave_inner=True, leave_outer=True)\nDATA_SPLIT_SEED = 2018\nclr = CyclicLR(base_lr=0.001, max_lr=0.002,\n               step_size=300, mode='exp_range',\n               gamma=0.99994)\ncallbacks1 = [earlystopping,modelcheckpoint, clr]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"644996a024d42136ba362ddc0e7a3c223d708dac"},"cell_type":"code","source":"# cw = {0:0.84121908, 1: 1.56515283} val f1 = 0.6866\n# cw = {0:1, 1: 1.6} val f1 = 0.6879","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39c31ac12976b58e4f6831055c9338672b3fc6ac"},"cell_type":"code","source":"cw = {0:2, 1: 2.6}\nclassifier.fit([X_train_tts,X_train_cts,feat_train ,ldavec_train], y_train, callbacks=callbacks1 ,validation_data=([X_dev_tts,X_dev_cts,feat_dev,ldavec_dev], y_dev), epochs=1, batch_size=128, verbose=1, class_weight=cw)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad00fd0616f0021456a66474577e97c719436389"},"cell_type":"code","source":"cw = {0:1, 1: 1.6}\nclassifier.fit([X_train_tts,X_train_cts,feat_train ,ldavec_train], y_train, callbacks=callbacks1 ,validation_data=([X_dev_tts,X_dev_cts,feat_dev,ldavec_dev], y_dev), epochs=1, batch_size=256, verbose=1, class_weight=cw)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d57beb31f9d8026b6901a67778f1cd01e2f9351"},"cell_type":"markdown","source":"# Threshold adjustment"},{"metadata":{"trusted":true,"_uuid":"e1e9316851a6e4d09042f1859eb08a1ff52be29a"},"cell_type":"code","source":"# pred_dev_c1 = classifier.predict([X_dev_tts,X_dev_cts,feat_dev,ldavec_dev], verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"043f56dbb3ebea1d4a1cfe936a28ed6379ba4d3c"},"cell_type":"code","source":"# thresholds = []\n# for thresh in np.arange(0.15, 0.601, 0.01):\n# #     for alpha1 in np.arange(0.1, 0.81, 0.1):\n# #         alpha2 = 1 - alpha1\n#         thresh = np.round(thresh, 2)\n# #         pred_dev_y = alpha1 * pred_dev_c1 + alpha2 * pred_dev_c2\n#         res = f1_score(y_dev, (pred_dev_c1 > thresh).astype(int))\n# #     thresholds.append([thresh, res, alpha1, alpha2])\n#         thresholds.append([thresh, res])\n#         print(\"F1 score at threshold {0}, is {1}\".format(thresh, np.round(res,2)))\n    \n# thresholds.sort(key=lambda x: x[1], reverse=True)\n# best_thresh = thresholds[0][0]\n# best_f1 = thresholds[0][1]\n# # best_alpha1 = thresholds[0][2]\n# # best_alpha2 = thresholds[0][3]\n# print(\"Best f1 score: {1} at threshold {0}\".format(best_thresh, best_f1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc94d59249e4078dd17bf6b8a495b537450116ef"},"cell_type":"markdown","source":"# Final Preds"},{"metadata":{"trusted":true,"_uuid":"1de13f989d390adfa513174ae19cafa7a38212e1"},"cell_type":"code","source":"# pred1 = classifier.predict([test_tts,test_cts,feat_test,ldavec_test], verbose=1)\n# pred2 = classifier2.predict([test_tts,test_cts,feat_test,ldavec_test], verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"071a901edf589ee1e2e5aa092b4a8b235026e0a8"},"cell_type":"code","source":"# pred = best_alpha1*pred1 + best_alpha2*pred2\npred = classifier.predict([test_tts,test_cts,feat_test,ldavec_test], verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"431fbf78a4886cae53dcca78f9a8e69b2b1cfb8a"},"cell_type":"code","source":"pred = (pred > 0.5).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da99ea8d3c3043e19771d6d2ab8487b57cff0d4c"},"cell_type":"code","source":"test['prediction'] = pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4513d0eccd17d46e67b0f0c4c737397d3a41a9ad"},"cell_type":"code","source":"final = test.drop(['question_text','feat_vect','fp', 'sp', 'tps', 'tpp',  'conjunction', 'special_chars', \n                      'commas', 'qm', 'len',  'sm', 'clean','clean_2'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75a28dcddcff6bf6387b59b7d596ece19f401041"},"cell_type":"code","source":"final.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4670b5e99e1c9406c544d85b0a4245e02ef691ca"},"cell_type":"code","source":"final.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}