{"cells":[{"metadata":{},"cell_type":"markdown","source":"Concatinate different embedding.\nUse some statistical features."},{"metadata":{"trusted":true},"cell_type":"code","source":"# import libraries\n# \n\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom nltk.corpus import stopwords \n#from wordcloud import WordCloud\nimport re\nfrom tqdm import tqdm\nimport random\n#import regex\nimport gensim\n\n#from sklearn.manifold import TSNE\nfrom sklearn.metrics import f1_score\n\nfrom sklearn.preprocessing import StandardScaler\nimport os\nimport nltk\n\nimport gc\n\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.model_selection import RandomizedSearchCV\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import make_scorer\nfrom sklearn.metrics import confusion_matrix, classification_report\n\nfrom keras.preprocessing.text import Tokenizer \nfrom keras.preprocessing.sequence import pad_sequences \nfrom keras.layers import Flatten\n\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, CuDNNLSTM, concatenate\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, Dropout, SpatialDropout1D, GlobalAveragePooling1D, GlobalMaxPooling1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\n\nfrom keras.layers.normalization import BatchNormalization \nfrom keras.initializers import RandomNormal \nfrom keras import regularizers\nfrom keras.callbacks import *\nimport keras\n\n\n\nfrom nltk.stem import PorterStemmer\nps = PorterStemmer()\nfrom nltk.stem.lancaster import LancasterStemmer\nlc = LancasterStemmer()\nfrom nltk.stem import SnowballStemmer\nsb = SnowballStemmer(\"english\")\nfrom nltk.stem import WordNetLemmatizer \nlemmatizer = WordNetLemmatizer() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\nprint (\"Number of data points:\", data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = pd.read_csv('../input/quora-insincere-questions-classification/test.csv') \nprint (\"Number of data points:\", test_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#nltk.download('stopwords')\nstopword = stopwords.words('english')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Question Length\ndata['q_len'] = data['question_text'].str.len()\n# Number of words in the question\ndata['q_words'] = data['question_text'].apply(lambda row: len(row.split(\" \")))\n# Number of Upper character in question\ndata['u_chars'] = data['question_text'].apply(lambda row: sum(1 for c in row if c.isupper()))\n# Number of lower character in question\ndata['l_chars'] = data['question_text'].apply(lambda row: sum(1 for c in row if c.islower()))\n# Number of stopwords in question\ndata['n_stopwords'] = data['question_text'].apply(lambda row: sum(1 for word in row.split(\" \") if word in stopword))\n# Number of capital words in question\ndata['n_cap_words'] = data['question_text'].apply(lambda row: sum(1 for word in row.split(\" \") if word.isupper()))\n# Number of different words in question\ndata['n_diff_words'] = data['question_text'].apply(lambda row: len(set(row.split(\" \"))))\n# Averge Word length\ndata['avg_word_len'] = data['question_text'].apply(lambda row: sum(len(i) for i in row.split(\" \"))/len(row.split(\" \")))\n# Number of numerical Values in the text\ndata['n_numerical_words'] = data['question_text'].apply(lambda row: sum(1 for word in row.split(\" \")if word.isnumeric()))\n\n\n\n# Number of simpley\ndata[\"nb_simley\"] = data['question_text'].apply(lambda row: sum(1 for word in row.split(\" \")if re.findall(r\"^(:\\(|:\\))+$\", word)))\n# Number of special symbols\ndata[\"nb_symbols\"] = data['question_text'].apply(lambda row: sum(1 for word in row.split(\" \")if re.findall(r\"[@_!#$%^&*()<>?/\\|}{~:]\", word)))\n# Number of Punctions\ndata[\"nb_punct\"] = data['question_text'].apply(lambda row: sum(1 for c in row if (c==\"'\" or c==';'  or c==\"/\" or c=='.')))\n\n\nprint (data.shape)\n#data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Question Length\ntest_data['q_len'] = test_data['question_text'].str.len()\n# Number of words in the question\ntest_data['q_words'] = test_data['question_text'].apply(lambda row: len(row.split(\" \")))\n# Number of Upper character in question\ntest_data['u_chars'] = test_data['question_text'].apply(lambda row: sum(1 for c in row if c.isupper()))\n# Number of lower character in question\ntest_data['l_chars'] = test_data['question_text'].apply(lambda row: sum(1 for c in row if c.islower()))\n# Number of stopwords in question\ntest_data['n_stopwords'] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(\" \") if word in stopword))\n# Number of capital words in question\ntest_data['n_cap_words'] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(\" \") if word.isupper()))\n# Number of different words in question\ntest_data['n_diff_words'] = test_data['question_text'].apply(lambda row: len(set(row.split(\" \"))))\n# Averge Word length\ntest_data['avg_word_len'] = test_data['question_text'].apply(lambda row: sum(len(i) for i in row.split(\" \"))/len(row.split(\" \")))\n# Number of numerical Values in the text\ntest_data['n_numerical_words'] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(\" \")if word.isnumeric()))\n\n# Number of simpley\ntest_data[\"nb_simley\"] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(\" \")if re.findall(r\"^(:\\(|:\\))+$\", word)))\n# Number of special symbols\ntest_data[\"nb_symbols\"] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(\" \")if re.findall(r\"[@_!#$%^&*()<>?/\\|}{~:]\", word)))\n# Number of Punctions\ntest_data[\"nb_punct\"] = test_data['question_text'].apply(lambda row: sum(1 for c in row if (c==\"'\" or c==';' or c==\"/\" or c=='.')))\n\nprint (test_data.shape)\n#test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/kentaronakanishi/18th-place-solution\npuncts = [\n    ',', '.', '\"', ':', ')', '(', '-', '!', '?','|', ';', \"'\", '$', '&',\n    '/', '[', ']', '%', '=', '#', '*', '+', '\\\\', '•', '~', '@', '£',\n    '·', '_', '{', '}', '©', '^', '®', '`', '→', '°', '€', '™', '›',\n    '♥', '←', '×', '§', '″', '′', 'Â', '█', 'à', '…', '“', '★', '”',\n    '–', '●', 'â', '►', '−', '¢', '¬', '░', '¶', '↑', '±',  '▾',\n    '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '⊕', '▼',\n    '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',\n    'è', '¸', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',\n    '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',\n    '¹', '≤', '‡', '₹', '´'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"abbreviations = {\n    \"ain't\": \"is not\",\n    \"aren't\": \"are not\",\n    \"can't\": \"cannot\",\n    \"'cause\": \"because\",\n    \"could've\": \"could have\",\n    \"couldn't\": \"could not\",\n    \"didn't\": \"did not\",\n    \"doesn't\": \"does not\",\n    \"don't\": \"do not\",\n    \"hadn't\": \"had not\",\n    \"hasn't\": \"has not\",\n    \"haven't\": \"have not\",\n    \"he'd\": \"he would\",\n    \"he'll\": \"he will\",\n    \"he's\": \"he is\",\n    \"how'd\": \"how did\",\n    \"how'd'y\": \"how do you\",\n    \"how'll\": \"how will\",\n    \"how's\": \"how is\",\n    \"I'd\": \"I would\",\n    \"I'd've\": \"I would have\",\n    \"I'll\": \"I will\",\n    \"I'll've\": \"I will have\",\n    \"I'm\": \"I am\",\n    \"I've\": \"I have\",\n    \"i'd\": \"i would\",\n    \"i'd've\": \"i would have\",\n    \"i'll\": \"i will\",\n    \"i'll've\": \"i will have\",\n    \"i'm\": \"i am\",\n    \"i've\": \"i have\",\n    \"isn't\": \"is not\",\n    \"it'd\": \"it would\",\n    \"it'd've\": \"it would have\",\n    \"it'll\": \"it will\",\n    \"it'll've\": \"it will have\",\n    \"it's\": \"it is\",\n    \"let's\": \"let us\",\n    \"ma'am\": \"madam\",\n    \"mayn't\": \"may not\",\n    \"might've\": \"might have\",\n    \"mightn't\": \"might not\",\n    \"mightn't've\": \"might not have\",\n    \"must've\": \"must have\",\n    \"mustn't\": \"must not\",\n    \"mustn't've\": \"must not have\",\n    \"needn't\": \"need not\",\n    \"needn't've\": \"need not have\",\n    \"o'clock\": \"of the clock\",\n    \"oughtn't\": \"ought not\",\n    \"oughtn't've\": \"ought not have\",\n    \"shan't\": \"shall not\",\n    \"sha'n't\": \"shall not\",\n    \"shan't've\": \"shall not have\",\n    \"she'd\": \"she would\",\n    \"she'd've\": \"she would have\",\n    \"she'll\": \"she will\",\n    \"she'll've\": \"she will have\",\n    \"she's\": \"she is\",\n    \"should've\": \"should have\",\n    \"shouldn't\": \"should not\",\n    \"shouldn't've\": \"should not have\",\n    \"so've\": \"so have\",\n    \"so's\": \"so as\",\n    \"this's\": \"this is\",\n    \"that'd\": \"that would\",\n    \"that'd've\": \"that would have\",\n    \"that's\": \"that is\",\n    \"there'd\": \"there would\",\n    \"there'd've\": \"there would have\",\n    \"there's\": \"there is\",\n    \"here's\": \"here is\",\n    \"they'd\": \"they would\",\n    \"they'd've\": \"they would have\",\n    \"they'll\": \"they will\",\n    \"they'll've\": \"they will have\",\n    \"they're\": \"they are\",\n    \"they've\": \"they have\",\n    \"to've\": \"to have\",\n    \"wasn't\": \"was not\",\n    \"we'd\": \"we would\",\n    \"we'd've\": \"we would have\",\n    \"we'll\": \"we will\",\n    \"we'll've\": \"we will have\",\n    \"we're\": \"we are\",\n    \"we've\": \"we have\",\n    \"weren't\": \"were not\",\n    \"what'll\": \"what will\",\n    \"what'll've\": \"what will have\",\n    \"what're\": \"what are\",\n    \"what's\": \"what is\",\n    \"what've\": \"what have\",\n    \"when's\": \"when is\",\n    \"when've\": \"when have\",\n    \"where'd\": \"where did\",\n    \"where's\": \"where is\",\n    \"where've\": \"where have\",\n    \"who'll\": \"who will\",\n    \"who'll've\": \"who will have\",\n    \"who's\": \"who is\",\n    \"who've\": \"who have\",\n    \"why's\": \"why is\",\n    \"why've\": \"why have\",\n    \"will've\": \"will have\",\n    \"won't\": \"will not\",\n    \"won't've\": \"will not have\",\n    \"would've\": \"would have\",\n    \"wouldn't\": \"would not\",\n    \"wouldn't've\": \"would not have\",\n    \"y'all\": \"you all\",\n    \"y'all'd\": \"you all would\",\n    \"y'all'd've\": \"you all would have\",\n    \"y'all're\": \"you all are\",\n    \"y'all've\": \"you all have\",\n    \"you'd\": \"you would\",\n    \"you'd've\": \"you would have\",\n    \"you'll\": \"you will\",\n    \"you'll've\": \"you will have\",\n    \"you're\": \"you are\",\n    \"you've\": \"you have\",\n    \"who'd\": \"who would\",\n    \"who're\": \"who are\",\n    \"'re\": \" are\",\n    \"tryin'\": \"trying\",\n    \"doesn'\": \"does not\",\n    'howdo': 'how do',\n    'whatare': 'what are',\n    'howcan': 'how can',\n    'howmuch': 'how much',\n    'howmany': 'how many',\n    'whydo': 'why do',\n    'doI': 'do I',\n    'theBest': 'the best',\n    'howdoes': 'how does',\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#! pip install regex","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import regex\nspells = {\n    'colour': 'color',\n    'centre': 'center',\n    'favourite': 'favorite',\n    'travelling': 'traveling',\n    'counselling': 'counseling',\n    'theatre': 'theater',\n    'cancelled': 'canceled',\n    'labour': 'labor',\n    'organisation': 'organization',\n    'wwii': 'world war 2',\n    'citicise': 'criticize',\n    'youtu.be': 'youtube',\n    'youtu ': 'youtube ',\n    'qoura': 'quora',\n    'sallary': 'salary',\n    'Whta': 'what',\n    'whta': 'what',\n    'narcisist': 'narcissist',\n    'mastrubation': 'masturbation',\n    'mastrubate': 'masturbate',\n    \"mastrubating\": 'masturbating',\n    'pennis': 'penis',\n    'Etherium': 'ethereum',\n    'etherium': 'ethereum',\n    'narcissit': 'narcissist',\n    'bigdata': 'big data',\n    '2k17': '2017',\n    '2k18': '2018',\n    '2k19': '2020',\n    'qouta': 'quota',\n    'exboyfriend': 'ex boyfriend',\n    'exgirlfriend': 'ex girlfriend',\n    'airhostess': 'air hostess',\n    'whst': 'what',\n    'watsapp': 'whatsapp',\n    'demonitisation': 'demonetization',\n    'demonitization': 'demonetization',\n    'demonetisation': 'demonetization',\n    'quorans': 'quora user',\n    'quoran': 'quora user',\n    'pokémon': 'pokemon',\n    'bacteries': 'batteries', \n    'yr old': 'years old',\n}\n\ncodes = ['\\x7f', '\\u200b', '\\xa0', '\\ufeff', '\\u200e', '\\u202a', '\\u202c', '\\u2060', '\\uf0d8', '\\ue019', '\\uf02d', '\\u200f', '\\u2061', '\\ue01b']\n\n\nlangs1 = r'[\\p{Katakana}\\p{Hiragana}\\p{Han}]' # regex\nlangs2 = r'[ஆய்தஎழுத்துஆயுதஎழுத்துशुषछछशुषدوउसशुष북한내제តើបងប្អូនមានមធ្យបាយអ្វីខ្លះដើម្បីរកឃើញឯកសារអំពីប្រវត្តិស្ត្រនៃប្រាសាទអង្គរវट्टरौरआदસંઘરાજ્યपीतऊनअहএকটিবাড়িএকটিখামারএরঅধীনেপদেরবাছাইপরীক্ষাএরপ্রশ্নওউত্তরসহকোথায়পেতেপারিص、。Емелядуракلكلمقاممقال수능ί서로가를행복하게기乡국고등학교는몇시간업니《》싱관없어나이रचा키کپڤ」मिलगईकलेजेकोठंडकऋॠऌॡर]'\ncompiled_langs1 = regex.compile(langs1)\ncompiled_langs2 = re.compile(langs2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _clean_math(x, compiled_re):\n    return compiled_re.sub(' <math> ', x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(x):\n    \n    x = str(x).lower()\n    return x\n    \ndef _clean_unicode(x):\n    for u in codes:\n        if u in x:\n            x = x.replace(u, '')\n    return x\n\ndef clean_math(df):\n    math_puncts = 'θπα÷⁴≠β²¾∫≥⇒¬∠＝∑Φ√½¼'\n    math_puncts_long = [r'\\\\frac', r'\\[math\\]', r'\\[/math\\]', r'\\\\lim']\n    compiled_math = re.compile('(%s)' % '|'.join(math_puncts))\n    compiled_math_long = re.compile('(%s)' % '|'.join(math_puncts_long))\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_math(x, compiled_math_long))\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_math(x, compiled_math))\n    return df\n\ndef clean_abbreviation(df, abbreviations):\n    compiled_abbreviation = re.compile('(%s)' % '|'.join(abbreviations.keys()))\n    #print (compiled_abbreviation)\n    def replace(match):\n        return abbreviations[match.group(0)]\n    df['question_text'] = df[\"question_text\"].apply(\n        lambda x: _clean_abreviation(x, compiled_abbreviation, replace)\n    )\n    return df\n\ndef _clean_abreviation(x, compiled_re, replace):\n    return compiled_re.sub(replace, x)\n\n\ndef clean_spells(df, spells):\n    compiled_spells = re.compile('(%s)' % '|'.join(spells.keys()))\n    def replace(match):\n        return spells[match.group(0)]\n    df['question_text'] = df[\"question_text\"].apply(\n        lambda x: _clean_spells(x, compiled_spells, replace)\n    )\n    return df\n    \ndef _clean_spells(x, compiled_re, replace):\n    return compiled_re.sub(replace, x)\n\ndef _clean_language(x, compiled_re):\n    return compiled_re.sub(' <lang> ', x)\n\n\ndef _clean_puncts(x, puncts):\n    x = str(x)\n    # added space around puncts after replace\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\n\n\ndef _clean_space(x, compiled_re):\n    return compiled_re.sub(\" \", x)\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndata[\"question_text\"] = data[\"question_text\"].fillna(\"\").apply(preprocess)\ndata[\"question_text\"] = data[\"question_text\"].fillna(\"\").apply(_clean_unicode)\ndata = clean_math(data)\ndata = clean_abbreviation(data, abbreviations)\ndata = clean_spells(data, spells)\ndata['question_text'] = data['question_text'].apply(lambda x: _clean_language(x, compiled_langs1))\ndata['question_text'] = data['question_text'].apply(lambda x: _clean_language(x, compiled_langs2))\ndata['question_text'] = data['question_text'].apply(lambda x: _clean_puncts(x, puncts))\ncompiled_re = re.compile(r\"\\s+\")\ndata['question_text'] = data[\"question_text\"].apply(lambda x: _clean_space(x, compiled_re))\n#data.to_csv('/content/gdrive/My Drive/CaseStudy/data_after_preprocessing7l.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntest_data[\"question_text\"] = test_data[\"question_text\"].fillna(\"\").apply(preprocess)\ntest_data[\"question_text\"] = test_data[\"question_text\"].fillna(\"\").apply(_clean_unicode)\ntest_data = clean_math(test_data)\ntest_data = clean_abbreviation(test_data, abbreviations)\ntest_data = clean_spells(test_data, spells)\ntest_data['question_text'] = test_data['question_text'].apply(lambda x: _clean_language(x, compiled_langs1))\ntest_data['question_text'] = test_data['question_text'].apply(lambda x: _clean_language(x, compiled_langs2))\ntest_data['question_text'] = test_data['question_text'].apply(lambda x: _clean_puncts(x, puncts))\ncompiled_re = re.compile(r\"\\s+\")\ntest_data['question_text'] = test_data[\"question_text\"].apply(lambda x: _clean_space(x, compiled_re))\n\ndel puncts, spells, codes, langs1, langs2, compiled_langs1, compiled_langs2; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://stackoverflow.com/questions/38420847/apply-standardscaler-to-parts-of-a-data-set\n\ncol_names = [ 'q_len', 'q_words', 'u_chars', 'l_chars',\n       'n_stopwords', 'n_cap_words', 'n_diff_words', 'avg_word_len',\n       'n_numerical_words', 'nb_simley', 'nb_symbols', 'nb_punct']\n\ndata_features = data[col_names]\ntest_data_features = test_data[col_names]\nscaler = StandardScaler().fit(data_features.values)\ntrain_features = scaler.transform(data_features.values)\ntest_features = scaler.transform(test_data_features.values)\n\ndata[col_names] = train_features\ntest_data[col_names] = test_features\ndel data_features, test_data_features, train_features, col_names, test_features, scaler; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#max_features = 194200\nX_train_question = data['question_text'] \nX_test_question = test_data['question_text']\ntokenizer = Tokenizer() \ntokenizer.fit_on_texts(list(X_train_question)) \nvocab_size = len(tokenizer.word_index) + 1 \nprint (vocab_size) \nmax_features = vocab_size -1\ntrain_encoded_docs = tokenizer.texts_to_sequences(X_train_question) \ntest_encoded_docs = tokenizer.texts_to_sequences(X_test_question) \nmax_length = 50\ntrain_padded_docs = pad_sequences(train_encoded_docs, maxlen=max_length, padding='post') \ntest_padded_docs = pad_sequences(test_encoded_docs, maxlen = max_length, padding='post') \nprint(train_padded_docs[0])\n\ndel X_train_question, X_test_question, train_encoded_docs, test_encoded_docs; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_padded_docs = pd.DataFrame(train_padded_docs)\ntest_padded_docs = pd.DataFrame(test_padded_docs)\ntrain_data = pd.concat([train_padded_docs, data], axis=1)\n#test_data = pd.concat([test_padded_docs, data], axis=1)\nprint (train_data.shape)\n#print (test_data.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train_data.target\nX = train_data.drop(['qid', 'target'], axis=1)\n#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state= 40, stratify= y)\ntest_data = test_data.drop(['qid'], axis= 1)\nprint (X.shape)\nprint (y.shape)\nprint (test_data.shape)\n#X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_padded_docs = X.iloc[:,[i for i in range(0,50)]] #[np.arange(0,65)] ]\n#cv_padded_docs = X_test.iloc[:,[i for i in range(0,65)]] #[np.arange(0,65)] ]\nprint (train_padded_docs.shape)\n\n#print (cv_padded_docs.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (len(X.columns))\nX_train = X.iloc[:, [i for i in range(50, len(X.columns))]]\n#X_cv = X_test.iloc[:, [i for i in range(65, len(X_test.columns))]]\nX_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://stackoverflow.com/questions/19371860/python-open-file-from-zip-without-temporary-extracting-it\nimport zipfile\narchive = zipfile.ZipFile('../input/quora-insincere-questions-classification/embeddings.zip', 'r')\n#glove_file = archive.read('glove.840B.300d/glove.840B.300d.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/wowfattie/3rd-place\ndef P(word): \n    \"Probability of `word`.\"\n    # use inverse of rank as proxy\n    # returns 0 if the word isn't in the dictionary\n    return - WORDS.get(word, 0)\n\ndef correction(word): \n    \"Most probable spelling correction for word.\"\n    return max(candidates(word), key=P)\ndef candidates(word): \n    \"Generate possible spelling corrections for word.\"\n    return (known([word]) or known(edits1(word)) or [word])\ndef known(words): \n    \"The subset of `words` that appear in the dictionary of WORDS.\"\n    return set(w for w in words if w in WORDS)\n\ndef edits1(word):\n    \"All edits that are one edit away from `word`.\"\n    letters    = 'abcdefghijklmnopqrstuvwxyz'\n    splits     = [(word[:i], word[i:])    for i in range(len(word) + 1)]\n    deletes    = [L + R[1:]               for L, R in splits if R]\n    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]\n    replaces   = [L + c + R[1:]           for L, R in splits if R for c in letters]\n    inserts    = [L + c + R               for L, R in splits for c in letters]\n    return set(deletes + transposes + replaces + inserts)\ndef edits2(word): \n    \"All edits that are two edits away from `word`.\"\n    return (e2 for e1 in edits1(word) for e2 in edits1(e1))\ndef singlify(word):\n    return \"\".join([letter for i,letter in enumerate(word) if i == 0 or letter != word[i-1]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\n\ndef get_coefs(word,*arr): \n    return word, np.asarray(arr, dtype='float32')\n\nembeddings_index = {}\nfor o in tqdm(archive.open('glove.840B.300d/glove.840B.300d.txt', 'r')):    \n    o = o.decode(\"utf-8\")\n    key , value =  get_coefs(*o.split(\" \"))\n    embeddings_index.update({key:value})\n\nwords =  list(embeddings_index.keys())\n\nw_rank = {}\n\nfor i,word in enumerate(words):\n    w_rank[word] = i\nWORDS = w_rank","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix_1 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n\nunknown_vector = np.zeros((300,), dtype=np.float32) - 1.\nprint(unknown_vector[:5])\nfor key, i in word_index.items():\n    word = key\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_1[i] = embedding_vector\n        continue\n    word = key.upper()\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_1[i] = embedding_vector\n        continue\n    word = key.capitalize()\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_1[i] = embedding_vector\n        continue\n    word = ps.stem(key)\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_1[i] = embedding_vector\n        continue\n    word = lc.stem(key)\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_1[i] = embedding_vector\n        continue\n    word = sb.stem(key)\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_1[i] = embedding_vector\n        continue\n \n    if i> 1:\n        word = correction(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix_1[i] = embedding_vector\n            continue\n    \n    embedding_matrix_1[i] = unknown_vector\n\ndel embeddings_index\ndel unknown_vector\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#embedding_matrix_1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embeddings_index = {}\ni = 0\nfor o in tqdm(archive.open('paragram_300_sl999/paragram_300_sl999.txt', 'r')):   \n    try:\n        o = o.decode(\"utf-8\").strip()\n        if len(o)>100:\n            key , value =  get_coefs(*o.split(\" \"))\n            if len(value) == 300:\n                embeddings_index.update({key:value})\n            else:\n                i += 1\n    except:\n        continue\nprint (i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix_2 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n\nunknown_vector = np.zeros((300,), dtype=np.float32) - 1.\nprint(unknown_vector[:5])\nfor key, i in word_index.items():\n    word = key\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_2[i] = embedding_vector\n        continue\n    word = key.upper()\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_2[i] = embedding_vector\n        continue\n    word = key.capitalize()\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_2[i] = embedding_vector\n        continue\n    word = ps.stem(key)\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_2[i] = embedding_vector\n        continue\n    word = lc.stem(key)\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_2[i] = embedding_vector\n        continue\n    word = sb.stem(key)\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_2[i] = embedding_vector\n        continue\n\n    \n    if i> 1:\n        word = correction(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix_2[i] = embedding_vector\n            continue\n    \n    embedding_matrix_2[i] = unknown_vector\n\ndel embeddings_index\ndel unknown_vector\ngc.collect()\n\ndel data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embeddings_index = {}\ni = 0\nfor o in tqdm(archive.open('wiki-news-300d-1M/wiki-news-300d-1M.vec', 'r')):   \n    try:\n        o = o.decode(\"utf-8\").strip()\n        if len(o)>100:\n            key , value =  get_coefs(*o.split(\" \"))\n            if len(value) == 300:\n                embeddings_index.update({key:value})\n            else:\n                i += 1\n    except:\n        continue\nprint (i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix_3 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n\nunknown_vector = np.zeros((300,), dtype=np.float32) - 1.\nprint(unknown_vector[:5])\nfor key, i in word_index.items():\n    word = key\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_3[i] = embedding_vector\n        continue\n    word = key.upper()\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_3[i] = embedding_vector\n        continue\n    word = key.capitalize()\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_3[i] = embedding_vector\n        continue\n    word = ps.stem(key)\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_3[i] = embedding_vector\n        continue\n    word = lc.stem(key)\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_3[i] = embedding_vector\n        continue\n    word = sb.stem(key)\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix_3[i] = embedding_vector\n        continue\n\n    \n    if i> 1:\n        word = correction(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix_3[i] = embedding_vector\n            continue\n    \n    embedding_matrix_3[i] = unknown_vector\n\ndel embeddings_index\ndel unknown_vector\ngc.collect()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#embedding_matrix = np.concatenate((embedding_matrix_1 , embedding_matrix_2), axis=1)\nembedding_matrix = np.concatenate((embedding_matrix_1 , embedding_matrix_2, embedding_matrix_3), axis=1)\ndel embedding_matrix_1, embedding_matrix_2, embedding_matrix_3;\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print (test_data.columns)\nX_train_stat = X_train.drop(['question_text'], axis = 1)\ntest_data1 = test_data.drop(['question_text'], axis=1)\nprint (test_data1.shape)\nprint (X_train_stat.shape)\ntest_padded_docs.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_cv = pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# code inspired from: https://github.com/anandsaha/pytorch.cyclic.learning.rate/blob/master/cls.py\n# https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate/code\n\nclass CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())\n    \n\ndef f1(y_true, y_pred):\n    '''\n    metric from here \n    https://stackoverflow.com/questions/43547402/how-to-calculate-f1-macro-in-keras\n    '''\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function API in Keras https://machinelearningmastery.com/keras-functional-api-deep-learning/ \nfrom keras.layers.normalization import BatchNormalization \n# from keras.initializers import RandomNormal \n# from keras import regularizers \n# from keras.layers import Conv1D\n# from keras.layers import Flatten","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input1 = Input(shape=(50,)) \n#input1 = Input(shape=(100,), name = 'Input_sequence_Text')\nx = Embedding(max_features, 300 * 3, weights=[embedding_matrix], input_length= 50,  trainable=False, name = 'Embedding')(input1)\nx = SpatialDropout1D(0.4, name='SpatialDropout')(x)\nx = Bidirectional(LSTM(256, return_sequences=True), name= 'BidirectionLSTM128')(x)\n#bidirectionLSTM2 = Bidirectional(CuDNNGRU(128, return_sequences=True), name= 'BidirectionLSTM2')(bidirectionLSTM)\nx = Conv1D(64, kernel_size= 1, name='1D_Convolution64')(x)\n#maxpool = AVGM\nmax_pool = GlobalMaxPooling1D(name=\"GlobalMaxPool\")(x) \n#flattan1 = Flatten(name= 'Flatten1')(max_pool)\n\ninput2 = Input(shape=(12,), name = 'input_stat_featues') \nembed2 = Embedding(22, 40)(input2)\nconv2 = Conv1D(64, kernel_size= 3, activation='relu', kernel_regularizer= regularizers.l2(0.002), name='Conv1d')(embed2) \nflatten2 = Flatten(name='Flatten2')(conv2)\n\nx = concatenate([max_pool, flatten2], name='Concatenate')\nx = Dense(128, activation=\"relu\", name='1Dense128')(x)\nx = Dropout(0.1, name='Dropout2')(x)\nx = BatchNormalization(name='BatchNormalization')(x)\n#dense3 = Dense(64, activation=\"relu\", name='2Dense64')(batchnormal)\n\n\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel1 = Model(inputs=[input1, input2], outputs=x)\n#print (model3.summary())\nmodel1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clr = CyclicLR(base_lr=0.001, max_lr=0.002,\n               step_size=300., mode='exp_range',\n               gamma=0.99994)\n\nmodel1.fit([train_padded_docs, X_train_stat], y, batch_size=512, epochs=4, callbacks = [clr,])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pred1 = model1.predict([train_padded_docs, X_train_stat], batch_size=512, verbose=1, callbacks = [clr, ]) \ntest_pred1 = model1.predict([test_padded_docs, test_data1], batch_size=512, verbose=1, callbacks=[clr, ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input1 = Input(shape=(50,)) \n#input1 = Input(shape=(100,), name = 'Input_sequence_Text')\nembedding = Embedding(max_features, 300 * 3, weights=[embedding_matrix], input_length= 50,  trainable=False, name = 'Embedding')(input1)\ndropout = SpatialDropout1D(0.4, name='SpatialDropout')(embedding)\nbidirectionLSTM = Bidirectional(LSTM(128, return_sequences=True), name= 'BidirectionLSTM128')(dropout)\nbidirectionLSTM2 = Bidirectional(LSTM(128, return_sequences=True), name= 'BidirectionLSTM2')(bidirectionLSTM)\n#conv11 = Conv1D(64, kernel_size= 1, name='1D_Convolution64')(bidirectionLSTM)\n#maxpool = AVGM\nmax_pool1 = GlobalMaxPooling1D(name=\"GlobalMaxPool\")(bidirectionLSTM)\nmax_pool2 = GlobalMaxPooling1D(name=\"GlobalMaxPool2\")(bidirectionLSTM2) \n#flattan1 = Flatten(name= 'Flatten1')(max_pool)\nconc = Concatenate()([max_pool1, max_pool2])\n\ninput2 = Input(shape=(12,), name = 'input_stat_featues') \nembed2 = Embedding(12, 50)(input2)\nconv2 = Conv1D(64, kernel_size= 3, activation='relu', kernel_regularizer= regularizers.l2(0.002), name='Conv1d')(embed2) \nflatten2 = Flatten(name='Flatten2')(conv2)\n\nmerge = concatenate([conc, flatten2], name='Concatenate')\ndense64 = Dense(128, activation=\"relu\", name='1Dense128')(merge)\ndropout2 = Dropout(0.1, name='Dropout2')(dense64)\nbatchnormal = BatchNormalization(name='BatchNormalization')(dropout2)\n#dense3 = Dense(64, activation=\"relu\", name='2Dense64')(batchnormal)\n\n\nfinal = Dense(1, activation=\"sigmoid\")(batchnormal)\nmodel1 = Model(inputs=[input1, input2], outputs=final)\n\nmodel1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1])\nmodel1.fit([train_padded_docs, X_train_stat], y, batch_size=512, epochs= 4, callbacks = [clr,])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pred2 = model1.predict([train_padded_docs, X_train_stat], batch_size=512, verbose=1, callbacks = [clr, ]) \ntest_pred2 = model1.predict([test_padded_docs, test_data1], batch_size=512, verbose=1, callbacks=[clr, ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input1 = Input(shape=(50,)) \n#input1 = Input(shape=(100,), name = 'Input_sequence_Text')\nembedding = Embedding(max_features, 300 * 3, weights=[embedding_matrix], input_length= 50,  trainable=False, name = 'Embedding')(input1)\ndropout = SpatialDropout1D(0.3, name='SpatialDropout')(embedding)\nbidirectionLSTM = Bidirectional(LSTM(256, return_sequences=True), name= 'BidirectionLSTM128')(dropout)\n\n\nx1 = Conv1D(100, activation='relu', kernel_size=1, \n                padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=110000))(bidirectionLSTM)\nx2 = Conv1D(80, activation='relu', kernel_size=2, \n                padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=120000))(bidirectionLSTM)\nx3 = Conv1D(30, activation='relu', kernel_size=3, \n                padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=130000))(bidirectionLSTM)\nx4 = Conv1D(12, activation='relu', kernel_size=5, \n                padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=140000))(bidirectionLSTM)\n\n\n\nx1 = GlobalMaxPooling1D()(x1)\nx2 = GlobalMaxPooling1D()(x2)\nx3 = GlobalMaxPooling1D()(x3)\n\nx4 = GlobalMaxPooling1D()(x4)\nc = concatenate([x1, x2, x3, x4])\n\n\n#flattan1 = Flatten(name= 'Flatten1')(max_pool)\n\ninput2 = Input(shape=(12,), name = 'input_stat_featues') \nembed2 = Embedding(12, 40)(input2)\nconv2 = Conv1D(64, kernel_size= 3, activation='relu', kernel_regularizer= regularizers.l2(0.002), name='Conv1d')(embed2) \nflatten2 = Flatten(name='Flatten2')(conv2)\n\nmerge = concatenate([c, flatten2], name='Concatenate')\nx = Dense(128, activation=\"relu\", name='1Dense128')(merge)\nx = Dropout(0.1, name='Dropout2')(x)\nx = BatchNormalization(name='BatchNormalization')(x)\n#dense3 = Dense(64, activation=\"relu\", name='2Dense64')(batchnormal)\n\n\nfinal = Dense(1, activation=\"sigmoid\")(x)\nmodel1 = Model(inputs=[input1, input2], outputs=final)\n\nmodel1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1.fit([train_padded_docs, X_train_stat], \n          y, \n          batch_size=512, epochs=4, callbacks = [clr,],\n          )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pred4 = model1.predict([train_padded_docs, X_train_stat], batch_size= 512, verbose=1, callbacks = [clr, ])\ntest_pred4 = model1.predict([test_padded_docs, test_data1], batch_size= 512, verbose=1, callbacks=[clr, ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_val_y = 0.3 * train_pred1  +  0.3 * train_pred2 + 0.4 * train_pred4 \npred_test_y = 0.3 * test_pred1 + 0.3 * test_pred2 + 0.4 * test_pred4 \n\nthresholds = []\nfor thresh in np.arange(0.1, 0.5, 0.01):\n    thresh = np.round(thresh, 2)\n    res = f1_score(y, (pred_val_y > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y = (pred_test_y > best_thresh ).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_cv['prediction'] = pred_test_y\nprint (submission_cv.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_cv.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}