{"cells":[{"metadata":{"_uuid":"281275d20b3c5f659236d880031e3da4b7977e06"},"cell_type":"markdown","source":"# Import Data"},{"metadata":{"_uuid":"ebb7ec8cdfe9b840f5b5ba1524f4e892994e46cc","trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\n\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"713132d2f045501c82e42290550d356ae0727c35","trusted":false},"cell_type":"code","source":"DATA_ROOT = '../input/'\nORIGINAL_DATA_FOLDER = os.path.join(DATA_ROOT, 'movie-review-sentiment-analysis-kernels-only')\nTMP_DATA_FOLDER = os.path.join(DATA_ROOT, 'kaggle_review_sentiment_tmp_data')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"98fd884ce5265326d434498bda1f5bb813f24e95","trusted":false},"cell_type":"code","source":"train_data_path = os.path.join(ORIGINAL_DATA_FOLDER, 'train.tsv')\ntest_data_path = os.path.join(ORIGINAL_DATA_FOLDER, 'test.tsv')\nsub_data_path = os.path.join(ORIGINAL_DATA_FOLDER, 'sampleSubmission.csv')\n\ntrain_df = pd.read_csv(train_data_path, sep=\"\\t\")\ntest_df = pd.read_csv(test_data_path, sep=\"\\t\")\nsub_df = pd.read_csv(sub_data_path, sep=\",\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ee07dfaceeb2d800b4c95430a5f44d7386fed119"},"cell_type":"markdown","source":"# EDA"},{"metadata":{"_uuid":"abc33cf6801467c6c8ccd5266961d5e74a70b030","trusted":false},"cell_type":"code","source":"import seaborn as sns\nfrom sklearn.feature_extraction import text as sktext","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"33485dab7ac4678cc77fed13b2c8e500d341f72f","trusted":false},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2e0e484aacbb21ebb2807c79beca5aa054c3830","trusted":false},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"641a74568482c8ad42d1ac1c8982a44b84641166","trusted":false},"cell_type":"code","source":"sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c4d3c60c638b8ed36a90102e2f5d0fcb034b9ac4"},"cell_type":"markdown","source":"## Find Overlapped Phrases Between Train and Test Data"},{"metadata":{"_uuid":"0a31536ca50b348303c4980a7535fdea69716d6e","trusted":false},"cell_type":"code","source":"overlapped = pd.merge(train_df[[\"Phrase\", \"Sentiment\"]], test_df, on=\"Phrase\", how=\"inner\")\noverlap_boolean_mask_test = test_df['Phrase'].isin(overlapped['Phrase'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2de53b6baefcd985866f4e3789cb923e7c8d57c8"},"cell_type":"markdown","source":"## Histogram of phrase length"},{"metadata":{"_uuid":"25a101f200f79abef0c91b2fdce721a22ad66b1d","trusted":false},"cell_type":"code","source":"print(\"training data phrase length distribution\")\nsns.distplot(train_df['Phrase'].map(lambda ele: len(ele)), kde_kws={\"label\": \"train\"})\n\nprint(\"testing data phrase length distribution\")\nsns.distplot(test_df[~overlap_boolean_mask_test]['Phrase'].map(lambda ele: len(ele)), kde_kws={\"label\": \"test\"})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22c203439a2310d54f0283dc4e117307a1afa82d"},"cell_type":"markdown","source":"## Explore Sentence Id"},{"metadata":{"_uuid":"c957e257b2865c258a6d9ce23b9ed15c316b90b8","trusted":false},"cell_type":"code","source":"print(\"training and testing data sentences hist:\")\nsns.distplot(train_df['SentenceId'], kde_kws={\"label\": \"train\"})\nsns.distplot(test_df['SentenceId'], kde_kws={\"label\": \"test\"})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ae6308dd0bb586f7cc6e75619e228f7984116ab","trusted":false},"cell_type":"code","source":"print(\"The number of overlapped SentenceId between training and testing data:\")\ntrain_overlapped_sentence_id_df = train_df[train_df['SentenceId'].isin(test_df['SentenceId'])]\nprint(train_overlapped_sentence_id_df.shape[0])\n\ndel train_overlapped_sentence_id_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_uuid":"030ab4a34391188a30160731259e21d4611762a2","trusted":false},"cell_type":"code","source":"pd.options.display.max_colwidth = 250\nprint(\"Example of sentence and phrases: \")\n\nsample_sentence_id = train_df.sample(1)['SentenceId'].values[0]\nsample_sentence_group_df = train_df[train_df['SentenceId'] == sample_sentence_id]\nsample_sentence_group_df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ebe14dacb40043c8d25657d7f64376f5e2fe26b"},"cell_type":"markdown","source":"## Explore Phrase Text"},{"metadata":{"_uuid":"c238331d1b5017f585082416782f099064d3f1b9","trusted":false},"cell_type":"code","source":"import nltk\nimport gensim\nimport operator \nfrom keras.preprocessing import text as ktext","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63cda17126818957a51f0f0e2080014fc1001be0","trusted":false},"cell_type":"code","source":"def build_vocab(texts):\n    tk = ktext.Tokenizer(lower = True, filters='')\n    tk.fit_on_texts(texts)\n    return tk.word_counts\n\ndef check_coverage(vocab, embeddings_index):\n    known_words = {}\n    unknown_words = {}\n    for word in vocab.keys():\n        if word in embeddings_index:\n            known_words[word] = vocab[word]\n            continue\n        unknown_words[word] = vocab[word]\n\n    print('Found embeddings for {:.3%} of vocab'.format(len(known_words) / len(vocab)))\n    num_known_words = np.sum(np.asarray(list(known_words.values())))\n    num_unknown_words = np.sum(np.asarray(list(unknown_words.values())))\n    print('Found embeddings for  {:.3%} of all text'.format(float(num_known_words) / (num_known_words + num_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n\n    return unknown_words","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"31f8de19d740ae069dd8b7ef50ba058908eb5eb5"},"cell_type":"markdown","source":"#### Build Vocabulary"},{"metadata":{"_uuid":"6f752749067c6c356693877e69be822cfa8a5b33","trusted":false},"cell_type":"code","source":"texts = list(train_df['Phrase'].values) + list(test_df['Phrase'].values)\nvocab = build_vocab(texts)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b3d513b3174d12e90a85bbe10902ae3ca945f261"},"cell_type":"markdown","source":"#### Load Embedding"},{"metadata":{"_uuid":"baac82443ed73deb1cb58dc953187b98cf3076b9","trusted":false},"cell_type":"code","source":"def load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr[:len(arr)-1], dtype='float32')\n    \n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o)>15)\n        \n    return embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a2a23b09c2d335df392c94308b2fe52490919015","trusted":false},"cell_type":"code","source":"pretrained_w2v_path = os.path.join(DATA_ROOT, \"nlpword2vecembeddingspretrained/GoogleNews-vectors-negative300.bin\")\nw2v_google = gensim.models.KeyedVectors.load_word2vec_format(pretrained_w2v_path, binary=True).wv\n\npretrained_w2v_path = os.path.join(DATA_ROOT, \"fasttext-crawl-300d-2m/crawl-300d-2M.vec\")\nw2v_fasttext = load_embed(pretrained_w2v_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4eb50a4c137afa85d6f14ab33fca67e048128f7"},"cell_type":"markdown","source":"#### Check Vocabulary Coverage"},{"metadata":{"_uuid":"631006a1d0ae669fbfd1d5a2eb4df0e7cd269b5c","trusted":false},"cell_type":"code","source":"print(\"google\")\nunknown_vocab = check_coverage(vocab, w2v_google)\nprint(\"unknown vocabulary:\")\nprint(unknown_vocab[:50])\n\nprint(\"\\n\")\n\nprint(\"fast text\")\nunknown_vocab = check_coverage(vocab, w2v_fasttext)\nprint(\"unknown vocabulary:\")\nprint(unknown_vocab[:50])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86959890d0944cd3c85201f0107881ae82ce2ad6"},"cell_type":"markdown","source":"1. There are overlapped phrase texts between training and testing data, which should assign training data labels directly instead of getting from prediction.\n2. Max text length should be set around 60.\n3. There is no overlapped sentence between training and testing data. Within each sentence group, the phraseId order is the in-order tanversal over the parsing tree of the sentence text. (This might be a very important information as we can utilized the composition as powerful predictive information). \n4. Fast Text has higher vocabulary coverage rate. We are able to correct some of oov tokens.\n"},{"metadata":{"_uuid":"2a37cd70ed44125b3c27e1f0f204c7207e09f175","trusted":false},"cell_type":"code","source":"w2v = w2v_fasttext\n# del w2v_google, w2v_fasttext, texts, vocab\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"09c94783b2b9f99c5d62ea251391642329ec46a6"},"cell_type":"markdown","source":"# Data Preprocessing"},{"metadata":{"_uuid":"f15786c0c632835ae7a008114319badd92dce4c8","trusted":false},"cell_type":"code","source":"from keras.preprocessing import sequence\nimport gensim\nfrom sklearn import preprocessing as skp","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b50c845cca35a18a4f2986d03b37eb826c17e109","trusted":false},"cell_type":"code","source":"max_len = 50\nembed_size = 300\nmax_features = 30000","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5cd1af62c09ef764aed364830a0d2485ec9bb9af"},"cell_type":"markdown","source":"## Clean Texts"},{"metadata":{"_uuid":"a9993bb5735c64ea02463a580ba2b8c720eaa1bf"},"cell_type":"markdown","source":"### Clean Contractions"},{"metadata":{"_uuid":"966cfe2876258a69abf637bb6a96c3d594ccc587","trusted":false},"cell_type":"code","source":"contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", \"n't\": \"not\", \"'ve\": \"have\"}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9d6602592302e1e443bda09a27b1f34126c07ebf","trusted":false},"cell_type":"code","source":"def known_contractions(embed):\n    known = []\n    for contract in contraction_mapping:\n        if contract in embed:\n            known.append(contract)\n    return known","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e4167a4154d902d1a0d9f0cc9f5e486decf8395","trusted":false},"cell_type":"code","source":"known_contract_list = known_contractions(w2v)\nprint(known_contract_list)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"099a292ee4b2bb5395bd88bf932b8b2ad4ff3a24","trusted":false},"cell_type":"code","source":"def clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"55d7df7c8ce9d61de3c508d9659b5778369fbd9f","trusted":false},"cell_type":"code","source":"train_df.loc[:, 'Phrase'] = train_df['Phrase'].map(lambda text: clean_contractions(text, contraction_mapping))\ntest_df.loc[:, 'Phrase'] = test_df['Phrase'].map(lambda text: clean_contractions(text, contraction_mapping))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7640bddb16132717157fd59155343ed2f9809eb5","trusted":false},"cell_type":"code","source":"full_text = list(train_df['Phrase'].values) + list(test_df['Phrase'].values)\nvocab = build_vocab(full_text)\ncheck_coverage(vocab, w2v)\nprint(\"\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cb26beaba5b28b122f15e319bec658334856a3c6"},"cell_type":"markdown","source":"### Clean Special Characters"},{"metadata":{"_uuid":"9cea2d92050dc84971a21586c8b172796cbdc4e2","trusted":false},"cell_type":"code","source":"punct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6e055fafbd9e6296dfac66af814b844941a3ef9","trusted":false},"cell_type":"code","source":"punct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43915a0a1afe622358d2a189e720e2ba31620357","trusted":false},"cell_type":"code","source":"def unknown_punct(embed, punct):\n    unknown = ''\n    for p in punct:\n        if p not in embed:\n            unknown += p\n            unknown += ' '\n    return unknown\n\ndef clean_special_chars(text, punct, mapping):\n    for p in mapping:\n        text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c0bb0f76b1b8186548094d7dc6fddc2b52e4feaa","trusted":false},"cell_type":"code","source":"print(unknown_punct(w2v, punct))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a0a364a0f3ddeb433ed2630cefb9522c66be2d59","trusted":false},"cell_type":"code","source":"train_df.loc[:, 'Phrase'] = train_df['Phrase'].map(lambda text: clean_special_chars(text, punct, punct_mapping))\ntest_df.loc[:, 'Phrase'] = test_df['Phrase'].map(lambda text: clean_special_chars(text, punct, punct_mapping))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"281a1ea87df13b90cb0e5435c4982b524c5c939d","trusted":false},"cell_type":"code","source":"full_text = list(train_df['Phrase'].values) + list(test_df['Phrase'].values)\nvocab = build_vocab(full_text)\nunknown_vocab = check_coverage(vocab, w2v)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7a1a1fb9bce9e51aa4d9df3c70be15e84627da2"},"cell_type":"markdown","source":"What left are actually names entities"},{"metadata":{"_uuid":"f1d6dc7fd2cc3c99e7865819ee6a80728cbef563"},"cell_type":"markdown","source":"### Map The Rest OOV Tokens to \"[ name ]\""},{"metadata":{"_uuid":"42212f7fadb89fedc561e8c19b7a5e8bf820fad5","trusted":false},"cell_type":"code","source":"def map_unknown_token(text, dst_token, unknown_vocab_set):\n#     token_list = []\n#     for t in text.split(\" \"):\n#         if t in unknown_vocab_set:\n#             token_list.append(dst_token)\n#         else:\n#             token_list.append(t)\n    \n#     return \" \".join(token_list)\n    return' '.join([dst_token if t.lower() in unknown_vocab_set else t for t in text.split(\" \")])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f1493d9ec62c60202552259541d0cd1a221e222","trusted":false},"cell_type":"code","source":"unknown_vocab_set = set(list(map(\n    lambda unknown_vocab_tuple: unknown_vocab_tuple[0],\n    unknown_vocab\n)))\ntrain_df.loc[:, 'Phrase'] = train_df['Phrase'].map(lambda ele: map_unknown_token(ele, \"[ name ]\", unknown_vocab_set))\ntest_df.loc[:, 'Phrase'] = test_df['Phrase'].map(lambda ele: map_unknown_token(ele, \"[ name ]\", unknown_vocab_set))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d5d8fd18f1bbb7c2f88a39684170ecb65533a99","trusted":false},"cell_type":"code","source":"full_text = list(train_df['Phrase'].values) + list(test_df['Phrase'].values)\nvocab = build_vocab(full_text)\nunknown_vocab = check_coverage(vocab, w2v)\nprint(unknown_vocab)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"69e929bce14433c86cda53d35abc91a3b3fd4e72"},"cell_type":"markdown","source":"### Tokenize Text"},{"metadata":{"_uuid":"f05a0bcf40751a8e33814fd3314b7f7e1d5149e5","trusted":false},"cell_type":"code","source":"full_text = list(train_df['Phrase'].values) + list(test_df[~overlap_boolean_mask_test]['Phrase'].values)\n\ntk = ktext.Tokenizer(lower = True, filters='')\ntk.fit_on_texts(full_text)\ntrain_tokenized = tk.texts_to_sequences(train_df['Phrase'])\ntest_tokenized = tk.texts_to_sequences(test_df[~overlap_boolean_mask_test]['Phrase'])\n\nX_train = sequence.pad_sequences(train_tokenized, maxlen = max_len)\nX_test = sequence.pad_sequences(test_tokenized, maxlen = max_len)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d42d7f7ad3c09838e1648beb5743dbf4e313a543"},"cell_type":"markdown","source":"### Build embedding matrix"},{"metadata":{"_uuid":"3467eb40c92d5490a26629f6ab6c7505b8439979","trusted":false},"cell_type":"code","source":"word_index = tk.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.zeros((nb_words + 1, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = None\n    if word in w2v:\n        embedding_vector = w2v[word]\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \ndel w2v\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26e06e8aeabd698b0fe2fb208f2c4ec2855babcd"},"cell_type":"markdown","source":"### Encode labels"},{"metadata":{"_uuid":"57a0df2d2e98b1b1a8edc5ef87dd26f4f1f5d63c","trusted":false},"cell_type":"code","source":"y_train = train_df['Sentiment']\n\nled = skp.LabelEncoder()\nled.fit(y_train.values)\n\ny_train = led.transform(y_train.values)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4675184968f46ed87fbd703696978dc3535ce89e"},"cell_type":"markdown","source":"# Define Keras Model"},{"metadata":{"_uuid":"391b187f5f79d60fc329dd8a805708ac71937b5b","trusted":false},"cell_type":"code","source":"import tensorflow as tf\n\nfrom keras import callbacks as kc\nfrom keras import optimizers as ko\nfrom keras import initializers, regularizers, constraints\nfrom keras.engine import Layer\nimport keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"413a0fcd52787baf4cb702cc1074b8086675b1ba"},"cell_type":"markdown","source":"## Define Attention Layer"},{"metadata":{"_uuid":"e59457ba4f41bd4082188ccacff7f1cf8601545c","trusted":false},"cell_type":"code","source":"def _dot_product(x, kernel):\n    \"\"\"\n    Wrapper for dot product operation, in order to be compatible with both\n    Theano and Tensorflow\n    Args:\n        x (): input\n        kernel (): weights\n    Returns:\n    \"\"\"\n    if K.backend() == 'tensorflow':\n        # todo: check that this is correct\n        return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)\n    else:\n        return K.dot(x, kernel)\n    \n    \nclass AttentionWeight(Layer):\n    \"\"\"\n        This code is a modified version of cbaziotis implementation:  GithubGist cbaziotis/AttentionWithContext.py\n        Attention operation, with a context/query vector, for temporal data.\n        Supports Masking.\n        Follows the work of Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf]\n        \"Hierarchical Attention Networks for Document Classification\"\n        by using a context vector to assist the attention\n        # Input shape\n            3D tensor with shape: `(samples, steps, features)`.\n        # Output shape\n            2D tensor with shape: `(samples, steps)`.\n        :param kwargs:\n        Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n        The dimensions are inferred based on the output shape of the RNN.\n        Example:\n            model.add(LSTM(64, return_sequences=True))\n            model.add(AttentionWeight())\n        \"\"\"\n\n    def __init__(self,\n                 W_regularizer=None, u_regularizer=None, b_regularizer=None,\n                 W_constraint=None, u_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.u_regularizer = regularizers.get(u_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.u_constraint = constraints.get(u_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        super(AttentionWeight, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1], input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        if self.bias:\n            self.b = self.add_weight((input_shape[-1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n\n        self.u = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_u'.format(self.name),\n                                 regularizer=self.u_regularizer,\n                                 constraint=self.u_constraint)\n\n        super(AttentionWeight, self).build(input_shape)\n\n    def compute_mask(self, input, input_mask=None):\n        # do not pass the mask to the next layers\n        return None\n\n    def call(self, x, mask=None):\n        uit = _dot_product(x, self.W)\n\n        if self.bias:\n            uit += self.b\n\n        uit = K.tanh(uit)\n        ait = _dot_product(uit, self.u)\n\n        a = K.exp(ait)\n\n        # apply mask after the exp. will be re-normalized next\n        if mask is not None:\n            # Cast the mask to floatX to avoid float64 upcasting in theano\n            a *= K.cast(mask, K.floatx())\n\n        # in some cases especially in the early stages of training the sum may be almost zero\n        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.\n        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        return a\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0], input_shape[1]\n\n    def get_config(self):\n        config = {\n            'W_regularizer': regularizers.serialize(self.W_regularizer),\n            'u_regularizer': regularizers.serialize(self.u_regularizer),\n            'b_regularizer': regularizers.serialize(self.b_regularizer),\n            'W_constraint': constraints.serialize(self.W_constraint),\n            'u_constraint': constraints.serialize(self.u_constraint),\n            'b_constraint': constraints.serialize(self.b_constraint),\n            'bias': self.bias\n        }\n        base_config = super(AttentionWeight, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3e4ae86f2e94275e3bc3de33a5eb9b1f8327caa"},"cell_type":"markdown","source":"## Define Models"},{"metadata":{"_uuid":"ab6fd1ea52e5cdeb110d5ef79e0cde12b195c586","trusted":false},"cell_type":"code","source":"def is_integer(val):\n    return isinstance(val, (int, np.int_))\n\ndef predict(keras_model, x, learning_phase=0):\n\n    if isinstance(keras_model.input, list):\n        f = backend.function(\n            keras_model.input + [backend.learning_phase()],\n            [keras_model.output, ]\n        )\n        y = f(tuple(x) + (learning_phase,))[0]\n    else:\n        f = backend.function(\n            [keras_model.input, backend.learning_phase()],\n            [keras_model.output, ]\n        )\n        y = f((x, learning_phase))[0]\n    return y\n    \n\ndef build_birnn_attention_model(\n        voca_dim, time_steps, output_dim, rnn_dim, mlp_dim, \n        item_embedding=None, rnn_depth=1, mlp_depth=1, num_att_channel=1,\n        drop_out=0.5, rnn_drop_out=0., rnn_state_drop_out=0.,\n        trainable_embedding=False, gpu=False, return_customized_layers=False):\n    \"\"\"\n    Create A Bidirectional Attention Model.\n\n    :param voca_dim: vocabulary dimension size.\n    :param time_steps: the length of input\n    :param output_dim: the output dimension size\n    :param rnn_dim: rrn dimension size\n    :param mlp_dim: the dimension size of fully connected layer\n    :param item_embedding: integer, numpy 2D array, or None (default=None)\n        If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor.\n        If item_embedding is a matrix, this matrix will be used as the embedding matrix.\n        If item_embedding is None, then connect input tensor to RNN layer directly.\n    :param rnn_depth: rnn depth\n    :param mlp_depth: the depth of fully connected layers\n    :param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism\n    :param drop_out: dropout rate of fully connected layers\n    :param rnn_drop_out: dropout rate of rnn layers\n    :param rnn_state_drop_out: dropout rate of rnn state tensor\n    :param trainable_embedding: boolean\n    :param gpu: boolean, default=False\n        If True, CuDNNLSTM is used instead of LSTM for RNN layer.\n    :param return_customized_layers: boolean, default=False\n        If True, return model and customized object dictionary, otherwise return model only\n    :return: keras model\n    \"\"\"\n    \n    if item_embedding is not None:\n        inputs = models.Input(shape=(time_steps,), dtype='int32', name='input0')\n        x = inputs\n\n        # item embedding\n        if isinstance(item_embedding, np.ndarray):\n            assert voca_dim == item_embedding.shape[0]\n            x = layers.Embedding(\n                voca_dim, item_embedding.shape[1], input_length=time_steps,\n                weights=[item_embedding, ], trainable=trainable_embedding,\n                mask_zero=False, name='embedding_layer0'\n            )(x)\n        elif utils.is_integer(item_embedding):\n            x = layers.Embedding(\n                voca_dim, item_embedding, input_length=time_steps,\n                trainable=trainable_embedding,\n                mask_zero=False, name='embedding_layer0'\n            )(x)\n        else:\n            raise ValueError(\"item_embedding must be either integer or numpy matrix\")\n    else:\n        inputs = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='input0')\n        x = inputs\n    \n    x = layers.SpatialDropout1D(rnn_drop_out, name='rnn_spatial_droutout_layer')(x)\n\n    if gpu:\n        # rnn encoding\n        for i in range(rnn_depth):\n            x = layers.Bidirectional(\n                layers.CuDNNLSTM(rnn_dim, return_sequences=True),\n                name='bi_lstm_layer' + str(i))(x)\n            x = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x)\n            x = layers.Dropout(rate=rnn_drop_out, name=\"rnn_dropout_layer\" + str(i))(x)\n    else:\n        # rnn encoding\n        for i in range(rnn_depth):\n            x = layers.Bidirectional(\n                layers.LSTM(rnn_dim, return_sequences=True, dropout=rnn_drop_out, recurrent_dropout=rnn_state_drop_out),\n                name='bi_lstm_layer' + str(i))(x)\n            x = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x)\n\n    # attention\n    attention_heads = []\n    x_per = layers.Permute((2, 1), name='permuted_attention_x')(x)\n    for h in range(max(1, num_att_channel)):\n        attention = AttentionWeight(name=\"attention_weights_layer\" + str(h))(x)\n        xx = layers.Dot([2, 1], name='focus_head' + str(h) + '_layer0')([x_per, attention])\n        attention_heads.append(xx)\n\n    if num_att_channel > 1:\n        x = layers.Concatenate(name='focus_layer0')(attention_heads)\n    else:\n        x = attention_heads[0]\n\n    x = layers.BatchNormalization(name='focused_batch_norm_layer')(x)\n    x = layers.Dropout(rate=rnn_drop_out, name=\"focused_dropout_layer\")(x)\n\n    # MLP Layers\n    for i in range(mlp_depth - 1):\n        x = layers.Dense(mlp_dim, activation='selu', kernel_initializer='lecun_normal', name='selu_layer' + str(i))(x)\n        x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)\n\n    outputs = layers.Dense(output_dim, activation=\"softmax\", name=\"softmax_layer0\")(x)\n\n    model = models.Model(inputs, outputs)\n\n    if return_customized_layers:\n        return model, {'AttentionWeight': AttentionWeight}\n    return model\n\n\ndef build_cnn_model(\n        voca_dim, time_steps, output_dim, mlp_dim, num_filters, filter_sizes,\n        item_embedding=None, mlp_depth=1,\n        drop_out=0.5, cnn_drop_out=0.5, pooling='max', padding='valid',\n        trainable_embedding=False, return_customized_layers=False):\n    \"\"\"\n    Create A CNN Model.\n\n    :param voca_dim: vocabulary dimension size.\n    :param time_steps: the length of input\n    :param output_dim: the output dimension size\n    :param num_filters: list of integers\n        The number of filters.\n    :param filter_sizes: list of integers\n        The kernel size.\n    :param mlp_dim: the dimension size of fully connected layer\n    :param item_embedding: integer, numpy 2D array, or None (default=None)\n        If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor.\n        If item_embedding is a matrix, this matrix will be used as the embedding matrix.\n        If item_embedding is None, then connect input tensor to RNN layer directly.\n    :param mlp_depth: the depth of fully connected layers\n    :param drop_out: dropout rate of fully connected layers\n    :param cnn_drop_out: dropout rate of between cnn layer and fully connected layers\n    :param pooling: str, either 'max' or 'average'\n        Pooling method.\n    :param padding: One of \"valid\", \"causal\" or \"same\" (case-insensitive).\n        Padding method.\n    :param trainable_embedding: boolean\n    :param return_customized_layers: boolean, default=False\n        If True, return model and customized object dictionary, otherwise return model only\n    :return: keras model\n    \"\"\"\n\n    if item_embedding is not None:\n        inputs = models.Input(shape=(time_steps,), dtype='int32', name='input0')\n        x = inputs\n\n        # item embedding\n        if isinstance(item_embedding, np.ndarray):\n            assert voca_dim == item_embedding.shape[0]\n            x = layers.Embedding(\n                voca_dim, item_embedding.shape[1], input_length=time_steps,\n                weights=[item_embedding, ], trainable=trainable_embedding,\n                mask_zero=False, name='embedding_layer0'\n            )(x)\n        elif utils.is_integer(item_embedding):\n            x = layers.Embedding(\n                voca_dim, item_embedding, input_length=time_steps,\n                trainable=trainable_embedding,\n                mask_zero=False, name='embedding_layer0'\n            )(x)\n        else:\n            raise ValueError(\"item_embedding must be either integer or numpy matrix\")\n    else:\n        inputs = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='input0')\n        x = inputs\n    \n    x = layers.SpatialDropout1D(cnn_drop_out, name='cnn_spatial_droutout_layer')(x)\n\n    pooled_outputs = []\n    for i in range(len(filter_sizes)):\n        conv = layers.Conv1D(num_filters[i], kernel_size=filter_sizes[i], padding=padding, activation='relu')(x)\n        if pooling == 'max':\n            conv = layers.GlobalMaxPooling1D(name='global_pooling_layer' + str(i))(conv)\n        else:\n            conv = layers.GlobalAveragePooling1D(name='global_pooling_layer' + str(i))(conv)\n        pooled_outputs.append(conv)\n\n    x = layers.Concatenate(name='concated_layer')(pooled_outputs)\n    x = layers.Dropout(cnn_drop_out, name='conv_dropout_layer')(x)\n    x = layers.BatchNormalization(name=\"batch_norm_layer\")(x)\n\n    # MLP Layers\n    for i in range(mlp_depth - 1):\n        x = layers.Dense(mlp_dim, activation='selu', kernel_initializer='lecun_normal', name='selu_layer' + str(i))(x)\n        x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)\n\n    outputs = layers.Dense(output_dim, activation=\"softmax\", name=\"softmax_layer0\")(x)\n\n    model = models.Model(inputs, outputs)\n\n    if return_customized_layers:\n        return model, dict()\n\n    return model\n\n\ndef build_birnn_cnn_model(\n        voca_dim, time_steps, output_dim, rnn_dim, mlp_dim, num_filters, filter_sizes,\n        item_embedding=None, rnn_depth=1, mlp_depth=1,\n        drop_out=0.5, rnn_drop_out=0.5, rnn_state_drop_out=0.5, cnn_drop_out=0.5, pooling='max', padding='valid',\n        trainable_embedding=False, gpu=False, return_customized_layers=False):\n    \"\"\"\n    Create A Bidirectional CNN Model.\n\n    :param voca_dim: vocabulary dimension size.\n    :param time_steps: the length of input\n    :param output_dim: the output dimension size\n    :param rnn_dim: rrn dimension size\n    :param num_filters: list of integers\n        The number of filters.\n    :param filter_sizes: list of integers\n        The kernel size.\n    :param mlp_dim: the dimension size of fully connected layer\n    :param item_embedding: integer, numpy 2D array, or None (default=None)\n        If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor.\n        If item_embedding is a matrix, this matrix will be used as the embedding matrix.\n        If item_embedding is None, then connect input tensor to RNN layer directly.\n    :param rnn_depth: rnn depth\n    :param mlp_depth: the depth of fully connected layers\n    :param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism\n    :param drop_out: dropout rate of fully connected layers\n    :param rnn_drop_out: dropout rate of rnn layers\n    :param rnn_state_drop_out: dropout rate of rnn state tensor\n    :param cnn_drop_out: dropout rate of between cnn layer and fully connected layers\n    :param pooling: str, either 'max' or 'average'\n        Pooling method.\n    :param padding: One of \"valid\", \"causal\" or \"same\" (case-insensitive).\n        Padding method.\n    :param trainable_embedding: boolean\n    :param gpu: boolean, default=False\n        If True, CuDNNLSTM is used instead of LSTM for RNN layer.\n    :param return_customized_layers: boolean, default=False\n        If True, return model and customized object dictionary, otherwise return model only\n    :return: keras model\n    \"\"\"\n\n    if item_embedding is not None:\n        inputs = models.Input(shape=(time_steps,), dtype='int32', name='input0')\n        x = inputs\n\n        # item embedding\n        if isinstance(item_embedding, np.ndarray):\n            assert voca_dim == item_embedding.shape[0]\n            x = layers.Embedding(\n                voca_dim, item_embedding.shape[1], input_length=time_steps,\n                weights=[item_embedding, ], trainable=trainable_embedding,\n                mask_zero=False, name='embedding_layer0'\n            )(x)\n        elif utils.is_integer(item_embedding):\n            x = layers.Embedding(\n                voca_dim, item_embedding, input_length=time_steps,\n                trainable=trainable_embedding,\n                mask_zero=False, name='embedding_layer0'\n            )(x)\n        else:\n            raise ValueError(\"item_embedding must be either integer or numpy matrix\")\n    else:\n        inputs = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='input0')\n        x = inputs\n        \n    x = layers.SpatialDropout1D(rnn_drop_out, name='rnn_spatial_droutout_layer')(x)\n\n    if gpu:\n        # rnn encoding\n        for i in range(rnn_depth):\n            x = layers.Bidirectional(\n                layers.CuDNNLSTM(rnn_dim, return_sequences=True),\n                name='bi_lstm_layer' + str(i))(x)\n            x = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x)\n            x = layers.Dropout(rate=rnn_drop_out, name=\"rnn_dropout_layer\" + str(i))(x)\n    else:\n        # rnn encoding\n        for i in range(rnn_depth):\n            x = layers.Bidirectional(\n                layers.LSTM(rnn_dim, return_sequences=True, dropout=rnn_drop_out, recurrent_dropout=rnn_state_drop_out),\n                name='bi_lstm_layer' + str(i))(x)\n            x = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x)\n\n    pooled_outputs = []\n    for i in range(len(filter_sizes)):\n        conv = layers.Conv1D(num_filters[i], kernel_size=filter_sizes[i], padding=padding, activation='relu')(x)\n        if pooling == 'max':\n            conv = layers.GlobalMaxPooling1D(name='global_pooling_layer' + str(i))(conv)\n        else:\n            conv = layers.GlobalAveragePooling1D(name='global_pooling_layer' + str(i))(conv)\n        pooled_outputs.append(conv)\n\n    x = layers.Concatenate(name='concated_layer')(pooled_outputs)\n    x = layers.BatchNormalization(name=\"batch_norm_layer\")(x)\n    x = layers.Dropout(cnn_drop_out, name='conv_dropout_layer')(x)\n\n    # MLP Layers\n    for i in range(mlp_depth - 1):\n        x = layers.Dense(mlp_dim, activation='selu', kernel_initializer='lecun_normal', name='selu_layer' + str(i))(x)\n        x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)\n\n    outputs = layers.Dense(output_dim, activation=\"softmax\", name=\"softmax_layer0\")(x)\n\n    model = models.Model(inputs, outputs)\n\n    if return_customized_layers:\n        return model, dict()\n\n    return model\n\ndef build_birnn_hierarchy_cnn_model(\n        voca_dim, time_steps, output_dim, rnn_dim, mlp_dim, num_filters, filter_sizes, \n        dilation_rates=1, strides=1,\n        item_embedding=None, rnn_depth=1, mlp_depth=1,\n        drop_out=0.5, rnn_drop_out=0.5, rnn_state_drop_out=0.5, cnn_drop_out=0.5, pooling='max', padding='valid',\n        trainable_embedding=False, gpu=False, return_customized_layers=False):\n    \"\"\"\n    Create A Bidirectional CNN Model.\n\n    :param voca_dim: vocabulary dimension size.\n    :param time_steps: the length of input\n    :param output_dim: the output dimension size\n    :param rnn_dim: rrn dimension size\n    :param num_filters: list of integers\n        The number of filters.\n    :param filter_sizes: list of integers\n        The kernel size.\n    :param mlp_dim: the dimension size of fully connected layer\n    :param item_embedding: integer, numpy 2D array, or None (default=None)\n        If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor.\n        If item_embedding is a matrix, this matrix will be used as the embedding matrix.\n        If item_embedding is None, then connect input tensor to RNN layer directly.\n    :param rnn_depth: rnn depth\n    :param mlp_depth: the depth of fully connected layers\n    :param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism\n    :param drop_out: dropout rate of fully connected layers\n    :param rnn_drop_out: dropout rate of rnn layers\n    :param rnn_state_drop_out: dropout rate of rnn state tensor\n    :param cnn_drop_out: dropout rate of between cnn layer and fully connected layers\n    :param pooling: str, either 'max' or 'average'\n        Pooling method.\n    :param padding: One of \"valid\", \"causal\" or \"same\" (case-insensitive).\n        Padding method.\n    :param trainable_embedding: boolean\n    :param gpu: boolean, default=False\n        If True, CuDNNLSTM is used instead of LSTM for RNN layer.\n    :param return_customized_layers: boolean, default=False\n        If True, return model and customized object dictionary, otherwise return model only\n    :return: keras model\n    \"\"\"\n\n    if item_embedding is not None:\n        inputs = models.Input(shape=(time_steps,), dtype='int32', name='input0')\n        x = inputs\n\n        # item embedding\n        if isinstance(item_embedding, np.ndarray):\n            assert voca_dim == item_embedding.shape[0]\n            x = layers.Embedding(\n                voca_dim, item_embedding.shape[1], input_length=time_steps,\n                weights=[item_embedding, ], trainable=trainable_embedding,\n                mask_zero=False, name='embedding_layer0'\n            )(x)\n        elif utils.is_integer(item_embedding):\n            x = layers.Embedding(\n                voca_dim, item_embedding, input_length=time_steps,\n                trainable=trainable_embedding,\n                mask_zero=False, name='embedding_layer0'\n            )(x)\n        else:\n            raise ValueError(\"item_embedding must be either integer or numpy matrix\")\n    else:\n        inputs = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='input0')\n        x = inputs\n        \n    x = layers.SpatialDropout1D(rnn_drop_out, name='rnn_spatial_droutout_layer')(x)\n\n    if gpu:\n        # rnn encoding\n        for i in range(rnn_depth):\n            x = layers.Bidirectional(\n                layers.CuDNNLSTM(rnn_dim, return_sequences=True),\n                name='bi_lstm_layer' + str(i))(x)\n            x = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x)\n            x = layers.Dropout(rate=rnn_drop_out, name=\"rnn_dropout_layer\" + str(i))(x)\n    else:\n        # rnn encoding\n        for i in range(rnn_depth):\n            x = layers.Bidirectional(\n                layers.LSTM(rnn_dim, return_sequences=True, dropout=rnn_drop_out, recurrent_dropout=rnn_state_drop_out),\n                name='bi_lstm_layer' + str(i))(x)\n            x = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x)\n\n    for i in range(len(filter_sizes)):\n        if is_integer(dilation_rates):\n            di_rate = dilation_rates\n        else:\n            di_rate = dilation_rates[i]\n        \n        if is_integer(strides):\n            std = strides\n        else:\n            std = strides[i]\n            \n        x = layers.Conv1D(num_filters[i], kernel_size=filter_sizes[i], padding=padding, activation='relu', dilation_rate=di_rate, strides=std)(x)\n        \n    if pooling == 'max':\n        x = layers.GlobalMaxPooling1D(name='global_pooling_layer')(x)\n    else:\n        x = layers.GlobalAveragePooling1D(name='global_pooling_layer')(x)\n\n    x = layers.BatchNormalization(name=\"batch_norm_layer\")(x)\n    x = layers.Dropout(cnn_drop_out, name='conv_dropout_layer')(x)\n\n    # MLP Layers\n    for i in range(mlp_depth - 1):\n        x = layers.Dense(mlp_dim, activation='selu', kernel_initializer='lecun_normal', name='selu_layer' + str(i))(x)\n        x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)\n\n    outputs = layers.Dense(output_dim, activation=\"softmax\", name=\"softmax_layer0\")(x)\n\n    model = models.Model(inputs, outputs)\n\n    if return_customized_layers:\n        return model, dict()\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0f8a54b45eb4838d7459e6db57e9f0a95bfe7e60"},"cell_type":"markdown","source":"# Build and Train Models"},{"metadata":{"_uuid":"9dd9a276408dd3543d44e4e82cd8086024f56b3f","trusted":false},"cell_type":"code","source":"from keras.utils import model_to_dot\nfrom keras import models\nfrom keras import layers\n\nimport matplotlib.pyplot as plt\nfrom IPython.display import SVG","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83515918a1861d9aa3ef7751aaf05ca1635d9620","trusted":false},"cell_type":"code","source":"histories = list()\niterations = list()\nmodel_builders = list()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d514af1525ce6dd02614bfdc4c1c707617eceade"},"cell_type":"markdown","source":"## CNN Model"},{"metadata":{"_uuid":"9fcdab9f1bfb6d88b2e3d110dea35f87c71dbf51","trusted":false},"cell_type":"code","source":"def build_model1():\n    voca_dim = embedding_matrix.shape[0]\n    time_steps = max_len\n    output_dim = led.classes_.shape[0]\n    mlp_dim = 50\n    num_filters = [128, 128, 128]\n    filter_sizes = [1, 3, 5]\n    item_embedding = embedding_matrix\n    mlp_depth = 2\n    cnn_drop_out = 0.2\n    mlp_drop_out = 0.2\n    padding = 'causal'\n\n    return build_cnn_model(\n        voca_dim, time_steps, output_dim, mlp_dim, num_filters, filter_sizes, \n        item_embedding=item_embedding, mlp_depth=2, cnn_drop_out=cnn_drop_out,\n        padding=padding,\n        return_customized_layers=True\n    )\n\nmodel_builders.append(build_model1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f5e58b1f129140130e4559b1610c307d50f15f8d","trusted":false},"cell_type":"code","source":"model, cnn_cl = build_model1()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"82b16272ecfe86ee0dc3e5731d4e33aa016842c3","trusted":false},"cell_type":"code","source":"adam = ko.Nadam()\nmodel.compile(adam, loss=\"sparse_categorical_crossentropy\", metrics=[\"sparse_categorical_accuracy\",])\n\nfile_path = \"best_cnn_model.hdf5\"\ncheck_point = kc.ModelCheckpoint(file_path, monitor = \"val_sparse_categorical_accuracy\", verbose = 1, save_best_only = True, mode = \"max\")\nearly_stop = kc.EarlyStopping(monitor = \"val_sparse_categorical_accuracy\", mode = \"max\", patience=3)\nhistory = model.fit(X_train, y_train, batch_size=500, epochs=20, validation_split=0.1, callbacks = [check_point, early_stop])\n\nhistories.append(np.max(np.asarray(history.history['val_sparse_categorical_accuracy'])))\niterations.append(np.argmax(np.asarray(history.history['val_sparse_categorical_accuracy'])))\ndel model, history\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db9ac105bd787c06a1371eaf61936b1159c1645c"},"cell_type":"markdown","source":"## Attention RNN Model"},{"metadata":{"_uuid":"46d854b13ad591f5d349af3007a0b17559f72535","trusted":false},"cell_type":"code","source":"def build_model2():\n    voca_dim = embedding_matrix.shape[0]\n    time_steps = max_len\n    output_dim = led.classes_.shape[0]\n    rnn_dim = 100\n    mlp_dim = 50\n    item_embedding = embedding_matrix\n    rnn_depth=1\n    mlp_depth = 2\n    rnn_drop_out = 0.3\n    rnn_state_drop_out = 0.3\n    mlp_drop_out = 0.2\n    num_att_channel = 1\n    gpu=True\n    \n    return build_birnn_attention_model(\n        voca_dim, time_steps, output_dim, rnn_dim, mlp_dim, \n        item_embedding=item_embedding, rnn_depth=rnn_depth, mlp_depth=mlp_depth, num_att_channel=num_att_channel,\n        rnn_drop_out=rnn_drop_out, rnn_state_drop_out=rnn_state_drop_out,\n        gpu=gpu, return_customized_layers=True\n    )\n\nmodel_builders.append(build_model2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a264d510a763478c67b572c14366ae2144ce3f8","trusted":false},"cell_type":"code","source":"model, rnn_cl = build_model2()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d40857a43116f8587f28ba89baef17b519c58d45","trusted":false},"cell_type":"code","source":"adam = ko.Nadam(clipnorm=2.0)\nmodel.compile(adam, loss=\"sparse_categorical_crossentropy\", metrics=[\"sparse_categorical_accuracy\",])\n\nfile_path = \"best_birnn_attention_model.hdf5\"\ncheck_point = kc.ModelCheckpoint(file_path, monitor = \"val_sparse_categorical_accuracy\", verbose = 1, save_best_only = True, mode = \"max\")\nearly_stop = kc.EarlyStopping(monitor = \"val_sparse_categorical_accuracy\", mode = \"max\", patience=3)\nhistory = model.fit(X_train, y_train, batch_size=500, epochs=20, validation_split=0.1, callbacks = [check_point, early_stop])\n\nhistories.append(np.max(np.asarray(history.history['val_sparse_categorical_accuracy'])))\niterations.append(np.argmax(np.asarray(history.history['val_sparse_categorical_accuracy'])))\ndel model, history\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e53b77f6a8507bc64a2de060a1718366ee24200f"},"cell_type":"markdown","source":"## RNN-CNN Model"},{"metadata":{"_uuid":"1a3294e306d5ac35c0bf0fa039fc17099a83f367","trusted":false},"cell_type":"code","source":"def build_model3():\n    voca_dim = embedding_matrix.shape[0]\n    time_steps = max_len\n    output_dim = led.classes_.shape[0]\n    rnn_dim = 100\n    mlp_dim = 50\n    item_embedding = embedding_matrix\n    rnn_depth=1\n    mlp_depth = 2\n    num_filters = [128, 128, 128]\n    filter_sizes = [1, 3, 5]\n    cnn_drop_out = 0.2\n    rnn_drop_out = 0.3\n    rnn_state_drop_out = 0.3\n    mlp_drop_out = 0.2\n    padding = 'causal'\n    gpu=True\n    \n    return build_birnn_cnn_model(\n        voca_dim, time_steps, output_dim, rnn_dim, mlp_dim, num_filters, filter_sizes, \n        item_embedding=item_embedding, rnn_depth=rnn_depth, mlp_depth=mlp_depth,\n        rnn_drop_out=rnn_drop_out, rnn_state_drop_out=rnn_state_drop_out, cnn_drop_out=cnn_drop_out,\n        padding=padding,\n        gpu=gpu, return_customized_layers=True\n    )\n\nmodel_builders.append(build_model3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9e49dcf31d32f1b4717a3ccf665aacad1a1e165","trusted":false},"cell_type":"code","source":"model, rc_cl = build_model3()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"39f08fded4a3019dfd07504da932064425f8a16b","trusted":false},"cell_type":"code","source":"adam = ko.Nadam(clipnorm=2.0)\nmodel.compile(adam, loss=\"sparse_categorical_crossentropy\", metrics=[\"sparse_categorical_accuracy\",])\n\nfile_path = \"best_birnn_cnn_model.hdf5\"\ncheck_point = kc.ModelCheckpoint(file_path, monitor = \"val_sparse_categorical_accuracy\", verbose = 1, save_best_only = True, mode = \"max\")\nearly_stop = kc.EarlyStopping(monitor = \"val_sparse_categorical_accuracy\", mode = \"max\", patience=3)\nhistory = model.fit(X_train, y_train, batch_size=500, epochs=20, validation_split=0.1, callbacks = [check_point, early_stop])\n\nhistories.append(np.max(np.asarray(history.history['val_sparse_categorical_accuracy'])))\niterations.append(np.argmax(np.asarray(history.history['val_sparse_categorical_accuracy'])))\ndel model, history\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b4165b5741a971bb4a651a9d513ee2e1d3c19aa5"},"cell_type":"markdown","source":"## RNN-HierarchyCNN"},{"metadata":{"_uuid":"0ee00f133652fa813b55f4f223b87beddb588bcf","trusted":false},"cell_type":"code","source":"def build_model4():\n    voca_dim = embedding_matrix.shape[0]\n    time_steps = max_len\n    output_dim = led.classes_.shape[0]\n    rnn_dim = 100\n    mlp_dim = 50\n    item_embedding = embedding_matrix\n    rnn_depth=1\n    mlp_depth = 2\n    num_filters = [128, 256, 512]\n    filter_sizes = [1, 3, 5]\n    dilation_rates = [1, 2, 4]\n    strides=1\n    cnn_drop_out = 0.2\n    rnn_drop_out = 0.3\n    rnn_state_drop_out = 0.3\n    mlp_drop_out = 0.2\n    padding = 'causal'\n    gpu=True\n    \n    return build_birnn_hierarchy_cnn_model(\n        voca_dim, time_steps, output_dim, rnn_dim, mlp_dim, num_filters, filter_sizes, \n        dilation_rates=dilation_rates, strides=strides,\n        item_embedding=item_embedding, rnn_depth=rnn_depth, mlp_depth=mlp_depth,\n        rnn_drop_out=rnn_drop_out, rnn_state_drop_out=rnn_state_drop_out, cnn_drop_out=cnn_drop_out,\n        padding=padding,\n        gpu=gpu, return_customized_layers=True\n    )\n\nmodel_builders.append(build_model4)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d85a1e00009b635853e8928c83b3733a32f37d48","trusted":false},"cell_type":"code","source":"model, rhc_cl = build_model4()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"276d7133a37091457cdead0efb311ce01e20f689","trusted":false},"cell_type":"code","source":"adam = ko.Nadam(clipnorm=2.0)\nmodel.compile(adam, loss=\"sparse_categorical_crossentropy\", metrics=[\"sparse_categorical_accuracy\",])\n\nfile_path = \"best_birnn_hierarchy_cnn_model.hdf5\"\ncheck_point = kc.ModelCheckpoint(file_path, monitor = \"val_sparse_categorical_accuracy\", verbose = 1, save_best_only = True, mode = \"max\")\nearly_stop = kc.EarlyStopping(monitor = \"val_sparse_categorical_accuracy\", mode = \"max\", patience=3)\nhistory = model.fit(X_train, y_train, batch_size=500, epochs=20, validation_split=0.1, callbacks = [check_point, early_stop])\n\nhistories.append(np.max(np.asarray(history.history['val_sparse_categorical_accuracy'])))\niterations.append(np.argmax(np.asarray(history.history['val_sparse_categorical_accuracy'])))\ndel model, history\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db9a896a94311439b6eda66aa247e828fb57eca2"},"cell_type":"markdown","source":"# Make Prediction"},{"metadata":{"_uuid":"428880b633b909db6d5056d4751ce3dabc9fd1a5","trusted":false},"cell_type":"code","source":"histories = np.asarray(histories)\n\nmodel_paths = [\n    \"best_cnn_model.hdf5\",\n    \"best_birnn_attention_model.hdf5\",\n    \"best_birnn_cnn_model.hdf5\",\n    \"best_birnn_hierarchy_cnn_model.hdf5\"\n]\n\ncls =[\n    cnn_cl, rnn_cl, rc_cl, rhc_cl\n]\n\npred = list()\nfor idx in range(len(model_paths)):\n    model = models.load_model(model_paths[idx], cls[idx])\n    pred_tmp = model.predict(X_test, batch_size = 1024, verbose = 1)\n    pred.append(np.round(np.argmax(pred_tmp, axis=1)).astype(int))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ef083a0a29b808560344a0d52d1fb5a77a3a2da","trusted":false},"cell_type":"code","source":"def majority_vote(preds_data_point):\n    unique, counts = np.unique(preds_data_point, return_counts=True)\n    idx = np.argmax(counts)\n    return unique[idx]\n\npred = np.asarray(pred)\npredictions = list()\nfor i in range(pred.shape[1]):\n    predictions.append(majority_vote(pred[:, i]))\npredictions = np.asarray(predictions)\n\ntest_not_overlap_df = test_df[~overlap_boolean_mask_test]\ntest_not_overlap_df['Sentiment'] = predictions\n\nres_df = pd.concat([overlapped, test_not_overlap_df], sort=True)[sub_df.columns.values.tolist()]\n\nassert sub_df.shape[0] == res_df.shape[0]\nassert sub_df.shape[1] == res_df.shape[1]\n\nres_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}