{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true},"cell_type":"code","source":"\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nimport os; os.environ['OMP_NUM_THREADS'] = '1'\nimport scipy\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nfrom collections import defaultdict\nfrom multiprocessing import Pool\nfrom gensim.models import Word2Vec\n\nfrom sklearn.model_selection import StratifiedKFold\nimport gc\nfrom functools import wraps\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nimport pandas as pd\nimport numpy as np\nfrom multiprocessing import Pool\n\nfrom collections import OrderedDict\nfrom collections import defaultdict\nfrom string import digits\n\nimport nltk\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.corpus import wordnet, stopwords\nfrom nltk.tokenize import word_tokenize\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\n\nimport torch\nimport torch as t\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch.utils.checkpoint as checkpoint\nimport torch.utils.data\nimport torch.nn.functional as F\nfrom torch.nn import Parameter\nfrom functools import wraps\nfrom torch.autograd import Variable\n\nimport re\nimport random\nimport os\nimport time\nimport gc\nimport shutil\n\nfrom tqdm import tqdm\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\nfrom multiprocessing.pool import ThreadPool\nfrom nltk.corpus import wordnet, stopwords\nfrom sklearn import metrics\nfrom contextlib import contextmanager\nimport string\nimport tensorflow as tf\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom scipy.sparse import csr_matrix, hstack\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer as SIA\nfrom keras.layers import Input, Dropout, Dense, concatenate, GRU, Embedding, Flatten, Activation, BatchNormalization, PReLU\nfrom keras.initializers import he_uniform\nfrom keras.layers import Conv1D\nfrom keras.layers import GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.optimizers import Adam, SGD\nfrom keras.models import Model\nfrom tqdm import tqdm_notebook as tqdm\nimport time\nimport pandas as pd\nimport numpy as np\nfrom multiprocessing import Pool\nfrom nltk.stem import WordNetLemmatizer\nfrom itertools import combinations\nfrom sklearn.feature_extraction.text import CountVectorizer\nLOCAL = False\nfrom sklearn.linear_model import LogisticRegression, Ridge\nfrom keras import backend as K\nimport scipy\nfrom gensim import utils\n\nseed =1018\nif LOCAL: \n    FILE_DIR = './'\nelse: \n    FILE_DIR = '../input'\n@contextmanager\ndef timer(task_name=\"timer\"):\n    # a timer cm from https://www.kaggle.com/lopuhin/mercari-golf-0-3875-cv-in-75-loc-1900-s\n    print(\"----{} started\".format(task_name))\n    t0 = time.time()\n    yield\n    print(\"----{} done in {:.0f} seconds\".format(task_name, time.time() - t0))\n# Any results you write to the current directory are saved as output.\ntrain = pd.read_csv(f'{FILE_DIR}/train.csv')\ntest = pd.read_csv(f'{FILE_DIR}/test.csv')\n\ndef do_rnns(dfs):\n    if dfs[2]:\n        split = -1\n        train_df, test_df = dfs[0], dfs[1]\n\n        print(train_df.shape, test_df.shape)\n\n        test_idx = list(test_df.qid.values)\n\n        test_df.drop('qid', axis=1, inplace=True)\n\n        training_labels = train_df['target'].values\n        if split>0:\n            testing_labels = test_df['target'].values\n\n        # PREPROCESS-----#\n        def get_special_feats(df):\n            df['question_text'] = df['question_text'].apply(lambda x:str(x))\n            df['total_length'] = df['question_text'].apply(len)\n            df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))\n            df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)\n            df['num_words'] = df.question_text.str.count('\\S+')\n            df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))\n            df['words_vs_unique'] = df['num_unique_words'] / df['num_words']\n            features = df[['caps_vs_length', 'words_vs_unique']].fillna(0).values\n            gc.collect()\n            return features\n        print('START SPECIAL FEATS')\n        train_features = get_special_feats(train_df)\n        test_features = get_special_feats(test_df)\n        print('END SPECIAL FEATS')\n        ss = StandardScaler()\n        ss.fit(np.vstack((train_features, test_features)))\n        train_features = ss.transform(train_features)\n        test_features = ss.transform(test_features)\n        print('SCALE SPECIAL FEATS')\n        punctuations = [\n            ',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#',\n            '*', '+', '\\\\', '•',  '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™',\n            '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢',\n            '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', \n            '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅',\n            '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬',\n            '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', '\\xa0', '高', '端', '大', '气', '上', '档', '次', '_', '½', 'π', '#', \n        '小', '鹿', '乱', '撞', '成', '语', 'ë', 'à', 'ç', '@', 'ü', 'č', 'ć', 'ž', 'đ', '°', 'द', 'े', 'श', '्', 'र', 'ो', 'ह', 'ि', 'प', 'स', 'थ', 'त', 'न', 'व', 'ा', 'ल', 'ं', '林', '彪', '€', '\\u200b', '˚', 'ö', '~', '—', '越', '人', 'च', 'म', 'क', \n        'ु', 'य', 'ी', 'ê', 'ă', 'ễ', '∞', '抗', '日', '神', '剧', '，', '\\uf02d', '–', 'ご', 'め', 'な', 'さ', 'い', 'す', 'み', 'ま', 'せ', 'ん', 'ó', 'è', '£', '¡', 'ś', '≤', '¿', 'λ', '魔', '法', '师', '）', 'ğ', 'ñ', 'ř', '그', '자', '식', '멀', \n        '쩡', '다', '인', '공', '호', '흡', '데', '혀', '밀', '어', '넣', '는', '거', '보', '니', 'ǒ', 'ú', '️', 'ش', 'ه', 'ا', 'د', 'ة', 'ل', 'ت', 'َ', 'ع', 'م', 'ّ', 'ق', 'ِ', 'ف', 'ي', 'ب', 'ح', 'ْ', 'ث', '³', '饭', '可', '以', '吃', '话', '不', '讲', \n        '∈', 'ℝ', '爾', '汝', '文', '言', '∀', '禮', 'इ', 'ब', 'छ', 'ड', '़', 'ʒ', '有', '「', '寧', '錯', '殺', '一', '千', '絕', '放', '過', '」', '之', '勢', '㏒', '㏑', 'ू', 'â', 'ω', 'ą', 'ō', '精', '杯', 'í', '生', '懸', '命', 'ਨ', 'ਾ', 'ਮ', 'ੁ', \n        '₁', '₂', 'ϵ', 'ä', 'к', 'ɾ', '\\ufeff', 'ã', '©', '\\x9d', 'ū', '™', '＝', 'ù', 'ɪ', 'ŋ', 'خ', 'ر', 'س', 'ن', 'ḵ', 'ā']\n\n        def clean_text( text):\n            text = str(text)\n            for p in punctuations:\n                if p in text:\n                    text = text.replace(p, f' {p} ')\n            return(text)  \n        print('CLEAN TEXT')\n        train_df['question_text'] = train_df['question_text'].apply(lambda x: clean_text(x))\n        test_df['question_text'] = test_df['question_text'].apply(lambda x: clean_text(x))\n        print('FINISH CLEANING TEXT')\n        contractions_dict = { \n            \"ain't\": \"am not\",\n            \"aren't\": \"are not\",\n            \"can't\": \"cannot\",\n            \"can't've\": \"cannot have\",\n            \"'cause\": \"because\",\n            \"could've\": \"could have\",\n            \"couldn't\": \"could not\",\n            \"couldn't've\": \"could not have\",\n            \"didn't\": \"did not\",\n            \"doesn't\": \"does not\",\n            \"don't\": \"do not\",\n            \"hadn't\": \"had not\",\n            \"hadn't've\": \"had not have\",\n            \"hasn't\": \"has not\",\n            \"haven't\": \"have not\",\n            \"he'd\": \"he had\",\n            \"he'd've\": \"he would have\",\n            \"he'll\": \"he shall\",\n            \"he'll've\": \"he shall have\",\n            \"he's\": \"he has\",\n            \"how'd\": \"how did\",\n            \"how'd'y\": \"how do you\",\n            \"how'll\": \"how will\",\n            \"how's\": \"how has\",\n            \"I'd\": \"I had\",\n            \"I'd've\": \"I would have\",\n            \"I'll\": \"I will\",\n            \"I'll've\": \"I will have\",\n            \"I'm\": \"I am\",\n            \"I've\": \"I have\",\n            \"isn't\": \"is not\",\n            \"it'd\": \"it would\",\n            \"it'd've\": \"it would have\",\n            \"it'll\": \"it will\",\n            \"it'll've\": \"it will have\",\n            \"it's\": \"it is\",\n            \"let's\": \"let us\",\n            \"ma'am\": \"madam\",\n            \"mayn't\": \"may not\",\n            \"might've\": \"might have\",\n            \"mightn't\": \"might not\",\n            \"mightn't've\": \"might not have\",\n            \"must've\": \"must have\",\n            \"mustn't\": \"must not\",\n            \"mustn't've\": \"must not have\",\n            \"needn't\": \"need not\",\n            \"needn't've\": \"need not have\",\n            \"o'clock\": \"of the clock\",\n            \"oughtn't\": \"ought not\",\n            \"oughtn't've\": \"ought not have\",\n            \"shan't\": \"shall not\",\n            \"sha'n't\": \"shall not\",\n            \"shan't've\": \"shall not have\",\n            \"she'd\": \"she would\",\n            \"she'd've\": \"she would have\",\n            \"she'll\": \"she will\",\n            \"she'll've\": \"she will have\",\n            \"she's\": \"she is\",\n            \"should've\": \"should have\",\n            \"shouldn't\": \"should not\",\n            \"shouldn't've\": \"should not have\",\n            \"so've\": \"so have\",\n            \"so's\": \"so is\",\n            \"that'd\": \"that had\",\n            \"that'd've\": \"that would have\",\n            \"that's\": \"that is\",\n            \"there'd\": \"there would\",\n            \"there'd've\": \"there would have\",\n            \"there's\": \"there is\",\n            \"they'd\": \"they would\",\n            \"they'd've\": \"they would have\",\n            \"they'll\": \"they will\",\n            \"they'll've\": \"they will have\",\n            \"they're\": \"they are\",\n            \"they've\": \"they have\",\n            \"to've\": \"to have\",\n            \"wasn't\": \"was not\",\n            \"we'd\": \"we would\",\n            \"we'd've\": \"we would have\",\n            \"we'll\": \"we will\",\n            \"we'll've\": \"we will have\",\n            \"we're\": \"we are\",\n            \"we've\": \"we have\",\n            \"weren't\": \"were not\",\n            \"what'll\": \"what will\",\n            \"what'll've\": \"what will have\",\n            \"what're\": \"what are\",\n            \"what's\": \"what is\",\n            \"what've\": \"what have\",\n            \"when's\": \"when is\",\n            \"when've\": \"when have\",\n            \"where'd\": \"where did\",\n            \"where's\": \"where is\",\n            \"where've\": \"where have\",\n            \"who'll\": \"who will\",\n            \"who'll've\": \"who will have\",\n            \"who's\": \"who is\",\n            \"who've\": \"who have\",\n            \"why's\": \"why is\",\n            \"why've\": \"why have\",\n            \"will've\": \"will have\",\n            \"won't\": \"will not\",\n            \"won't've\": \"will not have\",\n            \"would've\": \"would have\",\n            \"wouldn't\": \"would not\",\n            \"wouldn't've\": \"would not have\",\n            \"y'all\": \"you all\",\n            \"y'all'd\": \"you all would\",\n            \"y'all'd've\": \"you all would have\",\n            \"y'all're\": \"you all are\",\n            \"y'all've\": \"you all have\",\n            \"you'd\": \"you would\",\n            \"you'd've\": \"you would have\",\n            \"you'll\": \" you will\",\n            \"you'll've\": \"you will have\",\n            \"you're\": \"you are\",\n            \"you've\": \"you have\",\n            '隶书': 'Lishu', 'तपस्': 'Tapse', 'ให้': 'give', '宗教官': 'Religious officer', '没有神一样的对手': 'No god-like opponent', 'יהודיות': 'Jewish women', '我明明是中国人': 'I am obviously Chinese.', 'お早う': 'Good morning', 'मेरे': 'my', 'পরীক্ষা': 'Test', 'נעכטן': 'Yesterday', '中华民国': 'Republic of China', 'तकनीकी': 'Technical', 'यहाँ': 'here', '奇兵隊': 'Qibing', 'かﾟ': 'Or', '知识分子': 'Intellectuals', 'ごめなさい': \"I'm sorry\", '하기를': 'To', 'गाढवाचे': 'Donkey', 'इबादत': 'Worship', '千金': 'Thousand gold', 'ельзи': 'elzhi', '低端人口': 'Low-end population', '맛저': 'Mazur', 'दानि': 'Danias', 'юродство': 'foolishness', 'τον': 'him', '素质': 'Quality', '王晓菲': 'Wang Xiaofei', 'माहेर': 'Maher', 'لمصارى': 'For my advice', '送客！': 'see a visitor out!', '然而': 'however', 'जिग्यासा': 'Jigyaasa', '都市': 'city', 'ஜோடி': 'Pair', 'لاني': 'See', 'пельмени': 'dumplings', '请不要误会': \"Please don't misunderstand\", 'люис': 'Lewis', '不送！': 'Do not send!', 'के': 'Of', 'मस्का': 'Mascara', 'вoyѕ': 'voyce', 'बुन्ना': 'Bunna', '战略支援部队': 'Strategic support force', '平埔族': 'Pingpu', 'فهمت؟': 'I see?', 'कयामत': 'Apocalypse', 'ức': 'memory', 'ᗰeᑎ': 'ᗰe ᑎ', 'सेकना': 'Sexting', 'धकेलना': 'Shove', 'পারি': 'We can', 'مقام': 'Official', 'बेसति': 'Baseless', '歪果仁研究协会': 'Hazelnut Research Association', '短信': 'SMS', 'всегда': 'is always', '修身': 'Slim fit', 'إنَّ': 'that', '不是民国': 'Not the Republic of China', '写的好': 'Written well', 'õhtupoolik': 'afternoon', 'तालीन': 'Training', 'और': 'And', 'щит': 'shield', 'ᗩtteᑎtiᐯe': 'ᗩtte ᑎ ti ᐯ e', 'اسکوپیه': 'Script', 'çomar': 'fido', '和製英語': 'Japanglish', '吊着命': 'Hanging', 'много': 'much', 'समुराय': 'Samurai', 'भटकना': 'Wander', 'مقاومة': 'resistance', '싱관없어': 'I have no one.', '修身養性': 'Self-cultivation', 'मटका': 'pot', 'θιοψʼ': 'thiós', 'ㅎㅎㅎㅎ': 'Hehe', 'تساوى': 'Equal', 'बाट': 'From', '不地道啊。': 'Not authentic.', 'контакт': 'contact', '이런것도': 'This', 'तै': 'The', 'मेल': 'similarity', 'álvarez': 'Alvarez', 'नुक्स': 'Damage', '口訣': 'Mouth', 'масло': 'butter', 'परम्परा': 'Tradition', '学会': 'learn', 'کردن': 'Make up', 'öd': 'bile', 'टशन': 'Tashan', 'つらやましす': 'I am cheating', 'чего': 'what', '为什么总有人能在shoplex里免费获取iphone等重金商品？': 'Why do people always get free access to iphone and other heavy commodities in shoplex?', 'श्री': 'Mr', 'प्रेषक': 'Sender', 'خواندن': 'Read', 'बदलेंगे': 'Will change', 'बीएड': 'Breed', 'अदा': 'Paid', 'फैलाना': 'spread', 'ബുദ്ധിമാനായ': 'Intelligent', '谢谢六佬': 'Thank you, Liu Wei', 'উপস্থাপন': 'Present', 'بكلها': 'All of them', 'जाए': 'Go', '无可挑剔': 'Impeccable', 'आना': 'come', '太阴': 'lunar', 'القط': 'The cat', '있네': 'It is.', 'कर्म': 'Karma', 'आड़': 'Shroud', 'įit': 'IIT', 'जशने': 'Jashnay', 'से': 'From', 'åkesson': 'Åkesson', '乳名': 'Milk name', '我看起来像韩国人吗': 'Do I look like a Korean?', 'тнan': 'tn', 'العظام': 'Bones', '暹罗': 'Siam', '小肥羊鍋店': 'Little fat sheep pot shop', 'করেন': 'Do it', 'हरामी': 'bastard', 'डाले': 'Cast', 'استراحة': 'Break', '射道': 'Shooting', 'επιστήμη': 'science', 'χράομαι': \"I'm glad\", '脚踏车': 'bicycle', 'हिलना': 'To move', 'đa': 'multi', '標點符號': 'Punctuation', 'मैं': 'I', '不能化生水谷精微': \"Can't transform the water valley\", 'निवाला': 'Boar', 'दर्शनाभिलाषी': 'Spectacular', 'বাছাই': 'Picking', 'どくぜんてき': 'Dokedari', 'पीवें।': 'Pieve.', 'ʻoumuamua': 'Priority', 'में': 'In', 'चलाऊ': 'Run', 'निपटाना': 'To settle', 'ごはん': 'rice', 'चार्ज': 'Charge', 'शिथिल': 'Loosely', 'ястребиная': 'yastrebina', 'ложись': 'lie down', '李银河': 'Li Yinhe', 'へも': 'Also', 'हुलिया': 'Hulia', 'ऊब': 'Bored', 'छाछ': 'Buttermilk', 'عن': 'About', 'नहीं': 'No', 'जीव': 'creatures', 'जेहाद': 'Jehad', 'νερό': 'water', '열여섯': 'sixteen', 'मार': 'Kill', '早就': 'Long ago', '《齐天大圣》for': '\"Qi Tian Da Sheng\" for', 'الجنس': 'Sex', 'مع': 'With', 'रंगरलियाँ': 'Color palettes', 'जोल': 'Jol', 'बुद्धी': 'Intelligence', '罗思成': 'Luo Sicheng', '独善的': 'Self-righteousness', 'धर्मः': 'Religion', '中国大陆的同胞们': 'Chinese compatriots', '愛してない': 'I do not love you', '日本語が上手ですね': 'Japanese is good', 'ओठ': 'Lips', '如果你希望表達你的觀點': 'If you want to express your opinion', 'देशवाशी': 'Countryman', 'καί': 'and', '阮铭': 'Yu Ming', '跑步跑': 'Running', 'हो': 'Ho', '「褒められる」': '\"Praised\"', 'నా': 'My', '\\x10new': '?new', 'ゆいがどくそん': 'A graduate student', '白语': 'White language', 'वह': 'She', '白い巨塔': 'White big tower', '로리이고': 'Lori.', 'परि': 'Circumcision', 'æj': 'Aw', 'انضيع': 'Lost', '平民苗字必称義務令': 'Civilian Miaozi must be called an obligation', 'ਸਿੰਘ': 'Singh', 'уже': 'already', 'đầu': 'head', '雨热同期': 'Rain and heat', 'घुलना': 'Dissolve', 'мис': 'Miss', 'て下さいませんか': 'Could it be?', '猫头鹰': 'owl', 'चढ़': 'Climbing', '漢音': 'Hanyin', 'यही': 'This only', '只有猪一样的队友': 'Only pig-like teammates', 'übersoldaten': 'about soldiers', 'αθ': 'Ath', 'ουδεις': 'no', '党国合一': 'Party and country', 'रेड': 'Red', 'ढोलना': 'Drift', 'शाक': 'Shake', '같이': 'together', '攻克': 'capture', 'özalan': 'exclusive', 'काम': 'work', 'चाहना': 'Wish', '坚持一个中国原则': 'Adhere to the one-China principle', '배우고': 'Learn', '柴棍': 'Firewood stick', 'उसे': 'To him', 'на': 'on', '無手': 'No hands', 'čvrsnica': 'Čvrsnica', 'ज़ार': 'Tsar', 'ˆo': 'O', '後宮甄嬛傳': 'Harem of the harem', '意音文字': 'Sound character', 'बहारा': 'Bahara', 'てみる': 'Try', '老铁': 'Old iron', '野比のび太': 'Nobita Nobita', 'याद': 'remember', 'پیشی': 'Surpass', 'توقعك': 'Your expectation', 'はたち': 'Slender', 'فزت': 'I won', '伊藤': 'Ito', 'कलेजे': 'Liver', 'αγεν': 'ruin', 'ìmprovement': 'Improvement', 'では': 'Then.', 'பார்ப்பது': 'Viewing', '只留清气满乾坤': 'Only stay clear and full of energy', '이정도쯤': 'About this', 'すてん': 'Sponge', 'ما': 'What', '海南人の日本': \"Hainan people's Japan\", '小鹿乱撞': 'very excited', 'ῥωμαῖοι': 'ῥomaşii', 'वारी': 'Vary', 'भोज': 'Feast', '陈庚': 'Chen Geng', 'कट्टरवादि': 'Fanatic', '凌霸': 'Lingba', 'पार': 'The cross', 'कुचलना': 'To crush', 'रहा': 'Stayed', 'हम': 'We', 'бойся': 'be afraid', '大刀': 'Large sword', 'ন্ন': 'Done', '汽车': 'car', 'কে': 'Who', 'الکعبه': 'Alkaebe', '网络安全法': 'Cybersecurity law', 'नेपाली': 'Nepali', 'পদের': 'Position', '\\x92t': '??t', 'रास': 'Ras', 'لكل': 'for every', 'शूद्राणां': 'Shudran', '此问题只是针对外国友人的一次统计': 'This question is only a statistic for foreign friends.', 'ឃើញឯកសារអំពីប្រវត្តិស្ត្រនៃប្រាសាទអង្គរវត្ត': 'See the history of Angkor Wat', 'спасибо': 'thank', 'رب': 'God', 'वजूद': 'Non existent', 'पकड़': 'Hold', 'बरहा': 'Baraha', '雾草': 'Fog grass', 'мощный': 'powerful', 'বৃদ্ধাশ্রম': 'Old age', 'आई': 'Mother', 'खड़ी': 'Steep', '\\x1aaùõ\\x8d': '? aùõ ??', 'āto': 'standard', '在冰箱裡': 'in the refrigerator', 'đời': 'life', 'लड़का': 'The boy', 'τὸ': 'the', 'مدرسان': 'Instructors', 'பறை': 'Drum', 'índia': 'India', 'दिया': 'Gave', '小篆': 'Xiao Yan', '唐樂': 'Tang Le', '米国': 'USA', '文言文': 'Classical Chinese', 'उवाच': 'Uwach', 'هدف': 'Target', 'घेरा': 'Cordon', 'झूम': 'Zoom', 'εσσα': 'all the same', '和製漢語': 'And making Chinese', 'ऐलोपैथिक': 'Allopathic', 'создала': 'has created', 'إدمان': 'addiction', '游泳游': 'Swimming tour', '狮子头': 'Lion head', 'दिये': 'Given', 'पास': 'near', 'εντjα': 'in', 'まする': 'To worship', 'हाल': 'condition', '짱깨': 'Chin', 'জল': 'Water', 'चीज': 'thing', 'îmwe': 'one', '胡江南': 'Hu Jiangnan', 'तमाम': 'All', 'कट्टर': 'Hardcore', '魔法师': 'Magician', 'బుూ\\u200c': 'Buu', 'चनचल': 'Chanchal', 'सीधा': 'Straightforward', '不要人夸颜色好': \"Don't exaggerate the color\", 'हिमालय': 'Himalaya', 'सिंह': 'Lion', 'خراميشو': 'Wastewater', 'мoѕт': 'the moment', 'কোথায়': 'Where?', 'نصف': 'Half', 'رائع': 'Wonderful', 'добрый': 'kind', 'ज़र्रा': 'Zarra', 'يعقوب': 'Yacoub', 'सम्झौता': 'Agreement', 'ān': 'yes', '했다': 'did', 'ἀριστοκράτης': 'aristocrat', 'çan': 'Bell', '太阳': 'sun', 'सर': 'head', 'गुरु': 'Master', '嘉定': 'Jiading', '故乡': 'home', '安倍晋三': 'Shinzo Abe', 'मच्छर': 'Mosquito', 'सभी': 'All', '成語': 'idiom', '唯我独尊': 'Only me', 'นะค่ะ': 'Yes', 'นั่นคุณกำลังทำอะไร': 'What are you doing', 'ın': 's', 'الشامبيونزليج': 'Shamballons', '抗日神剧': 'Anti-Japanese drama', '小妹': 'Little sister', 'çok': 'very', 'लोड': 'Load', 'साहित्यिक': 'Literary', 'लाल।': 'Red.', 'فى': 'in a', '絕不放過一人」之勢': 'Never let go of one person', '国家主席': 'National president', '异议': 'objection', 'अनुस्वार': 'Anuswasar', 'តើបងប្អូនមានមធ្យបាយអ្វីខ្លះដើម្បីរក': 'What are some ways to find out', '允≱ၑご搢': '≱ ≱ ≱ 搢', '黒髪': 'Black hair', '自戳双目': 'Self-poke binocular', 'βροχέως': 'rainy', 'ని': 'The', '一带一路': 'Belt and Road', 'śliwińska': 'Śliwińska', 'κατὰ': 'against', '打古人名': 'Hit the ancient name', 'குறை': 'Low', 'üjin': 'Uji', 'öppning': 'opening', 'अन्जल': 'Anjal', '이와': 'And', 'आविष्कार': 'Invention', 'غنج': 'Gnostic', 'рассвет': 'dawn', 'होना': 'Happen', 'বাড়ি': 'Home', '入定': 'Enter', 'भेड़': 'The sheep', 'देगा': 'Will give', 'ब्रह्मा': 'Brahma', 'बीन': 'Bean', 'χράω': \"I'm glad\", 'तीन': 'three', 'ضرار': 'Drar', 'विराजमान': 'Seated', 'सैलाब': 'Salab', 'συνηθείας': 'communication', '구경하고': 'To see', 'चुल्ल': 'Chool', '红宝书': 'Red book', '羡慕和嫉妒是不一样的。': 'Envy and jealousy are not the same.', 'मेरा': 'my', 'कलटि': 'Katiti', 'हिमाती': 'Himalayan', 'ಸಾಧು': 'Sadhu', 'عطيتها': 'Her gift', 'छान': 'Nice', 'เกาหลี': 'Korea', 'íntruments': 'instruments', 'يتحمل': 'Bear', 'रुस्तम': 'Rustam', 'बरात': 'Baraat', 'रंग': 'colour', '나이': 'age', 'פרפר': 'butterfly', '老乡': 'Hometown', '謝謝': 'Thank you', 'í‰lbƒ': '‰ in lbƒ', 'हरजाई': 'Harjai', 'পেতে': 'Get to', '「勉強': '\"study', 'रड़कना': 'To cry', '清真诊所': 'Halal clinic', 'أفضل': 'Best', 'استطيع': 'I can', 'नाकतोडा': 'Nag', 'बड़ि': 'Elder', 'वैद्य': 'Vaidya', 'أنَّ': 'that', 'いたロリィis': 'There was Lory', 'ìn': 'print', '本人堅持一個中國的原則': 'I adhere to the one-China principle', 'океан': 'ocean', 'बाद': 'after', '禮儀': 'etiquette', 'सिषेविरे': 'Sisavir', 'अमृत': 'Honeydew', 'بدو': 'run', 'ὅς': 'that is', 'छोटा': 'small', 'स्वभावप्रभवैर्गुणै': 'Nature is bad', 'ענג': 'Tight', 'שלמה': 'Complete', 'डोरे': 'Dore', '我害怕得毛骨悚然': 'I am afraid of creeps.', 'झोलि': 'Jolly', 'единадесет': 'eleven', 'என்ன': 'What', 'आयुरवेद': 'Ayurveda', '堵天下悠悠之口': 'Blocking the mouth of the world', 'الجمعي': 'Collective', 'ángel': 'Angel', 'रोना': 'crying', 'हमराज़': 'Humraj', 'चिलमन': 'Drape', 'औषध': 'Medicine', 'निया': 'Nia', 'תעליעוכופור': 'Go up and go', '中国队大胜': 'Chinese team wins', 'δ##': 'd ##', 'चला': 'walked', 'पर': 'On', 'ἔργον': 'work', 'रंक': 'Rank', 'सिक्ताओ': 'Sixtao', '陳太宗': 'Chen Taizong', 'لومڤور': 'لوموور', 'விளையாட்டு': 'Sports', '的？': 'of?', '안녕하세요': 'Hi', 'נבאים': 'Prophets', 'ন্য': 'N.', 'şimdi': 'now', 'भरना': 'Fill', 'धरी': 'Clog', '漢字': 'Chinese character', 'इसि': 'This', 'आवर': 'Hour', 'काटना': 'cutting', '僕だけがいない街': 'City where I alone does not exist', 'जूठा': 'Lier', 'çekerdik': 'We Are', 'čaj': 'tea', 'నందొ': 'Nando', '核对无误': 'Check is correct', '無限': 'unlimited', 'समेटना': 'Crimp', 'żurek': 'sour soup', 'আছে': 'There are', 'مرتك': 'Committed', 'आपाद': 'Exorcism', 'चोरी': 'theft', 'బవిష్కతి': 'Baviskati', 'निकालना': 'removal', 'सीमा': 'Limit', '配信開始日': 'Distribution start date', '宝血': 'Blood', 'ग्यारह': 'Eleven', 'गरीब': 'poor', '있고': 'There', 'île': 'island', 'स्मि': 'Smile', 'енгел': 'engel', 'вы': 'you', 'οπότε': 'so', 'सूनापन': 'Desolation', 'áraszt': 'exudes', 'मारि': 'Marie', '서로를': 'To each other', 'घोपना': 'To announce', 'फितूर': 'Fitur', 'あからさまに機嫌悪いオーラ出してんのに話しかけてくるし': 'I will speak to the out-putting aura outright bad', '封柏荣': 'Feng Bairong', '「褒められたものではない」': '\"It is not a praise\"', '傳統文化已經失存了': 'Traditional culture has lost', '水木清华': 'Shuimu Tsinghua', 'पन्नी': 'Foil', 'ਰਾਜਬੀਰ': 'Rajbir', '煮立て': 'Boiling', '外国的月亮比较远': 'Foreign moon is far away', 'شغف': 'passion', 'గురించి': 'About', 'övp': 'ÖVP', '实名买票制': 'Real name ticket system', 'बिखरना': 'To scatter', 'التَعَمّقِ': 'Persecution', 'जीवें': 'The living', 'गई': 'Has been', '部頭合約': 'Head contract', 'دقیانوس': 'Precise', 'फंकी': 'Funky', '粵拼': 'Cantonese fight', 'ĉiohelpanto': 'all supporter', 'मारना': 'killing', 'मानजा': 'Manja', 'अष्टांगिक': 'Octagonal', 'への': 'To', 'रहना': 'live', 'खाना': 'food', 'ಕೋಕಿಲ': 'Kokila', 'చిన్న': 'Small', '金髪': 'Blond hair', '籍贯': 'Birthplace', 'उखाड़ना': 'Extirpate', 'α2': \"A'2\", 'להשתלט': 'take over control', '露西亚': 'Lucia', '大有「寧可錯殺一千': 'There is a lot of \"I would rather kill a thousand', 'संस्कृत': 'Sanskrit', 'христо': 'Christo', 'लगाना': 'to set', '선배님': 'Seniors', 'उड़ीसा': 'Orissa', 'ஆய்த': 'Ayta', 'तेरे': 'Your', 'आकांक्षी': 'Aspiring', 'बाज़ार': 'The market', 'हर्ज़': 'Herz', 'だします': 'Will do.', 'चक्कर': 'affair', '없어': 'no', 'चम्पक': 'Champak', 'ताल': 'rythm', 'āgamas': 'hobby', 'योद्धा': 'Warrior', 'αλυπος': 'chain', 'बेड़ा': 'Fleet', 'बात': 'talk', '做莫你酱紫的？': 'Do you make your sauce purple?', '見逃し': 'Miss', '鰹節': 'Bonito', 'तथैव': 'Fact', 'आध्यात्मिकता': 'Spirituality', '正弦': 'Sine', 'लिया': 'took', 'îndrăgire': 'love', '我願意傾聽': 'I am willing to listen', '家乡': 'hometown', '大败': 'Big defeat', 'मए': 'Ma', 'జ్ఞ\\u200cా': 'Sign', 'օօ': 'oo', 'खेमे': 'Camps', 'الشوكولاه': 'Chocolate', 'полностью': 'completely', '商品発売日': 'Product Release Date', '金継ぎ': 'Gold piecing', '烤全羊是多少人民币呢？': 'How much is the renminbi roasted in the whole sheep?', 'γолемата': 'golem', '孫瀛枚': 'Grandson', 'özlüyorum': 'I am missing', 'خلبوص': 'Libs', 'アンダージー': 'Undergar', '蝴蝶': 'butterfly', 'প্রশ্ন': 'The question', 'जरूरी': 'Necessary', 'बूरा': 'Bura', '有毒': 'poisonous', 'सान्त्वना': 'Comfort', '눈치': 'tact', '\\ufeffwhat': 'what', 'çeşme': 'fountain', 'ごっめんなさい': 'Please, sorry.', 'விளக்கம்': 'Description', '罗西亚': 'Rosia', 'गोटा': 'Knot', 'लेखावलिपुस्तके': 'Accounting Tips', 'सम्भोग': 'Sexual intercourse', '慢走': 'Slow walking', 'तुम': 'you', 'लीला': 'Leela', 'ฉันจะทำให้ดีที่สุด': 'I will do my best', 'चम्पु': 'Shampoo', '視聽華語': 'Audiovisual Chinese', '比比皆是': 'Abound', 'धावा': 'Run', '娘娘': 'Goddess', 'पहाड़': 'the mountain', 'राजा': 'King', '茹西亚': 'Rusia', 'ब्राह्मणक्षत्रियविशां': 'Brahmin Constellations', '수업하니까': \"I'm in class.\", 'చెల్లెలు': 'Sister', 'साजे': 'Made', '支那人': 'Zhina', '团表': 'Group table', '讲得': 'Speak', 'へからねでて': 'From to', 'über': 'over', 'քʀɛʋɛռt': 'kʀɛʋɛr t', 'นเรศวร': 'Naresuan', '方言': 'dialect', 'पना': 'Find out', '怕樱': 'Afraid of cherry', 'घुल': 'Dissolve', '米帝': 'Midi', 'طيب': 'Ok', 'प्रेम': 'Love', 'पढ़ाई': 'studies', '他穿褲子': 'He wears pants', 'मेरी': 'Mine', 'উত্তর': 'Reply', 'स्थिति': 'Event', '\\x1b\\xadü': '?\\xadü', 'तैश': 'Tachsh', '写得好': 'Well written', 'मॉलूम': 'Known', '创意梦工坊': 'Creative Dream Workshop', 'आपकी': 'your', 'मिलना': 'Get', '배웠어요': 'I learned it', '許自東': 'Xu Zidong', 'जाऊँ': 'Go', 'अहिंसा': 'Nonviolence', 'джоу': 'Joe', '金繕い': 'Gold patting', '好心没好报': 'No return on a good deed', 'çevapsiz': 'unanswered', 'मिल': 'The mill', '日本語': 'Japanese', 'اخذ': 'Get', '水军': 'Water army', 'बिना': 'without', 'बनाना': 'Make', 'التوبه': 'Repentance', '一个灵运行在我的家': 'a spirit running in my home', '한국어를': 'Korean', 'कि': 'That', 'εισα': 'import', 'लगना': 'feel', 'गपोड़िया': 'Gopodiya', 'ārūpyadhātu': 'exhyadadate', 'आए': 'Returns', '吃好吃金': 'Eat delicious gold', 'นั่น': 'that', 'बढ़ाना': 'raise up', '보니': 'Bonnie', '爱着': 'Love', '선배': 'Elder', '刷屏': 'Brush screen', '人性化': 'Humanize', 'خرسانة': 'Concrete', '麻辣乾鍋': 'Spicy dry pot', '\\x10œø': '? œø', 'सतरंगी': 'Satarangi', '磨合': 'Run-in', 'को': 'To', 'شهادة': 'Degree', 'একটি': 'A', 'へと': 'To', 'يلي': 'Following', '光本': 'Light source', '褒め殺し': 'Praise kill', 'आँचल': 'Anchal', 'এর': 'Of it', 'पिटा': 'Pita', 'بديش': 'Badish', 'गंगु': 'Gangu', '可是': 'but', 'की': 'Of', '谢谢。台灣同胞': 'Thank you. Taiwan compatriots', 'दम': 'power', 'मैकशि': 'Macshi', 'రాజ': 'King', '玉蘭花': 'Magnolia', '江ノ島盾子': 'Enoshima Junko', 'ѕтυpιd': 'ѕtυpιd', 'जिसका': 'Whose', 'எழுத்து': 'Letter', '甲骨文': 'Oracle', 'चूरमा': 'Churma', 'चूलें': 'Chulen', 'प्रविभक्तानि': 'Interpretation', 'いる': 'To have', 'مقال': 'article', 'पाय': 'Feet', 'अतीत': 'Past', 'ármin': 'Armin', '東夷': 'Dongyi', 'आदमकद': 'Life expectancy', 'किये': 'Done', 'पतवार': 'Helm', '楽曲派アイドル': 'Song musical idols', 'डगमगाना': 'Waver', '북한': 'North Korea', '禮記': 'Book of Rites', '西魏': 'Xi Wei', '過労死': 'Death from overwork', 'बेबुनियाद': 'Unbounded', '仙人跳': 'Fairy jump', '港女': 'Hong Kong girl', '虞朝': 'Sui Dynasty', 'µ0': 'μ0', '字母词': 'Letter word', 'अर्धांगिनि': 'Arghangini', '真功夫': 'real Kong Fu', '飯糰': 'Rice ball', 'علم': 'Science', 'गुड़': 'Jaggery', 'гречку': 'buckwheat', '我方记账数字与贵方一致': 'Our billing figures are consistent with yours', 'आने': 'To arrive', 'कब': 'When', '宋楚瑜': 'James Soong', 'διητ': 'filter', 'राई': 'Rye', 'விரதம்': 'Fasting', 'बदल': 'change', '成语': 'idiom', 'ĺj': 'junk', 'какая': 'which one', '文翰': 'Wen Han', 'کـ': 'K', 'ʀɛċօʍʍɛռɖɛɖ': 'ʀɛċ oʍʍɛрɖɛɖ', 'दिलों': 'Hearts', '星期七': 'Sunday', '傳送': 'Transfer', '陳云根': 'Chen Yunen', 'ʿalaʾ': 'Allah', 'गुल': 'Gul', 'ख़बर': 'The news', 'मन्च': 'Manch', '国家知识产权局': 'State Intellectual Property Office', '행복하게': 'happily', 'बबीता': 'Babita', 'юродивый': 'holy fool', 'सफाई': 'clean', 'вода': 'water', 'लाठि': 'End', 'γὰρ': 'γσρ', '在日朝鮮人／韓国人': 'Koreans/Koreans in Japan', '学过': 'Learned', 'जमाना': 'Solidification', 'इकोनॉमिक्स': 'Economics', 'क्या': 'what', 'ドア': 'door', '中国的存在本身就是原罪': 'The existence of China itself is the original sin', 'मौसमि': 'Season', 'ठाठ': 'Chic', 'ἡμετέραν': 'another', '火了mean': 'Fired mean', 'せえの': 'Set out', 'ஆயுத': 'Arms', 'कंपनी': 'company', 'अहम्': 'Ego', 'भर': 'Filled', 'फिरना': 'To revolve', '高山族': 'Gaoshan', '王飞飞是一个哑巴的婊子。': 'Wang Feifei is a dumb nephew.', '反清復明': 'Anti-clearing', '关门放狗': 'Close the dog', '中国民族伟大复兴？i': 'The great revival of the Chinese nation? i', '漢名': 'Han name', 'ín': 'into the', '이름은': 'name is', '뽀비엠퍼러': 'Pobi Emperor', '堅決反對台獨言論': 'Resolutely oppose Taiwan independence speech', 'ड़': 'D', 'ادب': 'Literary', '〖2x〗': '〖〗 2x', 'في': 'in a', 'परन्तप': 'Parantap', '不正常人類研究中心': 'Abnormal human research center', 'метью': 'by the way', 'σε': 'in', '罗马炮架': 'Roman gun mount', 'đỗ': 'parked', '二哈': 'Erha', '中國': 'China', 'मुझे': 'me', 'бджа': 'bzha', 'छुटाना': 'To leave', 'সহ': 'Including', 'δίδυμος': 'twin', 'दौरा': 'Tour', 'आया': 'He Came', 'ľor': 'Lor', 'شاف': 'balmy', 'افشلك': 'I miss you', 'पता': 'Address', 'śląska': 'Silesian', '我还有几个条件呢。': 'I still have a few conditions.', '元寇': 'Yuan', 'सहायक': 'Assistant', 'टाइम': 'Time', 'समुदाय': 'Community', 'टिपटिपवा': 'Tiptipava', '五毛党': 'Wu Mao Party', 'दे': 'Give', '屠城': 'Slaughter city', 'कहने': 'To say', 'कलमूहा': 'Kalamuha', 'בפנים': 'in', 'कर्माणि': 'Creation', 'अथवा': 'Or', 'રાજ્ય': 'State', 'घसोना': 'Shed', '今そう言う冗談で笑える気分じゃないから一人にしてって言ったら何があったの？話聞くよって言われたんだけど': 'I do not feel like being laughing with what I say now, so what happened when I told you to be alone? I was told by listening and talking', 'परमो': 'Paramo', 'υφηιρειτω': 'I suppose', 'כתובים': 'Written', '能夠': 'were able', '고등학교는': 'High School', 'बजरि': 'Breeze', 'जानेदिल': 'Knowingly', '大胜': 'Big win', 'काल': 'period', 'すみません': \"I'm sorry\", 'हिमाकत': 'Snow plant', 'þîfû': 'iphone', 'よね': 'right', 'هالنحس': 'Halting', 'الحجازية': 'Hijaz', 'жизнь': 'a life', 'किया': 'Did', '沸騰する': 'To boil', 'นะครับ': 'Yes', 'پیش': 'Before', 'परदाफास': 'Bustard', 'वफाएं': 'Affection', 'सैनानि': 'Sanani', 'ölü': 'dead', 'हैं': 'Are', '宋美齡': 'Song Meiling', 'கவாடம்': 'Valve', '我是一名来自大陆的高中生': 'I am a high school student from the mainland.', 'اجمل': 'The most beautiful', '가용': 'Available', 'चरम': 'Extreme', '象形文字': 'Hieroglyphics', '男女授受不亲': 'Men and women don’t kiss', 'моя': 'my', '疑义': 'Doubt', '管中閔': 'In the tube', 'çekseydin': 'pulls you were', 'عم': 'uncle', 'ルージュ': 'Rouge', '永遠': 'forever and always', 'بيت': 'a house', '「褒められた」': '\"I was praised\"', 'कर्णजित्': 'Karnajit', 'あたまわるい': 'Bad headache', 'ε0': 'e0', 'şafak': 'dawn', 'जलाओ': 'Burn', '観世音菩薩': '観世音', 'पीत': 'Yellow', 'दारी': 'Dari', 'የሚያየኝን': 'What i see', 'هاپو': 'Dog', '发声点': 'Vocal point', 'être': 'be', 'மாண்பாளன்': 'Manpalan', '감겨드려유': 'You can wrap it.', '知音': 'Bosom friend', 'एक': 'One', '\\x01jhó': '? Jho', 'かんぜおんぼさつ': 'Punctuation', 'ødegaard': 'Ødegaard', '得理也要让人': 'It’s ok to make people', 'مرة': 'Once', 'दो': 'two', 'ने': 'has', '用乡村包围城市': 'Surround the city with the countryside', 'مكتب': 'Office', 'řepa': 'beet', 'день': 'day', '江青': 'Jiang Qing', 'ángeles': 'angels', 'çonstant': 'constant', 'टिपटिपुआ': 'Tip Tip', 'ஆன்லைன்': 'Online', '盧麗安': \"Lu Li'an\", 'самый': 'most', 'からかってくる男に': 'To the coming men', 'بعرف': 'I know', '知乎': 'Know almost', 'पहेलि': 'Puzzles', 'बहार': 'spring', 'भंगिमाँ': 'Bhangima', 'くわんぜおんぼさつ': 'Honey', '河殤': 'River', 'شو': 'Shaw', 'محمد': 'Mohammed', 'спереди': 'in front', '没毛病': 'No problem', 'árbenz': 'Arbenz', 'लार्वा': 'Larvae', 'चढ़ा': 'Ascend', 'तो': 'so', 'يلعب': 'Play', 'घिसा': 'Ginger', 'रेखागड़ित': 'Sketchy', 'मुरदा': 'Murada', 'चित्र': 'picture', 'தாக்கல்': 'Filing', '大败美国队': 'Big defeat to the US team', 'искусственный': 'artificial', 'चाल': 'Trick', 'घुटता': 'Kneeling', 'बिगड़ना': 'Deteriorate', '您这口味奇特也就罢了': 'Your taste is strange.', 'लायक': 'Worth', '大篆': 'Daxie', 'खिलाना': 'To feed', 'ماهو': 'What is the', 'पहुँचनेके': 'To reach', '外来語': 'Foreign language', 'चटका': 'Click', 'کوالا': 'Koala bear', '不过': 'but', 'पड़ना': 'Fall', 'पानी': 'Water', '저의': 'my', '一生懸命': 'Hard', 'مش': 'not', 'लिखामि': 'Written', 'गया': 'Gaya', 'компания': 'company', 'तेरि': 'Teri', '茶髪': 'Brown hair', '하다': 'Do', '不是共和国；中华人民共和国和朝鲜民主主义人民共和国是共和国': 'Not a republic; the People’s Republic of China and the Democratic People’s Republic of Korea are Republic', '烤全羊多少人民币呢？': 'How much is the price of roast whole sheep?', 'घर': 'Home', 'पाला': 'Frost', '左右手': 'Left and right hand', 'کوالالامپور': 'Kuala Lumpur', 'āsanas': 'asanas', 'ոո': 'e', 'сегодня': 'Today', '저는': 'I am', 'русофил': 'blossoming', '甘蓝': 'Cabbage', '流浪到淡水': 'Wandering to fresh water', 'единайсет': 'eleven', 'परात': 'Underwear', '\\x7fhow': '?how', '¡que': 'what', 'ἀχιλλεύς': 'Achilles', 'так': 'So', 'آداب': 'Rituals', '\\x10i': '?i', 'मट्ठा': 'Whey', '平天下悠悠之口': 'The mouth of the world', 'लुन्डा': 'Lunda', 'ся': 'camping', 'вареники': 'Vareniks', 'δο': 'gt;', 'öyle': 'so', 'पुरे': 'Enough', '看他不顺眼': 'Seeing him not pleasing to the eye', '「o」': '\"O\"', 'とする': 'To', 'হচ্ছে': 'Being', 'लाखों': 'Millions', 'α1': \"A'1\", 'نهائي': 'Final', 'ɾ̃': 'ɾ', 'तिलक': 'Tilak', 'لا': 'No', 'صور': 'photo', '怒怼': 'Roar', 'மௌன': 'Mauna', 'परिस्थिति': 'Situation', '서로가': 'Mutually', '山進': 'Shanjin', '蝴蝶蛋': 'Butterfly egg', 'ксш': 'ksş', '饭可以乱吃': 'Rice can be eaten', '渡します': 'I will hand it over.', 'österreich': 'Austria', 'øÿ\\x13': 'øÿ?', 'ممتاز': 'Excellent', '蝴蝶卵': 'Butterfly egg', 'ही': 'Only', 'ठोकरे': 'Knock', '湿婆': 'Shiva', 'обожаю': 'love', 'लस्सी': 'Lassi', '操你妈': 'Fuck your mother', 'फौलादि': 'Fauladi', 'жизни': 'of life', 'đổi': 'change', '阻天下悠悠之口': 'Block the mouth of the world', 'オメエだよオメエと話してんのが苦痛なんだよ。シネ。': \"It's omn it is painful to talk to Oume. Cine.\", 'खाट': 'The cot', '译文': 'Translation', 'सँवरना': 'To embellish', 'дванадесет': 'twelve', '陳雲': 'Chen Yun', 'дванайсет': 'twelve', 'आँखें': 'Eyes', 'पूछते': 'Inquires', 'भंगी': 'Posture', 'सूना': 'Deserted', 'प्याला': 'Cup', 'には': 'To', 'доктора': 'the doctors', 'देना': 'give', 'बिरेन्र्द': 'Forget', 'गला': 'throat', 'रखा': 'Kept', 'हाले': 'Haley', '欢迎入坑': 'Welcome to the pit', 'डलि': 'Dallie', 'ôš': 'OS', 'يوسف': 'Yousuf', 'छोंकना': 'Strain', 'пп': 'pp', 'उसपर': 'on that', 'υρολογιστών': 'computers', '新年快乐！学业进步！身体健康！谢谢您们读我的翻译篇章': 'happy New Year! Academic progress! Healthy body! Thank you for reading my translation chapter.', '人民': 'people', 'घंटे': 'Hours', 'شباط': 'February', '食べる': 'eat', 'صلاح': 'Salah', '土澳': 'Tuao', '干嘛天天跟我说韩语': 'Why do you speak Korean with me every day?', 'ōnogi': 'climate', 'صاحب': 'owner', 'اكل': 'ate', '大唐': 'Datang', 'مطالعه': 'Study', '养生': 'Health', '车子': 'Car', 'कचरा': 'Garbage', 'महरबानि': 'Mehraban', 'शुष्टि': 'Shutti', 'интеллект': 'intelligence', '阮鏐': '阮镠', '鲁玥': 'Reckless', '입니다': 'is', 'ῥιζὤματα': 'Threads', 'люблю': 'love', 'ɛxɛʀċɨsɛ': 'it is not', 'कपड़े': 'dresses', 'उड़न': 'Flying', '广电总局': 'SARFT', '骂人': 'curse', 'የየየኝን': 'What do i say', 'बेलना': 'Crib', 'பல்லாக்குப்': 'Pallakkup', 'बराबर': 'equal', 'ظرف': 'Dish', 'होनोपैथिक': 'Homeopathic', '君子': 'Gentleman', '河和湖': 'Kawahata lake', '精一杯': 'Utmost', 'है': 'is', '非要以此为依据对人家批判一番': 'I have to criticize others on this basis.', 'வச்சன்': 'Vaccan', 'நான்': 'I', 'का': 'Of', '三味線': 'Shamisen', 'šwejk': 'švejk', 'дурак': 'fool', '风琴': 'organ', 'हिंसा': 'Violence', 'βιον': 'bio', 'नारी': 'Woman', '知らない': 'Do not know', 'मुँह': 'The mouth', 'अपरम्पार': 'Unperturbed', '秋季新款': 'Autumn new style', 'আমার': 'Me', 'عبقري': 'genius', 'आहें': 'Ah', 'ਨਾਮ': 'Name', 'महफ़िल': 'Mehfil', 'बटेर': 'Quail', '林彪': 'Lin Wei', 'जाने': 'Know', 'डोंगरिचाल': 'Mountain move', 'εἰρήνη': 'Irene', 'प्रतिलेखनम्': 'Transcript', 'дп': 'dp', 'उसने': 'He', '몇시간': 'how many hours', 'नैया': 'Naiya', 'ἐξήλλακτο': 'inexpensive', '彩蛋': 'Egg', 'उलटफेर': 'Reverse', '台湾最美的风景是人': 'The most beautiful scenery in Taiwan is people.', '馄饨': 'ravioli', 'सदके': 'Shake', '饺子': 'Dumplings', 'भूमिका': 'role', '为什么说': 'Why do you say', '요즘': 'Nowadays', 'गिरि': 'Giri', '中國話': 'Chinese words', 'द्रोहि': 'Drohi', '中庸之道': 'The doctrine of the mean', 'хочу': 'want', 'ḵarasāna': 'ḵarasana', '走gǒ': 'Go gǒ', 'जमाल': 'Jamal', 'मन': 'The mind', 'तेलि': 'Oilseed', '非诚勿扰': 'You Are the One', 'атом': 'atom', '中华民国和大韩民国是民国': 'The Republic of China and the Republic of Korea are the Republic of China', 'नलि': 'Nile', 'हाथ': 'hand', 'खाजला': 'Itching', 'ōe': 'yes', '것이다': 'will be', 'şoųl': 'şoùl', 'तश्तरि': 'Cleverness', 'χρῆσιν': 'use', '民族': 'Nationality', 'الرياضيات': 'Mathematics', 'にじゅうさい': 'Twelve months', 'ὠς': 'as', '恋に落ちないからよく悲しい': 'It is often sad because it does not fall in love', 'ਸ਼ੀਂਹ': 'Lion', '黎氏玉英': 'Li Shiyuying', 'فبراير': 'February', '白濮': 'Chalk', 'অধীনে': 'Under', 'प्रशंसा': 'appreciation', 'ขอพระเจ้าอยู่ด้วย': 'May God be with you', 'अडला': 'Bent', 'ده': 'Ten', 'पसीजना': 'Exudate', 'कोई': 'someone', 'கருக்குமட்டை': 'Karukkumattai', 'कन्नी': 'Kanni', 'यात्रा': 'journey', '白酒': 'Liquor', 'τὴν': 't', 'करना': 'do', 'उल': 'ul', '俄罗斯': 'Russia', 'உனக்கு': 'You', 'जौहर': 'Johar', 'ಸ್ವರಕ್ಷರಗಳು': 'Self-defense', '论语': 'Analects', 'šakotis': 'branching', '儿臣惶恐': 'Childier fear', '讲的？': 'Said?', 'প্রতিনিয়ত': 'Every day', 'சன்னல்': 'Sill', '蠢的像猪一样': 'Stupid like a pig', 'رح': 'Please', 'بس': 'Yes', 'εὔιδον': 'you see', 'सदबुद्धि': 'Good sense', 'भगवान': 'God', '사랑해': 'I love you', 'בתוך': 'Inside', 'čeferin': 'чеферин', '民国': 'Republic of China', '但是': 'but', 'सकता': 'can', 'घाट': 'Wharf', 'čechy': 'Bohemia', '抹黑': 'Smear', 'γλαυκῶπις': 'greyhounds', 'నీకెందుకు': 'Nikenduku', 'चमत्कार': 'Miracle', 'दुनिया': 'world', 'یہاں': 'Here', 'اللي': 'Elly', 'খামার': 'The farm', '一呼百诺': 'One call', 'വിഡ്ഢി': 'Stupid', 'दिल': 'heart', 'тeenage': 'trinage', '皇上': 'emperor', 'टटोलना': 'Grope', '犬子': 'Dog', '我希望有一天你沒有公王病': 'I hope that one day you don’t have a king', '并无分裂中国的意图': 'No intention to split China', 'óscar': 'oscar', 'ኤልሮኢ': 'Alright', 'ŷhat': 'hhat', '천사': 'Angel', 'दी।': 'Given', 'रड़क': 'Raze', 'कानी': 'Kani', '江之島盾子': 'Enokima Shiko', '老生常谈': 'Old talk', 'των': 'of', '星期日': 'on Sunday', 'पैन्डा': 'Panda', 'マリも仲直りしました': 'Mari also made up.', 'ты': 'you', 'देश': 'Country', 'ठंडक': 'Coolness', 'নামল': 'Get down', 'जर्मनी': 'Germany', 'шли': 'walked', 'たべる': 'To eat', 'लिए': 'for', '鸡汤文': 'Chicken soup', 'มวยไทย': 'Thai boxing', '簡訊': 'Newsletter', 'منزل': 'Home', 'कर': 'Tax', '生女眞': 'Daughter-in-law', 'ек': 'ek', 'સંઘ': 'Union', '\\u200bsalarpuria': 'Salphary', 'ţara': 'the country', 'नही': 'No', 'मगज': 'Mercury', 'অক্ষয়': 'Akshay', 'حال': 'Now', 'चिन्दी': 'Chindee', 'τῆς': 'her', '福哒柄': 'Good fortune', '청하': 'Qinghai', '越人': 'Yueren', 'なかなかに謎だな': \"It's quite a mystery.\", 'रखना': 'keep', 'பரை': 'Parai', 'करके': 'By doing', 'فِي': 'F', 'गुथना': 'Knit', '话不可以乱讲': \"Can't talk nonsense\", 'वैकल्पिक': 'Alternative', 'ਨਾਮੁ': 'Name', '你别高兴得太早': \"Don't be too happy too early\", '煎餅': 'Pancake', '한다': 'do', 'सबब': 'Cause', 'বিষয়টি': 'Matter', 'कोसना': 'To crack', 'ㅜㅜ': 'ㅜ', 'অক্সয়': 'Akshay', 'الدوالي': 'Varicose veins', 'பயிர்ப்பு': 'Yields', 'अजा': 'SC', 'あの色々': 'That kind of variety', 'емеля': 'emel', 'मेवा': 'Meva', 'जलवाफरोज़': 'Jalwa Phoroz', '中庸': 'Moderate', 'उसके': 'his', 'अहम': 'Important', 'वहम': 'Vanity', 'ís': 'ice', 'कलात्मक': 'Artistic', 'ἀχιλῆος': 'Achilles', '民族罪人': 'National sinner', 'मुकाम': 'Peer', 'ão': 'to', '한국': 'Korea', 'ادهر': 'Idir', '一長': 'One long', 'てくれませんか': 'Would you please', 'ブレイク': 'break', 'शहनाई': 'the clarinet', 'तीरन्दाज़ि': 'Arrows', 'रूढ़ीवादि': 'Conservative', 'झाँकी': 'Peeping', 'सत्यवादी': 'Truthful', '郑琳': 'Zheng Lin', 'युनानी': 'Unani', 'φώνας': 'light', 'जमना': 'Solidify', '〖plg〗': '〖Plg〗', 'हरी': 'Green', 'بطولة': 'championship', '这些词怎么读？这些词怎么说？这些词怎么念？which': 'How do you read these words? What do these words say? How do you read these words? Which', 'алтерман': 'alterman', 'اَلبَحْثِ': 'البحث', 'موجود': 'Available', 'कश्ति': 'Power', 'اسم': 'Noun', 'लाज़मि': 'Shameful', 'तुमने': 'you', '공화국': 'republic', 'लुक्का': 'Lukka', '史记': 'Historical record', '포경수술': 'Circumcised', '高端大气上档次into': 'High-end atmospheric grade into', 'что': 'what', 'योध': 'Rift', 'धर्म': 'religion', 'दरदर': 'Tariff', '訓読み': 'Kun Readings', 'впереди': 'in front', '민국': 'Republic of Korea', 'εντσα': 'in', '我搜了这本小说': 'I searched this novel.', '杨皎滢？': 'Yang Wei?', 'भारतीयों': 'Indians', '巴蜀': 'Bayu', '\\x02tñ\\x7f': '? tñ?', 'αβtαβ': 'aba', 'जल': 'water', 'बाध्य': 'Bound', 'মহাবিশ্ব': 'Universe', 'প্রসারিত': 'Stretch', 'अन्जाम': 'Anjaam', 'जीतना': 'win', 'कड़ा': 'Hard', '刁民': 'Untouchable', 'ขอให้พระเจ้าอยู่ด้วย': 'May God live too.', '油腻': 'Greasy', 'ᗯoᗰeᑎ': 'ᗯoᗰe ᑎ', 'להתראות': 'Goodbye', 'वाले': 'Ones', 'አየሁ': 'I saw', 'ओर': 'And', 'ずand': 'Without', 'निगोड़ा': 'Nigoda', 'эй': 'Hey'\n\n        }\n\n        def _get_contractions(contractions_dict):\n            contractions_re = re.compile('(%s)' % '|'.join(contractions_dict.keys()))\n            return contractions_dict, contractions_re\n\n        contractions, contractions_re = _get_contractions(contractions_dict)\n        def replace_contractions(text):\n            def replace(match):\n                return contractions[match.group(0)]\n            return contractions_re.sub(replace, text)\n        print('REPLACE CONTRACTIONS')\n        train_df['question_text'] = train_df['question_text'].apply(lambda x: replace_contractions(x))\n        test_df['question_text'] = test_df['question_text'].apply(lambda x: replace_contractions(x))\n        print('FINISH REPLACE CONTRACTIONS')\n        list_sentences_train = train_df[\"question_text\"].fillna(\"_na_\").values\n        list_sentences_test = test_df[\"question_text\"].fillna(\"_na_\").values\n\n        # TOKENIZE-----#\n        num_words=120000\n        maxlen=72\n        print('TOKENIZING')\n        tokenizer = Tokenizer(num_words=num_words, char_level=False, lower=False)\n        all_text  = list(list_sentences_train) + list(list_sentences_test)\n        tokenizer.fit_on_texts(all_text)\n        del all_text\n        gc.collect()\n        print('FINISH FITTING TOKENIZER')\n        train_sequences = tokenizer.texts_to_sequences(list_sentences_train)\n        train_sequences = pad_sequences(train_sequences, maxlen=maxlen)\n\n        test_sequences = tokenizer.texts_to_sequences(list_sentences_test)\n        test_sequences = pad_sequences(test_sequences, maxlen=maxlen)\n        print('FINISH prep sequences')\n        del list_sentences_train, list_sentences_test, contractions_dict\n        gc.collect()\n\n\n        # EMBEDDINGS-----#\n        def load_embedding(embedding):\n            print(f'Loading {embedding} embedding..')\n            def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n            def load_word2vec(fname, encoding='utf8', unicode_errors='strict',datatype=np.float32):\n                embedding_index = dict()\n                with utils.smart_open(fname) as fin:\n                    header = utils.to_unicode(fin.readline(), encoding=encoding)\n                    vocab_size, vector_size = (int(x) for x in header.split())\n                    binary_len = np.dtype(datatype).itemsize * vector_size\n                    for _ in tqdm(range(vocab_size)):\n                        # mixed text and binary: read text first, then binary\n                        word = []\n                        while True:\n                            ch = fin.read(1)\n                            if ch == b' ':\n                                break\n                            if ch == b'':\n                                raise EOFError(\"unexpected end of input\")\n                            if ch != b'\\n':\n                                word.append(ch)\n                        word = utils.to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)\n                        weights = np.fromstring(fin.read(binary_len), dtype=datatype).astype(datatype)\n                        embedding_index[word] = weights\n                return embedding_index\n            if embedding == 'glove':\n                EMBEDDING_FILE = f'{FILE_DIR}/embeddings/glove.840B.300d/glove.840B.300d.txt'\n                embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\"))\n            elif embedding == 'wiki-news':\n                EMBEDDING_FILE = f'{FILE_DIR}/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n                embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\") if len(o)>100)\n            elif embedding == 'paragram':\n                EMBEDDING_FILE = f'{FILE_DIR}/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n                embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n            elif embedding == 'google-news':\n                from gensim.models import KeyedVectors\n                EMBEDDING_FILE = f'{FILE_DIR}/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\n                #embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)\n                embeddings_index = load_word2vec(EMBEDDING_FILE)\n            return embeddings_index\n\n        def build_embedding_matrix(embeddings_index_1, one=True, embedding_name_1='random'): \n            if one:\n                embeddings_index_1 = load_embedding(embeddings_index_1)\n\n            wl = WordNetLemmatizer().lemmatize\n            word_index = tokenizer.word_index\n            nb_words = min(num_words, len(word_index))\n            embedding_matrix = np.zeros((nb_words, 301))\n\n            def all_caps(word):\n                return len(word) > 1 and word.isupper()\n\n            if embedding_name_1 == 'google-news':\n                something_1 = embeddings_index_1.word_vec(\"something\")\n                something = np.zeros((301,))\n                something[:300,] = something_1\n                something[300,] = 0\n\n                hit = 0\n                def embed_word(embedding_matrix,i,word):\n                    embedding_vector_1 = embeddings_index_1.word_vec(word)\n                    if embedding_vector_1 is not None: \n                        if all_caps(word):\n                            last_value = np.array([1])\n                        else:\n                            last_value = np.array([0])\n                        embedding_matrix[i,:300] = embedding_vector_1\n                        embedding_matrix[i,300] = last_value\n\n                for word, i in word_index.items():\n                    if i >= nb_words: continue\n                    if word not in embeddings_index_1.vocab:\n                        embedding_matrix[i] = something \n                    else:\n                        if embeddings_index_1.word_vec(word) is not None:\n                            embed_word(embedding_matrix,i,word)\n                            hit += 1\n                        else:\n                            if len(word) > 20:\n                                embedding_matrix[i] = something\n                            else:\n                                word2 = wl(wl(word, pos='v'), pos='a')\n                                if embeddings_index_1.word_vec(word2) is not None:\n                                    embed_word(embedding_matrix,i,word2)\n                                    hit += 1\n                                else:                   \n                                    if len(word) < 3: continue\n                                    word2 = word.lower()\n                                    if embeddings_index_1.word_vec(word2) is not None:\n                                        embed_word(embedding_matrix,i,word2)\n                                        hit += 1\n                                    else:\n                                        word2 = word.lower()\n                                        word2 = wl(wl(word2, pos='v'), pos='a')\n                                        if embeddings_index_1.word_vec(word2) is not None:\n                                            embed_word(embedding_matrix,i,word2)\n                                            hit += 1\n                                        else:\n                                            word2 = word.upper()\n                                            if embeddings_index_1.get(word2) is not None:\n                                                embed_word(embedding_matrix,i,word2)\n                                                hit += 1\n                                            else:\n                                                word2 = word.upper()\n                                                word2 = wl(wl(word2, pos='v'), pos='a')\n                                                if embeddings_index_1.get(word2) is not None:\n                                                    embed_word(embedding_matrix,i,word2)\n                                                    hit += 1\n                                                else:\n                                                    embedding_matrix[i] = something \n                             \n            else:\n                something_1 = embeddings_index_1.get(\"something\")\n                something = np.zeros((301,))\n                something[:300,] = something_1\n                something[300,] = 0\n\n                hit = 0\n                def embed_word(embedding_matrix,i,word):\n                    embedding_vector_1 = embeddings_index_1.get(word)\n                    if embedding_vector_1 is not None: \n                        if all_caps(word):\n                            last_value = np.array([1])\n                        else:\n                            last_value = np.array([0])\n                        embedding_matrix[i,:300] = embedding_vector_1\n                        embedding_matrix[i,300] = last_value\n\n                for word, i in word_index.items():\n                    if i >= nb_words: continue\n                    if embeddings_index_1.get(word) is not None:\n                        embed_word(embedding_matrix,i,word)\n                        hit += 1\n                    else:\n                        if len(word) > 20:\n                            embedding_matrix[i] = something\n                        else:\n                            word2 = wl(wl(word, pos='v'), pos='a')\n                            if embeddings_index_1.get(word2) is not None:\n                                embed_word(embedding_matrix,i,word2)\n                                hit += 1\n                            else:                   \n                                if len(word) < 3: continue\n                                word2 = word.lower()\n                                if embeddings_index_1.get(word2) is not None:\n                                    embed_word(embedding_matrix,i,word2)\n                                    hit += 1\n                                else:\n                                    word2 = word.lower()\n                                    word2 = wl(wl(word2, pos='v'), pos='a')\n                                    if embeddings_index_1.get(word2) is not None:\n                                        embed_word(embedding_matrix,i,word2)\n                                        hit += 1\n                                    else:\n                                        word2 = word.upper()\n                                        if embeddings_index_1.get(word2) is not None:\n                                            embed_word(embedding_matrix,i,word2)\n                                            hit += 1\n                                        else:\n                                            word2 = word.upper()\n                                            word2 = wl(wl(word2, pos='v'), pos='a')\n                                            if embeddings_index_1.get(word2) is not None:\n                                                embed_word(embedding_matrix,i,word2)\n                                                hit += 1\n                                            else:\n                                                embedding_matrix[i] = something \n            del embeddings_index_1\n            gc.collect()\n            print(\"Matched Embeddings: found {} out of total {} words at a rate of {:.2f}%\".format(hit, nb_words, hit * 100.0 / nb_words))\n            return embedding_matrix\n        def build_concatenate_embedding_matrix_google(embeddings_index_1, embeddings_index_2, one=True, two=True): \n            if one:\n                embeddings_index_1 = load_embedding(embeddings_index_1)\n            if two:\n                embeddings_index_2 = load_embedding(embeddings_index_2)\n\n\n            wl = WordNetLemmatizer().lemmatize\n            word_index = tokenizer.word_index\n            nb_words = min(num_words, len(word_index))\n            embedding_matrix = np.zeros((nb_words, 401))\n\n            something_1 = embeddings_index_1.get(\"something\")\n            something = np.zeros((401,))\n            something[:300,] = something_1\n            something[400,] = 0\n\n            def all_caps(word):\n                return len(word) > 1 and word.isupper()\n\n            hit = 0\n            def embed_word(embedding_matrix,i,word, ori_word):\n                embedding_vector_1 = embeddings_index_1.get(word)\n                if embedding_vector_1 is not None: \n                    if all_caps(word):\n                        last_value = np.array([1])\n                    else:\n                        last_value = np.array([0])\n                    embedding_matrix[i,:300] = embedding_vector_1\n                    embedding_matrix[i,400] = last_value\n                    if word in embeddings_index_2.vocab:\n                        embedding_vector_2 = embeddings_index_2.word_vec(ori_word)\n                        embedding_matrix[i,300:400] = embedding_vector_2\n\n            for word, i in word_index.items():\n                if i >= nb_words: continue\n                if embeddings_index_1.get(word) is not None:\n                    embed_word(embedding_matrix,i,word, word)\n                    hit += 1\n                else:\n                    if len(word) > 20:\n                        embedding_matrix[i] = something\n                    else:\n                        word2 = wl(wl(word, pos='v'), pos='a')\n                        if embeddings_index_1.get(word2) is not None:\n                            embed_word(embedding_matrix,i,word2, word)\n                            hit += 1\n                        else:                   \n                            if len(word) < 3: continue\n                            word2 = word.lower()\n                            if embeddings_index_1.get(word2) is not None:\n                                embed_word(embedding_matrix,i,word2, word)\n                                hit += 1\n                            else:\n                                word2 = word.lower()\n                                word2 = wl(wl(word2, pos='v'), pos='a')\n                                if embeddings_index_1.get(word2) is not None:\n                                    embed_word(embedding_matrix,i,word2, word)\n                                    hit += 1\n                                else:\n                                    word2 = word.upper()\n                                    if embeddings_index_1.get(word2) is not None:\n                                        embed_word(embedding_matrix,i,word2, word)\n                                        hit += 1\n                                    else:\n                                        word2 = word.upper()\n                                        word2 = wl(wl(word2, pos='v'), pos='a')\n                                        if embeddings_index_1.get(word2) is not None:\n                                            embed_word(embedding_matrix,i,word2, word)\n                                            hit += 1\n                                        else:\n                                            embedding_matrix[i] = something \n            print(\"Matched Embeddings: found {} out of total {} words at a rate of {:.2f}%\".format(hit, nb_words, hit * 100.0 / nb_words))\n            del embeddings_index_1, embeddings_index_2\n            gc.collect()\n            return embedding_matrix\n        def build_concatenate_embedding_matrix(embeddings_index_1, embeddings_index_2, one=True, two=True): \n            if one:\n                embeddings_index_1 = load_embedding(embeddings_index_1)\n            if two:\n                embeddings_index_2 = load_embedding(embeddings_index_2)\n\n\n            wl = WordNetLemmatizer().lemmatize\n            word_index = tokenizer.word_index\n            nb_words = min(num_words, len(word_index))\n            embedding_matrix = np.zeros((nb_words, 601))\n\n            something_1 = embeddings_index_1.get(\"something\")\n            something_2 = embeddings_index_2.get(\"something\")\n            something = np.zeros((601,))\n            something[:300,] = something_1\n            something[300:600,] = something_2\n            something[600,] = 0\n\n            def all_caps(word):\n                return len(word) > 1 and word.isupper()\n\n            hit = 0\n            def embed_word(embedding_matrix,i,word):\n                embedding_vector_1 = embeddings_index_1.get(word)\n                if embedding_vector_1 is not None: \n                    if all_caps(word):\n                        last_value = np.array([1])\n                    else:\n                        last_value = np.array([0])\n                    embedding_matrix[i,:300] = embedding_vector_1\n                    embedding_matrix[i,600] = last_value\n                    embedding_vector_2 = embeddings_index_2.get(word)\n                    if embedding_vector_2 is not None:\n                        embedding_matrix[i,300:600] = embedding_vector_2\n\n            for word, i in word_index.items():\n                if i >= nb_words: continue\n                if embeddings_index_1.get(word) is not None:\n                    embed_word(embedding_matrix,i,word)\n                    hit += 1\n                else:\n                    if len(word) > 20:\n                        embedding_matrix[i] = something\n                    else:\n                        word2 = wl(wl(word, pos='v'), pos='a')\n                        if embeddings_index_1.get(word2) is not None:\n                            embed_word(embedding_matrix,i,word2)\n                            hit += 1\n                        else:                   \n                            if len(word) < 3: continue\n                            word2 = word.lower()\n                            if embeddings_index_1.get(word2) is not None:\n                                embed_word(embedding_matrix,i,word2)\n                                hit += 1\n                            else:\n                                word2 = word.lower()\n                                word2 = wl(wl(word2, pos='v'), pos='a')\n                                if embeddings_index_1.get(word2) is not None:\n                                    embed_word(embedding_matrix,i,word2)\n                                    hit += 1\n                                else:\n                                    word2 = word.upper()\n                                    if embeddings_index_1.get(word2) is not None:\n                                        embed_word(embedding_matrix,i,word2)\n                                        hit += 1\n                                    else:\n                                        word2 = word.upper()\n                                        word2 = wl(wl(word2, pos='v'), pos='a')\n                                        if embeddings_index_1.get(word2) is not None:\n                                            embed_word(embedding_matrix,i,word2)\n                                            hit += 1\n                                        else:\n                                            embedding_matrix[i] = something \n            print(\"Matched Embeddings: found {} out of total {} words at a rate of {:.2f}%\".format(hit, nb_words, hit * 100.0 / nb_words))\n            del embeddings_index_1, embeddings_index_2\n            gc.collect()\n            return embedding_matrix\n\n\n        # MODELS-----#\n        def f1_smart(y_true, y_pred):\n            args = np.argsort(y_pred)\n            tp = y_true.sum()\n            fs = (tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)\n            res_idx = np.argmax(fs)\n            return 2 * fs[res_idx], (y_pred[args[res_idx]] + y_pred[args[res_idx + 1]]) / 2\n\n        def threshold_search(y_true, y_proba):\n            best_threshold = 0\n            best_score = 0\n            for threshold in [i * 0.01 for i in range(100)]:\n                score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n                if score > best_score:\n                    best_threshold = threshold\n                    best_score = score\n            search_result = {'threshold': best_threshold, 'f1': best_score}\n            return search_result\n\n        def dropout_mask(x, sz, dropout):\n            return x.new(*sz).bernoulli_(1-dropout)/(1-dropout)\n\n        class LockedDropout(nn.Module):\n            def __init__(self, p=0.5):\n                super().__init__()\n                self.p=p\n\n            def forward(self, x):\n                if not self.training or not self.p: return x\n                m = dropout_mask(x.data, (1, x.size(1), x.size(2)), self.p)\n                return Variable(m, requires_grad=False) * x\n\n        def sigmoid(x):\n            return 1 / (1 + np.exp(-x))\n\n        class AverageMeter(object):\n            \"\"\"Computes and stores the average and current value\"\"\"\n\n            def __init__(self):\n                self.reset()\n\n            def reset(self):\n                self.val = 0\n                self.avg = 0\n                self.sum = 0\n                self.count = 0\n\n            def update(self, val, n=1):\n                self.val = val\n                self.sum += val * n\n                self.count += n\n                self.avg = self.sum / self.count\n\n        def accuracy(output, target, topk=(1,)):\n            \"\"\"Computes the accuracy over the k top predictions for the specified values of k\"\"\"\n            with torch.no_grad():\n                maxk = max(topk)\n                batch_size = target.size(0)\n\n                _, pred = output.topk(maxk, 1, True, True)\n                pred = pred.t()\n                correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n                res = []\n                for k in topk:\n                    correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)\n                    res.append(correct_k.mul_(100.0 / batch_size))\n            return res\n\n        def train_model(train_loader, model, criterion, optimizer, epoch, gpu=None, print_freq=100):\n            batch_time = AverageMeter()\n            data_time = AverageMeter()\n            losses = AverageMeter()\n            top1 = AverageMeter()\n\n            model.train()\n\n            epoch_time = time.time()\n            end = time.time()\n\n            for i, (x_batch, x_feats_batch, y_batch) in enumerate(train_loader):\n                data_time.update(time.time() - end)\n                if gpu is not None:\n                    x_batch = x_batch.cuda(gpu, non_blocking=True)\n                    x_feats_batch= x_feats_batch.cuda(gpu, non_blocking=True)\n                y_batch = y_batch.cuda(gpu, non_blocking=True)\n\n                output = model(x_batch, x_feats_batch)\n                loss = criterion(output, y_batch)\n\n                # measure accuracy\n                acc1 = (y_batch == (output > 0.5).float()).float()\n                losses.update(loss.item(), x_batch.size(0))\n                top1.update(acc1.mean().item(), x_batch.size(0))\n\n                # compute gradient and do SGD step\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n                # measure elapsed time\n                batch_time.update(time.time() - end)\n                end = time.time()\n\n                if i % print_freq == 0:\n                    print('Epoch: [{0}][{1}/{2}]\\t'\n                          'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n                          'Data {data_time.val:.3f} ({data_time.avg:.3f})\\t'\n                          'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n                          'Acc@1 {top1.avg:.3f}'\n                          .format(\n                              epoch, i, len(train_loader), batch_time=batch_time,\n                              data_time=data_time, loss=losses, top1=top1))\n\n            print('Epoch time: {:.4f}min'.format((time.time() - epoch_time) / 60))\n            return\n        def train_model_boost(train_loader, model, criterion, optimizer, epoch, gpu=None, print_freq=100):\n            batch_time = AverageMeter()\n            data_time = AverageMeter()\n            losses = AverageMeter()\n            top1 = AverageMeter()\n\n            model.train()\n\n            epoch_time = time.time()\n            end = time.time()\n\n            for i, (x_batch, x_feats_batch, y_batch) in enumerate(train_loader):\n                data_time.update(time.time() - end)\n                if gpu is not None:\n                    x_batch = x_batch.cuda(gpu, non_blocking=True)\n                    x_feats_batch= x_feats_batch.cuda(gpu, non_blocking=True)\n                y_batch = y_batch.cuda(gpu, non_blocking=True)\n\n                output = model(x_batch, x_feats_batch)\n                output = torch.nn.Tanh()(output)\n                loss = criterion(output, y_batch)\n\n                # measure accuracy\n                acc1 = (y_batch == (output > 0.5).float()).float()\n                losses.update(loss.item(), x_batch.size(0))\n                top1.update(acc1.mean().item(), x_batch.size(0))\n\n                # compute gradient and do SGD step\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n                # measure elapsed time\n                batch_time.update(time.time() - end)\n                end = time.time()\n\n                if i % print_freq == 0:\n                    print('Epoch: [{0}][{1}/{2}]\\t'\n                          'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n                          'Data {data_time.val:.3f} ({data_time.avg:.3f})\\t'\n                          'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n                          'Acc@1 {top1.avg:.3f}'\n                          .format(\n                              epoch, i, len(train_loader), batch_time=batch_time,\n                              data_time=data_time, loss=losses, top1=top1))\n\n            print('Epoch time: {:.4f}min'.format((time.time() - epoch_time) / 60))\n            return\n        def test_model(loader, model, gpu=None, return_true=False):\n            model.eval()\n            outputs = []\n\n            with torch.no_grad():\n                end = time.time()\n                for i, (x_batch, x_feats_batch) in enumerate(loader):\n                    if gpu is not None:\n                        x_batch = x_batch.cuda(gpu, non_blocking=True)\n                        x_feats_batch = x_feats_batch.cuda(gpu, non_blocking=True)\n\n                    output = model(x_batch, x_feats_batch)\n\n                    y_pred = output.detach()\n                    outputs.append(sigmoid(y_pred.cpu().numpy())[:, 0])\n\n            outputs = np.concatenate(outputs)\n            return outputs\n        def test_model_boost(loader, model, gpu=None, return_true=False):\n            model.eval()\n            outputs = []\n\n            with torch.no_grad():\n                end = time.time()\n                for i, (x_batch, x_feats_batch) in enumerate(loader):\n                    if gpu is not None:\n                        x_batch = x_batch.cuda(gpu, non_blocking=True)\n                        x_feats_batch = x_feats_batch.cuda(gpu, non_blocking=True)\n\n                    output = model(x_batch, x_feats_batch)\n                    output = torch.nn.Tanh()(output)\n                    y_pred = output.detach()\n                    outputs.append(y_pred.cpu().numpy())\n\n            outputs = np.concatenate(outputs)\n            return outputs\n\n        # zoo\n        class ModelRNN(nn.Module):\n            def __init__(self, max_features, embedding_matrix, hidden_size, linear1_in, linear1_out, lockout, dropout):\n                super(ModelRNN, self).__init__()\n                self.max_features = max_features\n                self.embedding_matrix = embedding_matrix\n                self.hidden_size = hidden_size\n                self.linear1_in = linear1_in\n                self.linear1_out = linear1_out\n                self.locked_dropout = LockedDropout(lockout)\n                self.dropout = dropout\n\n                embed_size = self.embedding_matrix.shape[1]\n\n                self.embedding = nn.Embedding(self.max_features, embed_size)\n                self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n                self.embedding.weight.requires_grad = False\n\n                self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n                self.gru = nn.GRU(hidden_size*2, hidden_size*2, bidirectional=True, batch_first=True)\n\n                self.linear = nn.Linear(self.linear1_in, self.linear1_out)\n                self.relu = nn.ReLU()\n                # self.dropout = nn.Dropout(self.dropout)\n                self.out = nn.Linear(self.linear1_out, 1)\n\n            def forward(self, x, x_feats):\n                h_embedding = self.embedding(x)\n                # h_embedding = self.locked_dropout(h_embedding)\n\n                h_lstm, _ = self.lstm(h_embedding)\n                h_gru, _ = self.gru(h_lstm)\n\n                avg_pool = torch.mean(h_gru, 1)\n                max_pool, _ = torch.max(h_gru, 1)\n                last_state = h_gru[:, -1, :]\n\n                conc = torch.cat((avg_pool, max_pool, last_state, x_feats), 1)\n\n                conc = self.relu(self.linear(conc))\n                # conc = self.dropout(conc)\n                out = self.out(conc)\n\n                return out\n\n        class ModelRNN_fc2(nn.Module):\n            def __init__(self, max_features, embedding_matrix, hidden_size, linear1_in, linear1_out, linear2_out, lockout, dropout):\n                super(ModelRNN_fc2, self).__init__()\n                self.max_features = max_features\n                self.embedding_matrix = embedding_matrix\n                self.hidden_size = hidden_size\n                self.linear1_in = linear1_in\n                self.linear1_out = linear1_out\n                self.linear2_out = linear2_out\n                self.locked_dropout = LockedDropout(lockout)\n                self.dropout = dropout\n\n                embed_size = self.embedding_matrix.shape[1]\n\n                self.embedding = nn.Embedding(self.max_features, embed_size)\n                self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n                self.embedding.weight.requires_grad = False\n\n                self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n                self.gru = nn.GRU(hidden_size*2, hidden_size*2, bidirectional=True, batch_first=True)\n\n                self.linear1 = nn.Linear(self.linear1_in, self.linear1_out)\n                self.relu1 = nn.ReLU()\n                self.linear2 = nn.Linear(self.linear1_out, self.linear2_out)\n                self.relu2 = nn.ReLU()\n                self.dropout = nn.Dropout(self.dropout)\n\n                self.out = nn.Linear(self.linear2_out, 1)\n\n            def forward(self, x, x_feats):\n                h_embedding = self.embedding(x)\n                # h_embedding = self.locked_dropout(h_embedding)\n\n                h_lstm, _ = self.lstm(h_embedding)\n                h_gru, _ = self.gru(h_lstm)\n\n                avg_pool = torch.mean(h_gru, 1)\n                max_pool, _ = torch.max(h_gru, 1)\n                last_state = h_gru[:, -1, :]\n\n                conc = torch.cat((avg_pool, max_pool, last_state, x_feats), 1)\n\n                conc = self.relu1(self.linear1(conc))\n                conc = self.dropout(conc)\n                conc = self.relu2(self.linear2(conc))\n                conc = self.dropout(conc)\n                out = self.out(conc)\n\n                return out\n\n        class DPCNNTextClassifier(nn.Module):\n            \"\"\"\n            DPCNN for sentences classification.\n            \"\"\"\n            def __init__(self, max_features, embedding_matrix, hidden_size, input_dropout_p):\n                super(DPCNNTextClassifier, self).__init__()\n                self.max_features = max_features\n                self.embedding_matrix = embedding_matrix\n                self.hidden_size = hidden_size\n                self.input_dropout_p = input_dropout_p\n\n                embed_size = self.embedding_matrix.shape[1]\n\n                self.embedding = nn.Embedding(self.max_features, embed_size)\n                self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n                self.embedding.weight.requires_grad = False\n\n                self.input_dropout = nn.Dropout(p=input_dropout_p)\n\n                self.channel_size = 150\n                self.conv_region_embedding = nn.Conv2d(1, self.channel_size, (3, embed_size), stride=1)\n                self.conv3 = nn.Conv2d(self.channel_size, self.channel_size, (3, 1), stride=1)\n                self.pooling = nn.MaxPool2d(kernel_size=(3, 1), stride=2)\n                self.padding_conv = nn.ZeroPad2d((0, 0, 1, 1))\n                self.padding_pool = nn.ZeroPad2d((0, 0, 0, 1))\n                self.act_fun = nn.ReLU()\n                self.linear_out = nn.Linear(self.channel_size, 1)\n\n            def forward(self, input_var, lengths=None):\n                embeded = self.embedding(input_var)\n                embeded = self.input_dropout(embeded)\n                batch, width, height = embeded.shape\n                embeded = embeded.view((batch, 1, width, height))\n\n                # Region embedding\n                x = self.conv_region_embedding(embeded)\n                x = self.padding_conv(x)\n                x = self.act_fun(x)\n                x = self.conv3(x)\n                x = self.padding_conv(x)\n                x = self.act_fun(x)\n                x = self.conv3(x)\n\n                while x.size()[-2] >= 2:\n                    x = self._block(x)\n\n                x = x.view(batch, self.channel_size)\n                x = self.linear_out(x)\n\n                return x\n\n            def _block(self, x):\n                # Pooling\n                x = self.padding_pool(x)\n                px = self.pooling(x)\n\n                # Convolution\n                x = self.padding_conv(px)\n                x = F.relu(x)\n                x = self.conv3(x)\n\n                x = self.padding_conv(x)\n                x = F.relu(x)\n                x = self.conv3(x)\n\n                # Short Cut\n                x = x + px\n\n                return x\n\n        hidden_size = 90\n        gru_len = hidden_size\n\n        Routings = 4 #5\n        Num_capsule = 5\n        Dim_capsule = 5#16\n        dropout_p = 0.25\n        rate_drop_dense = 0.28\n        LR = 0.001\n        T_epsilon = 1e-7\n        num_classes = 30\n        class Embed_Layer(nn.Module):\n            def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=601):\n                super(Embed_Layer, self).__init__()\n                self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)\n                if use_pretrained_embedding:\n                    # self.encoder.weight.data.copy_(t.from_numpy(np.load(embedding_path))) # 方法一，加载np.save的npy文件\n                    self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))  # 方法二\n\n            def forward(self, x, dropout_p=0.25):\n                return nn.Dropout(p=dropout_p)(self.encoder(x))\n\n\n        class GRU_Layer(nn.Module):\n            def __init__(self):\n                super(GRU_Layer, self).__init__()\n                self.gru = nn.GRU(input_size=300,\n                                  hidden_size=gru_len,\n                                  bidirectional=True)\n                '''\n                自己修改GRU里面的激活函数及加dropout和recurrent_dropout\n                如果要使用，把rnn_revised import进来，但好像是使用cpu跑的，比较慢\n               '''\n                # # if you uncomment /*from rnn_revised import * */, uncomment following code aswell\n                # self.gru = RNNHardSigmoid('GRU', input_size=300,\n                #                           hidden_size=gru_len,\n                #                           bidirectional=True)\n\n            # 这步很关键，需要像keras一样用glorot_uniform和orthogonal_uniform初始化参数\n            def init_weights(self):\n                ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n                hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n                b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n                for k in ih:\n                    nn.init.xavier_uniform_(k)\n                for k in hh:\n                    nn.init.orthogonal_(k)\n                for k in b:\n                    nn.init.constant_(k, 0)\n\n            def forward(self, x):\n                return self.gru(x)\n\n\n        # core caps_layer with squash func\n        class Caps_Layer(nn.Module):\n            def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \\\n                         routings=Routings, kernel_size=(9, 1), share_weights=True,\n                         activation='default', **kwargs):\n                super(Caps_Layer, self).__init__(**kwargs)\n\n                self.num_capsule = num_capsule\n                self.dim_capsule = dim_capsule\n                self.routings = routings\n                self.kernel_size = kernel_size  # 暂时没用到\n                self.share_weights = share_weights\n                if activation == 'default':\n                    self.activation = self.squash\n                else:\n                    self.activation = nn.ReLU(inplace=True)\n\n                if self.share_weights:\n                    self.W = nn.Parameter(\n                        nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))\n                else:\n                    self.W = nn.Parameter(\n                        t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))  # 64即batch_size\n\n            def forward(self, x):\n\n                if self.share_weights:\n                    u_hat_vecs = t.matmul(x, self.W)\n                else:\n                    print('add later')\n\n                batch_size = x.size(0)\n                input_num_capsule = x.size(1)\n                u_hat_vecs = u_hat_vecs.view((batch_size, input_num_capsule,\n                                              self.num_capsule, self.dim_capsule))\n                u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)  # 转成(batch_size,num_capsule,input_num_capsule,dim_capsule)\n                b = t.zeros_like(u_hat_vecs[:, :, :, 0])  # (batch_size,num_capsule,input_num_capsule)\n\n                for i in range(self.routings):\n                    b = b.permute(0, 2, 1)\n                    c = F.softmax(b, dim=2)\n                    c = c.permute(0, 2, 1)\n                    b = b.permute(0, 2, 1)\n                    outputs = self.activation(t.einsum('bij,bijk->bik', (c, u_hat_vecs)))  # batch matrix multiplication\n                    # outputs shape (batch_size, num_capsule, dim_capsule)\n                    if i < self.routings - 1:\n                        b = t.einsum('bik,bijk->bij', (outputs, u_hat_vecs))  # batch matrix multiplication\n                return outputs  # (batch_size, num_capsule, dim_capsule)\n\n            # text version of squash, slight different from original one\n            def squash(self, x, axis=-1):\n                s_squared_norm = (x ** 2).sum(axis, keepdim=True)\n                scale = t.sqrt(s_squared_norm + T_epsilon)\n                return x / scale\n\n        class Capsule_Main(nn.Module):\n            def __init__(self, embedding_matrix=None, vocab_size=None):\n                super(Capsule_Main, self).__init__()\n                self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)\n                self.gru_layer = GRU_Layer()\n                # 【重要】初始化GRU权重操作，这一步非常关键，acc上升到0.98，如果用默认的uniform初始化则acc一直在0.5左右\n                self.gru_layer.init_weights()\n                self.caps_layer = Caps_Layer()\n                self.dense_layer = Dense_Layer()\n\n            def forward(self, content):\n                content1 = self.embed_layer(content)\n                content2, _ = self.gru_layer(\n                    content1)  # 这个输出是个tuple，一个output(seq_len, batch_size, num_directions * hidden_size)，一个hn\n                content3 = self.caps_layer(content2)\n                output = self.dense_layer(content3)\n                return output\n\n        class Attention(nn.Module):\n            def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n                super(Attention, self).__init__(**kwargs)\n\n                self.supports_masking = True\n\n                self.bias = bias\n                self.feature_dim = feature_dim\n                self.step_dim = step_dim\n                self.features_dim = 0\n\n                weight = torch.zeros(feature_dim, 1)\n                nn.init.xavier_uniform_(weight)\n                self.weight = nn.Parameter(weight)\n\n                if bias:\n                    self.b = nn.Parameter(torch.zeros(step_dim))\n\n            def forward(self, x, mask=None):\n                feature_dim = self.feature_dim\n                step_dim = self.step_dim\n\n                eij = torch.mm(\n                    x.contiguous().view(-1, feature_dim), \n                    self.weight\n                ).view(-1, step_dim)\n\n                if self.bias:\n                    eij = eij + self.b\n\n                eij = torch.tanh(eij)\n                a = torch.exp(eij)\n\n                if mask is not None:\n                    a = a * mask\n\n                a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n                weighted_input = x * torch.unsqueeze(a, -1)\n                return torch.sum(weighted_input, 1)\n\n\n        class CapsNet(nn.Module):\n            def __init__(self, max_features, embedding_matrix, hidden_size, linear1_in):\n                super(CapsNet, self).__init__()\n                self.max_features = max_features\n                self.embedding_matrix = embedding_matrix\n                self.hidden_size = hidden_size\n                self.linear1_in = linear1_in\n\n                fc_layer = 16\n                fc_layer1 = 16\n\n                embed_size = self.embedding_matrix.shape[1]\n\n                self.embedding = nn.Embedding(self.max_features, embed_size)\n                self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n                self.embedding.weight.requires_grad = False\n\n                self.embedding_dropout = nn.Dropout2d(0.1)\n                self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n                self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n\n                self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n                self.bn = nn.BatchNorm1d(16, momentum=0.5)\n                self.linear = nn.Linear(self.linear1_in, fc_layer1) #643:80 - 483:60 - 323:40\n                self.relu = nn.ReLU()\n                self.dropout = nn.Dropout(0.1)\n                self.fc = nn.Linear(fc_layer**2,fc_layer)\n                self.out = nn.Linear(fc_layer, 1)\n                self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)\n                self.caps_layer = Caps_Layer()\n\n            def forward(self, x, x_feats):\n                h_embedding = self.embedding(x)\n\n                h_lstm, _ = self.lstm(h_embedding)\n                h_gru, _ = self.gru(h_lstm)\n\n                ##Capsule Layer        \n                content3 = self.caps_layer(h_gru)\n                content3 = self.dropout(content3)\n                batch_size = content3.size(0)\n                content3 = content3.view(batch_size, -1)\n                content3 = self.relu(self.lincaps(content3))\n\n                # global average pooling\n                avg_pool = torch.mean(h_gru, 1)\n                # global max pooling\n                max_pool, _ = torch.max(h_gru, 1)\n                last_state = h_gru[:, -1, :]\n\n                conc = torch.cat((content3, avg_pool, max_pool, last_state, x_feats), 1) # h_lstm_atten, h_gru_atten\n                conc = self.relu(self.linear(conc))\n                conc = self.bn(conc)\n                conc = self.dropout(conc)\n\n                out = self.out(conc)\n\n                return out\n        # TRAIN-----#\n        gpu = 0\n        batch_size = 512\n        start_epoch = 0\n        np.save('train_sequences.npy',train_sequences)\n        np.save('training_labels.npy',training_labels)\n        np.save('train_features.npy',train_features)\n        np.save('test_sequences.npy',test_sequences)\n        np.save('test_features.npy',test_features)\n        print('PREPARE FOR PYTORCH')\n        x_train = torch.tensor(train_sequences, dtype=torch.long).cuda()\n        y_train = torch.tensor(training_labels[:, np.newaxis], dtype=torch.float32).cuda()\n        x_train_feats = torch.tensor(train_features, dtype=torch.float32).cuda()\n        print('FINISH TRAIN PREMINILARY FOR PYTORCH')\n        x_train_data = torch.utils.data.TensorDataset(x_train, x_train_feats, y_train)\n        train = torch.utils.data.DataLoader(x_train_data, batch_size=batch_size, shuffle=True)\n        del x_train, x_train_feats, x_train_data, y_train\n        gc.collect()\n        print('FINISH TRAIN FOR PYTORCH')\n        x_test = torch.tensor(test_sequences, dtype=torch.long).cuda()\n        x_test_feats = torch.tensor(test_features, dtype=torch.float32).cuda()\n        x_test_data = torch.utils.data.TensorDataset(x_test, x_test_feats)\n        test = torch.utils.data.DataLoader(x_test_data, batch_size=batch_size, shuffle=False)\n\n        del train_sequences, train_features, test_sequences, test_features\n        gc.collect()\n\n        del x_test, x_test_feats, x_test_data\n        gc.collect()\n\n        # fasttext-RNN2\n        embed_google_index = load_embedding('google-news')\n        embedding_matrix = build_embedding_matrix(embed_google_index, one=False)\n\n        rnn_kwargs = {\n            'max_features': num_words,\n            'embedding_matrix': embedding_matrix,\n            'hidden_size': 90,\n            'linear1_in': 1082,\n            'linear1_out': 70,\n            'linear2_out':16,\n            'lockout': 0.1,\n            'dropout': 0.1,\n        }\n        del embedding_matrix\n        gc.collect()\n        epochs = 4\n        model = ModelRNN_fc2(**rnn_kwargs)\n        del rnn_kwargs\n        gc.collect()\n        model.cuda()\n        criterion = torch.nn.BCEWithLogitsLoss(reduction=\"sum\")\n        lr=0.003\n        for epoch in range(start_epoch, epochs):\n            optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n            train_model(train, model, criterion, optimizer, epoch, gpu=gpu)\n            lr = lr*0.8\n\n        preds = np.zeros((len(test_idx)))\n        preds = test_model(test, model, gpu=gpu)\n        preds = preds.ravel()\n        if split>0:\n            best_score, _ = f1_smart(testing_labels, preds)\n            print ('RNN2-googlenews :', best_score)   \n        np.save('preds_googlenews.npy', preds)\n\n        del model, criterion, optimizer, preds\n        gc.collect()\n        \n        # paragram-google-RNN2\n        embedding_matrix = build_concatenate_embedding_matrix('paragram',embed_google_index, two=False)\n        del embed_google_index\n        gc.collect()\n        rnn_kwargs = {\n            'max_features': num_words,\n            'embedding_matrix': embedding_matrix,\n            'hidden_size': hidden_size,\n            'linear1_in': 543,\n        }\n\n        del embedding_matrix\n        gc.collect()\n        start_epoch = 0\n        model = CapsNet(**rnn_kwargs)\n        del rnn_kwargs\n        gc.collect()\n        model.cuda()\n        criterion = torch.nn.BCEWithLogitsLoss(reduction=\"sum\")\n        epochs = 3\n        lr=0.003\n        for epoch in range(start_epoch, epochs):\n            optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n            train_model(train, model, criterion, optimizer, epoch, gpu=gpu)\n            lr = lr*0.8\n\n        preds = np.zeros((len(test_idx)))\n        preds = test_model(test, model, gpu=gpu)\n        preds = preds.ravel()\n        if split>0:\n            best_score, _ = f1_smart(testing_labels, preds)\n            print ('RNN2-paragram_google :', best_score)   \n        np.save('preds_paragram_google.npy', preds)\n        \n        del train, test\n        gc.collect()\n        \n        train_sequences = np.load('train_sequences.npy')\n        #training_labels = np.load('training_labels.npy')\n        train_features = np.load('train_features.npy')\n        test_sequences = np.load('test_sequences.npy')\n        test_features = np.load('test_features.npy')\n        oof_pred_LR = np.load('oof_pred_LR.npy')\n\n        training_labels_boost = training_labels - oof_pred_LR\n        del oof_pred_LR\n        gc.collect()\n        print('PREPARE FOR PYTORCH')\n        x_train = torch.tensor(train_sequences, dtype=torch.long).cuda()\n        y_train = torch.tensor(training_labels_boost[:, np.newaxis], dtype=torch.float32).cuda()\n        x_train_feats = torch.tensor(train_features, dtype=torch.float32).cuda()\n        print('FINISH TRAIN PREMINILARY FOR PYTORCH')\n        x_train_data = torch.utils.data.TensorDataset(x_train, x_train_feats, y_train)\n        train = torch.utils.data.DataLoader(x_train_data, batch_size=batch_size, shuffle=True)\n        del x_train, x_train_feats, x_train_data, y_train\n        gc.collect()\n        print('FINISH TRAIN FOR PYTORCH')\n        x_test = torch.tensor(test_sequences, dtype=torch.long).cuda()\n        x_test_feats = torch.tensor(test_features, dtype=torch.float32).cuda()\n        x_test_data = torch.utils.data.TensorDataset(x_test, x_test_feats)\n        test = torch.utils.data.DataLoader(x_test_data, batch_size=batch_size, shuffle=False)\n\n        del train_sequences, train_features, test_sequences, test_features, training_labels_boost\n        gc.collect()\n        \n        # glove-RNN2\n        embed_glove_index = load_embedding('glove')\n        #model = Word2Vec(all_text, size=100, window=5, min_count=10, workers=1)\n        #del all_text\n        #gc.collect()\n        embedding_matrix = build_embedding_matrix(embed_glove_index, one=False)\n        del model\n        gc.collect()\n        rnn_kwargs = {\n            'max_features': num_words,\n            'embedding_matrix': embedding_matrix,\n            'hidden_size': 90,\n            'linear1_in': 1082,\n            'linear1_out': 70,\n            'lockout': 0.1,\n            'dropout': 0.1,\n        }\n        epochs = 3\n        model = ModelRNN(**rnn_kwargs)\n        del embedding_matrix, rnn_kwargs\n        gc.collect()\n        model.cuda()\n        criterion = torch.nn.MSELoss()\n        lr=0.003\n        for epoch in range(start_epoch, epochs):\n            optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n            train_model_boost(train, model, criterion, optimizer, epoch, gpu=gpu)\n            lr = lr*0.8\n\n        preds = np.zeros((len(test_idx)))\n        preds = test_model_boost(test, model, gpu=gpu)\n        preds = preds.ravel()\n        test_pred_LR = np.load('test_pred_LR.npy')\n        preds = preds + test_pred_LR\n        del test_pred_LR\n        gc.collect()\n        if split>0:\n            best_score, _ = f1_smart(testing_labels, preds)\n            print ('RNN2-glove :', best_score)   \n        np.save('preds_glove.npy', preds)\n\n        del model, criterion, optimizer, preds, train\n        gc.collect()\n        \n        \n        train_sequences = np.load('train_sequences.npy')\n        #training_labels = np.load('training_labels.npy')\n        train_features = np.load('train_features.npy')\n\n        print('PREPARE FOR PYTORCH')\n        x_train = torch.tensor(train_sequences, dtype=torch.long).cuda()\n        y_train = torch.tensor(training_labels[:, np.newaxis], dtype=torch.float32).cuda()\n        x_train_feats = torch.tensor(train_features, dtype=torch.float32).cuda()\n        print('FINISH TRAIN PREMINILARY FOR PYTORCH')\n        x_train_data = torch.utils.data.TensorDataset(x_train, x_train_feats, y_train)\n        train = torch.utils.data.DataLoader(x_train_data, batch_size=batch_size, shuffle=True)\n        del x_train, x_train_feats, x_train_data, y_train\n        gc.collect()\n\n        del train_sequences, train_features\n        gc.collect()\n        # glove+fasttext-RNN\n        embedding_matrix = build_concatenate_embedding_matrix(embed_glove_index, 'wiki-news', one=False)\n        del embed_glove_index\n        gc.collect()\n        \n        \n        rnn_kwargs = {\n            'max_features': num_words,\n            'embedding_matrix': embedding_matrix,\n            'hidden_size': hidden_size,\n            'linear1_in': 543,\n        }\n        epochs = 3\n        model = CapsNet(**rnn_kwargs)\n        del embedding_matrix, rnn_kwargs\n        gc.collect()\n        model.cuda()\n        criterion = torch.nn.BCEWithLogitsLoss(reduction=\"sum\")\n        lr=0.003\n        for epoch in range(start_epoch, epochs):\n            optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n            train_model(train, model, criterion, optimizer, epoch, gpu=gpu)\n            lr=lr*0.8\n\n        preds = np.zeros((len(test_idx)))\n        preds = test_model(test, model, gpu=gpu)\n        preds = preds.ravel()\n        if split>0:\n            best_score, _ = f1_smart(testing_labels, preds)\n            print ('RNN-glove+fasttext :', best_score)   \n        np.save('preds_glove_fasttext.npy', preds)\n\n        del model, criterion, optimizer, preds\n        gc.collect()\n\n        x_train = {'sparse_data_one': scipy.sparse.load_npz('train_char_features.npz'),\n                'sparse_data_two': scipy.sparse.load_npz('train_name_bi.npz'),\n                'sparse_data_three': scipy.sparse.load_npz('train_word_features.npz'),\n                'feats': np.load('train_sparse_feats.npy'),\n            }\n\n        def sparseNN():   \n            sparse_data_one = Input( shape=[x_train[\"sparse_data_one\"].shape[1]], \n                dtype = 'float32',   sparse = True, name='sparse_data_one') \n            sparse_data_two = Input( shape=[x_train[\"sparse_data_two\"].shape[1]], \n                dtype = 'float32',   sparse = True, name='sparse_data_two') \n            sparse_data_three = Input( shape=[x_train[\"sparse_data_three\"].shape[1]], \n                dtype = 'float32',   sparse = True, name='sparse_data_three') \n\n            feats = Input(shape=[6], name=\"feats\")\n\n            x_one = Dense(200 , kernel_initializer=he_uniform(seed=0) )(sparse_data_one)    \n            x_one = PReLU()(x_one)\n\n            x_two = Dense(200 , kernel_initializer=he_uniform(seed=0) )(sparse_data_two)    \n            x_two = PReLU()(x_two)\n\n            x_three = Dense(200 , kernel_initializer=he_uniform(seed=0) )(sparse_data_three)    \n            x_three = PReLU()(x_three)\n\n            x = concatenate( [x_one, x_two, x_three, feats] )\n            x = Dense(200 , kernel_initializer=he_uniform(seed=0) )(x)\n            x = PReLU()(x)\n            x = Dense(100 , kernel_initializer=he_uniform(seed=0) )(x)\n            x = PReLU()(x)\n            x= Dense(1, activation='sigmoid')(x)\n            model = Model([sparse_data_one, sparse_data_two, sparse_data_three, feats], x)\n            optimizer = Adam(.0005)\n            model.compile(loss=\"binary_crossentropy\", optimizer=optimizer)\n            return model\n        BATCH_SIZE = 1024\n        epochs = 1\n        sparse_nn = sparseNN()\n        sparse_nn.fit(  x_train, training_labels, batch_size=BATCH_SIZE, epochs=1, verbose=1 )\n        del x_train, training_labels\n        gc.collect()\n        x_test = {'sparse_data_one': scipy.sparse.load_npz('test_char_features.npz'),\n                'sparse_data_two': scipy.sparse.load_npz('test_name_bi.npz'),\n                'sparse_data_three': scipy.sparse.load_npz('test_word_features.npz'),\n                'feats': np.load('test_sparse_feats.npy'),\n            }\n        preds = sparse_nn.predict(x_test)[:,0]\n        np.save('preds_sparsenn.npy', preds)\n    \nimport scipy\ndef do_sparses(dfs):\n    def is_chinese(x):\n        if re.search(u'[\\u4e00-\\u9fff]', x):\n            return 1\n        else: \n            return 0\n    def get_sentiment(df):\n        sid = SIA()\n        df['nltk'] = df['question_text'].apply(lambda x: sid.polarity_scores(x))\n        df['neg'] = df['nltk'].apply(lambda x: x['neg'])\n        df['neu'] = df['nltk'].apply(lambda x: x['neu'])\n        df['pos'] = df['nltk'].apply(lambda x: x['pos'])\n        df['compound'] = df['nltk'].apply(lambda x: x['compound'])     \n        df['chinese']= df['question_text'].apply(lambda x: is_chinese(x))\n    def create_count_features(df_data):\n        def lg(text):\n            text = [x for x in text.split() if x!='']\n            return len(text)\n        df_data['nb_words_description'] = df_data['question_text'].apply(lg).astype(np.uint16)\n        return df_data\n    train = pd.read_csv(dfs[0])\n    test = pd.read_csv(dfs[1])\n    train_target = train['target'].values\n    np.save('train_target.npy',train_target)\n    print('AHA IM HERE fitting new tfidf !')\n    TOKENIZER = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')\n    def tokenize(s):\n        return TOKENIZER.sub(r' \\1 ', s).split()\n    def lemmatize_sentence(sentence):\n        tokens=nltk.word_tokenize(sentence)\n        lemmatizer = WordNetLemmatizer()\n        lem = [lemmatizer.lemmatize(t) for t in tokens]\n        return \" \".join(lem)\n    train['question_text_lemma'] = train['question_text'].apply(lambda x: lemmatize_sentence(x))\n    tfidf_vectorizer = TfidfVectorizer(\n        ngram_range=(1,3),\n        tokenizer=tokenize,\n        min_df=3,\n        max_df=0.9,\n        strip_accents='unicode',\n        use_idf=True,\n        smooth_idf=True,\n        sublinear_tf=True,\n        max_features = 100000\n    ).fit(train['question_text_lemma'])\n    print('FINISH fitting new tfidf !')\n    X_tfidf = tfidf_vectorizer.transform(train['question_text_lemma'])\n    scipy.sparse.save_npz('X_tfidf.npz', X_tfidf)\n    del X_tfidf, train['question_text_lemma']\n    gc.collect()\n    print('FINISH transforming new tfidf in train!')\n    test['question_text_lemma'] = test['question_text'].apply(lambda x: lemmatize_sentence(x))\n    X_tfidf_test = tfidf_vectorizer.transform(test['question_text_lemma'])\n    scipy.sparse.save_npz('X_tfidf_test.npz', X_tfidf_test)\n    del X_tfidf_test, tfidf_vectorizer, test['question_text_lemma']\n    gc.collect()\n    \n    \n    char_vectorizer = TfidfVectorizer(\n        sublinear_tf=True,\n        strip_accents='unicode',\n        analyzer='char',\n        stop_words='english',\n        ngram_range=(2, 4),\n        max_features=30000)\n    char_vectorizer.fit(train['question_text'])\n    print('Char TFIDF 1/3')\n    train_char_features = char_vectorizer.transform(train['question_text'])\n    scipy.sparse.save_npz('train_char_features.npz', train_char_features)\n    del train_char_features\n    gc.collect()\n    print('Char TFIDF 2/3')\n    test_char_features = char_vectorizer.transform(test['question_text'])\n    scipy.sparse.save_npz('test_char_features.npz', test_char_features)\n    del char_vectorizer, test_char_features\n    gc.collect()\n    print('Char TFIDF 3/3')\n\n    word_vectorizer = TfidfVectorizer(\n        sublinear_tf=True,\n        strip_accents='unicode',\n        analyzer='word',\n        token_pattern=r'\\w{1,}',\n        ngram_range=(1, 2),\n        max_features=30000)\n    word_vectorizer.fit(train['question_text'])\n    print('Word TFIDF 1/3')\n    train_word_features = word_vectorizer.transform(train['question_text'])\n    scipy.sparse.save_npz('train_word_features.npz', train_word_features)\n    del train_word_features\n    gc.collect()\n    print('Word TFIDF 2/3')\n    test_word_features = word_vectorizer.transform(test['question_text'])\n    scipy.sparse.save_npz('test_word_features.npz', test_word_features)\n    del word_vectorizer, test_word_features\n    gc.collect()\n    print('Word TFIDF 3/3')   \n    \n    get_sentiment(train)\n    train = create_count_features(train)\n    np.save('train_sparse_feats.npy',train[['chinese','neg','neu','pos','compound','nb_words_description']].values)\n    train.drop(['chinese','neg','neu','pos','compound','nb_words_description'],axis=1, inplace=True)\n    \n    get_sentiment(test)\n    test = create_count_features(test)\n    np.save('test_sparse_feats.npy',test[['chinese','neg','neu','pos','compound','nb_words_description']].values)\n    test.drop(['chinese','neg','neu','pos','compound','nb_words_description'],axis=1, inplace=True)\n    \n    print('START LOADING TFIDF')\n    train_target = np.load('train_target.npy')\n    n_folds = 5\n    test_pred_LR = 0\n    oof_pred_LR = np.zeros([train_target.shape[0],])\n    X_tfidf = scipy.sparse.load_npz('X_tfidf.npz')\n    skf = list(StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed).split(X_tfidf, train_target))\n    del train_target,  X_tfidf\n    gc.collect()\n    for i, (train_index, val_index) in tqdm(enumerate(skf)):\n        X_tfidf = scipy.sparse.load_npz('X_tfidf.npz')\n        x_train, x_val = X_tfidf[list(train_index)], X_tfidf[list(val_index)]\n        del X_tfidf\n        gc.collect()\n        train_target = np.load('train_target.npy')\n        y_train, y_val = train_target[train_index], train_target[val_index]\n        del train_target\n        gc.collect()\n        #classifier = LogisticRegression(C=5, solver='sag')\n        classifier = Ridge()\n        classifier.fit(x_train, y_train)\n        del x_train, y_train\n        gc.collect()\n        val_preds = classifier.predict(x_val)\n        X_tfidf_test = scipy.sparse.load_npz('X_tfidf_test.npz')\n        preds = classifier.predict(X_tfidf_test)\n        del X_tfidf_test,x_val\n        gc.collect()\n        test_pred_LR += 0.2*preds\n        oof_pred_LR[val_index] = val_preds\n        print(f1_score(y_val, val_preds > 0.26))\n        del classifier, preds, val_preds\n        gc.collect()\n    np.save('oof_pred_LR.npy',oof_pred_LR)\n    del oof_pred_LR\n    gc.collect()\n    np.save('test_pred_LR.npy',test_pred_LR)\n    del test_pred_LR\n    gc.collect()\n    \n    print('END Parallel')\n    \n    wordnet_lemmatizer = WordNetLemmatizer()\n    def word_count(text, dc):\n        text = set( text.split(' ') ) \n        for w in text:\n            dc[w]+=1\n    def remove_low_freq(text, dc):\n        return ' '.join( [w for w in text.split() if w in dc] )\n    stop_words = set(stopwords.words('english'))\n    def create_bigrams(text):\n        #try:\n        text = np.unique( [ wordnet_lemmatizer.lemmatize(w) for w in text.split() if w not in stop_words ] )\n        lst_bi = []\n        for combo in combinations(text, 2):\n            cb1=combo[0]+combo[1]\n            cb2=combo[1]+combo[0]\n            in_dict=False\n            if cb1 in word_count_dict_one:\n                new_word = cb1\n                in_dict=True\n            if cb2 in word_count_dict_one:\n                new_word = cb2\n                in_dict=True\n            if not in_dict:\n                new_word = combo[0]+'___'+combo[1]\n            if len(cb1)>=0:\n                lst_bi.append(new_word)\n        return ' '.join( lst_bi )\n        #except:\n        #    return ' '\n    def create_bigrams_df(df):\n        return df.apply( create_bigrams )\n    cores = 2\n    max_text_length=60###################\n    min_df_one=5\n    min_df_bi=5\n    def parallelize_dataframe(df, func):\n        df_split = np.array_split(df, cores)\n        pool = Pool(cores)\n        df = pd.concat(pool.map(func, df_split))\n        pool.close()\n        pool.join()\n        return df\n\n    word_count_dict_one = defaultdict(np.uint32)\n\n    train['question_text'].apply(lambda x : word_count(x, word_count_dict_one) )\n    rare_words = [key for key in word_count_dict_one if  word_count_dict_one[key]<min_df_one ]\n    for key in rare_words :\n        word_count_dict_one.pop(key, None)\n\n    train['question_text']      = train['question_text'].apply( lambda x : remove_low_freq(x, word_count_dict_one) )\n    word_count_dict_one=dict(word_count_dict_one)\n\n\n    start_time = time.time()\n    word_count_dict_bi=defaultdict(np.uint32)\n    def word_count_bi(text):\n        text =  text.split(' ') \n        for w in text:\n            word_count_dict_bi[w]+=1\n\n    train['question_textbi'] = train['question_text'].apply( lambda x : ' '.join( x.split()[5:] ))\n    test['question_textbi'] = test['question_text'].apply( lambda x : ' '.join( x.split()[5:] ))\n\n    train['name_bi']  = create_bigrams_df(train['question_textbi'])\n    test['name_bi']  = create_bigrams_df(test['question_textbi'])\n    train.drop('question_textbi',axis=1, inplace=True)\n    test.drop('question_textbi',axis=1, inplace=True)\n    gc.collect()\n    train['name_bi'].apply(word_count_bi )\n    rare_words = [key for key in word_count_dict_bi if  word_count_dict_bi[key]<min_df_bi ]\n    for key in rare_words :\n        word_count_dict_bi.pop(key, None)\n    train['name_bi']      = train['name_bi'].apply( lambda x : remove_low_freq(x, word_count_dict_bi) )\n    test['name_bi']      = test['name_bi'].apply( lambda x : remove_low_freq(x, word_count_dict_bi) )\n\n    print('[{}] Finished CREATING BIGRAMS...'.format(time.time() - start_time))\n\n    #####################################\n\n    start_time = time.time()\n    word_count_dict_bi = dict(word_count_dict_bi)\n    vocabulary_one = word_count_dict_one.copy()\n    vocabulary_bi = word_count_dict_bi.copy()\n    for dc in [vocabulary_one,  vocabulary_bi]:\n        cpt=0\n        for key in dc:\n            dc[key]=cpt\n            cpt+=1\n    print('[{}] Finished CREATING VOCABULARY ...'.format(time.time() - start_time))\n\n    def tokenize(text):\n        return [w for w in text.split()]\n    start_time = time.time()\n    vect_item_one            = CountVectorizer(vocabulary= vocabulary_one,   dtype=np.uint8, \n                                               tokenizer=tokenize, binary=True ) \n    train_item_one  = vect_item_one.fit_transform( train['question_text']  )\n    scipy.sparse.save_npz('train_item_one.npz', train_item_one)\n    \n    \n    \n    del train_item_one, vocabulary_one\n    gc.collect()\n    test_item_one = vect_item_one.transform(     test['question_text']  )\n    scipy.sparse.save_npz('test_item_one.npz', test_item_one)\n    del test_item_one\n    gc.collect()\n    print('[{}] Finished Vectorizing Onegram Item Description'.format(time.time() - start_time))\n    start_time = time.time()\n    vect_name_bi           = CountVectorizer(vocabulary= vocabulary_bi,   dtype=np.uint8, \n                                         tokenizer=tokenize, binary=True ) \n    train_name_bi  = vect_name_bi.fit_transform( train['name_bi']  )\n    scipy.sparse.save_npz('train_name_bi.npz', train_name_bi)\n    del train['name_bi'], train_name_bi, vocabulary_bi\n    gc.collect()\n    print('[{}] Finished Vectorizing BiGram Name'.format(time.time() - start_time))\n    \n    test_name_bi = vect_name_bi.transform(     test['name_bi']  )\n    scipy.sparse.save_npz('test_name_bi.npz', test_name_bi)\n    del test_name_bi, word_count_dict_one, rare_words\n    #time.sleep(10*60)\n    \n\n#     X_tfidf = scipy.sparse.load_npz('X_tfidf.npz')\n#     #classifier = LogisticRegression(C=5, solver='sag')\n#     print('LOADED TFIDF')\n#     classifier = Ridge()\n#     classifier.fit(X_tfidf, train_target)\n#     del X_tfidf\n#     gc.collect()\n#     print('fitted tfidf')\n#     X_tfidf_test = scipy.sparse.load_npz('X_tfidf_test.npz')\n#     test_pred_LR = classifier.predict(X_tfidf_test)\n#     del X_tfidf_test\n#     gc.collect()\n#     print('Predicted tfidf')\n#     np.save(test_pred_LR, 'test_pred_LR.npy')\n#     del test_pred_LR\n#     gc.collect()\n    \n    print('END Parallel')\n\n    \n\nfrom multiprocessing import Pool\ndf_two = [train, test, True]\ntest_idx = list(test.qid.values)\ndel train, test\ngc.collect()\n\n\npool = Pool(1)\npool.apply_async(do_sparses, [[f'{FILE_DIR}/train.csv', f'{FILE_DIR}/test.csv']])\ndo_rnns(df_two)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d6d384a904cfa4923c9e3e7dc2fba1b4c51a25f"},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d6a2c0fc1ded74913e4de883b5affa026e7f69f"},"cell_type":"code","source":"models_predictions = defaultdict(list)\nmodels_predictions['RNN2-googlenews'] += list(np.load('preds_googlenews.npy'))  \nmodels_predictions['RNN2-paragram_google'] += list(np.load('preds_paragram_google.npy')) \nmodels_predictions['RNN2-glove'] += list(np.load('preds_glove.npy'))  \nmodels_predictions['RNN2-glove_fasttext'] += list(np.load('preds_glove_fasttext.npy'))\nmodels_predictions['RNN2-sparsenn'] += list(np.load('preds_sparsenn.npy'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c03ad97546e30890b7effbcce6719762eeacb59"},"cell_type":"code","source":"submission_preds_df = pd.DataFrame(models_predictions)\nsubmission_preds_df.corr()\nmysubmission=pd.DataFrame()\nmysubmission['qid'] = test_idx\npreds = submission_preds_df.mean(axis=1)\n#preds = submission_preds_df['RNN2-googlenews']*0.1 + submission_preds_df['RNN2-paragram_google']*0.2 + submission_preds_df['RNN2-glove']*0.35 + \\\n#        submission_preds_df['RNN2-glove_fasttext']*0.2 + submission_preds_df['RNN2-sparsenn']*0.15\nmysubmission['prediction'] = preds > 0.44190952125936747 #0.7028983812108168 CV\nmysubmission.to_csv('submission.csv', index=False)\nprint (mysubmission.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06559326cc4c668df9bf11ee5003622df429f76b"},"cell_type":"code","source":"mysubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccc5355f5a78d2fb9f0c248e0215bf03f9e3fa09"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5df062fd438a25acf7080906e435a5becff426b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}