{"cells":[{"metadata":{"trusted":true,"_uuid":"ef658033c7be825232d6ad40374f45537ba4c36f"},"cell_type":"code","source":"# Baseline Torch - 03.65","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"014f67cba3b8afda767743e646aebb0b1e1a7eca","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def f1():\n    # Just to initialize Global Variables\n    return\nhidden_size = 60","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true,"_uuid":"a46c2af9d3276ab419192aeb1f6ce05f99c5b7d5"},"cell_type":"code","source":"import time\nimport random\ndef set_all_parameters():\n    \n    global epoc_used\n    global mn             # Model Number - For essembling\n    global batch_size\n    global patience\n    global nsplits\n    global stop_split\n    global emb\n    global hidden_size\n    global linear_size\n    global rate_drop_hidden\n    global rate_drop_linear\n    global rate_drop_embedding\n    global loss\n    global act\n    global opt\n    global sch\n    global es_mon\n    global es_mode\n    global Trainable\n    global lr\n    global method\n    global model_list\n    global maxlen\n    global max_vocab\n    global pretext_proc\n    global bstart\n    global Full_Train_Validation\n    global Weight_Method\n    global SEED\n    global embed_size\n    global embedding_matrix\n    global use_pretrained_embedding\n    global emb_out_of_vocab\n    global include_extra_features\n    global build_vocab\n    global debug\n    global subsampling\n    global upsampling\n    global split_to_run\n    debug =False\n    subsampling=0\n    upsampling=False\n\n    # Start time - \n    bstart = time.time()\n    #Possible embeddings [\"Glove\",\"Paragram\",\"Google\",\"Wiki\",\"Combined\",\"Concatenated\"]**\n    ## Early Stop Monitor and Early Stop Mode\n    ## Early Stop Monitor Options es_mon = \"val_f1\" es_mode=\"max\"  for met=[f1]\n    #                             es_mon = \"val_acc\" es_mode=\"max\" for met=[\"accuracy\"]   \n    #                             es_mon = \"val_loss\"\n\n    ###  4 Splits = Validation Split=0.25%\n    ###  5 Splits = Validation Split=0.20 %\n    ### 10 Splits = Validation Split=0.10 %\n    ### 12 Splits = Validation Split=0.0833 %\n    ns=4  # Standar Number of Splits\n    ### For Stratified k-fold - Validation -> nsplits=stop_splits\n    #model_list.append([model_number, weight, model_eval, embeddings,epochs,patience,batch_size, size hidden, size dense, drop hidden, drop dense, drop spatial, loss,optimizer,metric])\n\n    model_list=[]\n\n    ### Embeddings Gl-Glove, P-Paragram, W-Wiki, Go-Google, Q-Quora\n    #model_list.append([\"6\",1,\"X\",\"Combined3\",5,1, 512,5,5,0,60,16,0.1,0.1,0.1,'binary_crossentropy','adam',[\"None\"],'val_loss','min'])\n    model_list.append([\"6\",1,\"X\",\"Combined3\",5,1, 512,5,5,0,60,16,0.1,0.1,0.1,'binary_crossentropy','adam',[\"clr\"],'val_loss','min'])\n    #   model_list.append([\"0\",1,\"X\",\"Combined\",8,1,1024,4,1,2,60,16,0.1,0.1,0.1,'binary_crossentropy','adam',[\"est\"],'val_acc','max'])\n #   model_list.append([\"1\",1,\"X\",\"Combined\",5,2, 512,4,1,3,60,16,0.1,0.1,0.1,'binary_crossentropy','adam2',[\"clr\"],'val_loss','min'])\n #   model_list.append([\"3\",1,\"X\",\"Combined\",6,1,1024,4,1,4,60,12,0.1,0.1,0.1,'binary_crossentropy','adam2',[\"clr\"],'val_loss','min'])\n   # Weight_Method=\"Score\"\n    Weight_Method=\"Manual\"\n    ###\n    ###  Load Train,Val and Test \n    ###\n    maxlen = 70 # Maximum Sequence Size \n    max_vocab = 120000 # Maximum Number of Words in Dictionary\n    pretext_proc=True\n    SEED = 1029\n    print (SEED)\n    ###\n    build_vocab = \"both\"   ## Other Values train_only or test_only\n    ###  Embeddings\n    use_pretrained_embedding = True\n    emb_out_of_vocab=\"specific\" # other values:  mean or specific\n    Trainable=False  # Embedding Layers trainable(True) or not(False)\n    ### Include Meta Features in the final model\n    include_extra_features=False\n    ### Run Valitation in the Full Train Dataset or Just in Stratified Validation Set\n    Full_Train_Validation=False\nset_all_parameters()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"**Import Libraries**"},{"metadata":{"trusted":true,"_uuid":"b3feca02384cc7e6b6e7043453d0615f810ef05a","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from IPython.display import display, HTML\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\n\n# Any results you write to the current directory are saved as output.\nfrom sklearn.model_selection import train_test_split,StratifiedKFold\nfrom tqdm import tqdm, tqdm_notebook\nimport math\nimport re\nimport random\nfrom datetime import timedelta, datetime\nimport time\n\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nimport colorama\nfrom colorama import Fore\n\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer\nfrom string import punctuation\n\n\nfrom sklearn import metrics\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import precision_recall_curve, auc\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import StandardScaler\n\nfrom gensim.models import KeyedVectors\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Flatten, Dense, Input, LSTM, Embedding, Dropout, Activation, SpatialDropout1D, Reshape, Concatenate\nfrom keras.layers.merge import concatenate\nfrom keras.models import Model\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, Callback\nfrom keras.layers import Bidirectional, GlobalMaxPool1D,GlobalMaxPooling1D,GlobalAveragePooling1D ,Conv1D, MaxPooling1D, GRU,CuDNNLSTM,CuDNNGRU, Reshape, MaxPooling1D,AveragePooling1D\nfrom keras.optimizers import RMSprop, SGD, Nadam, Adamax, Adam\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\n#from keras import initializations\nfrom keras import initializers, regularizers, constraints\nfrom keras.layers import Conv2D, MaxPool2D\nimport keras.backend as K\n%matplotlib inline\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\nimport torch as t\nimport torch.nn.functional as F\nfrom torch.optim.optimizer import Optimizer\nfrom torch import optim\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48a8000b527e7c48f9a9509e5ea1dd948cfe4a88"},"cell_type":"markdown","source":"**Seed_Torch**"},{"metadata":{"trusted":true,"_uuid":"c2dc0914afe4e13ab00472a20210d25130065b30","_kg_hide-input":true},"cell_type":"code","source":"def seed_torch(seed=1029):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_torch(SEED)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cc869c3d63fad65554af06bede87874480094247"},"cell_type":"markdown","source":"**Text Functions**"},{"metadata":{"trusted":true,"_uuid":"980cb40feddb64959b3f3b39d06db5bac0ede20f","_kg_hide-input":true},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75e1fcc06a41c58f3dc0fd78aeb9db0541d3dca7"},"cell_type":"markdown","source":"**Add Extra Features**"},{"metadata":{"trusted":true,"_uuid":"06cc54fd79953aace8b6638fcd63b898b0831750","_kg_hide-input":true},"cell_type":"code","source":"\ndef add_features(df):\n    \n    df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))\n    df['total_length'] = df['question_text'].progress_apply(len)\n    df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),\n                                axis=1)\n    df['num_words'] = df.question_text.str.count('\\S+')\n    df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))\n    df['words_vs_unique'] = df['num_unique_words'] / df['num_words']  \n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba9829226e300f9653f3a06d0a115dce728a3a86"},"cell_type":"markdown","source":"**Load and Process - Train and Test - Datasets**"},{"metadata":{"trusted":true,"_uuid":"32d13453390bbe904f1b394f80b1ff039ce30962","_kg_hide-input":true},"cell_type":"code","source":"import nltk \n# Define a function to split a review into parsed sentences\ndef review_to_sentences( review ):\n    # Function to split a review into parsed sentences. Returns a \n    # list of sentences, where each sentence is a list of words\n    #\n    # 1. Use the NLTK tokenizer to split the paragraph into sentences\n    raw_sentences = nltk.tokenize.sent_tokenize(review.strip())\n    #\n    # 2. Loop over each sentence\n    sentences = []\n    for raw_sentence in raw_sentences:\n        # If a sentence is empty, skip it\n        if len(raw_sentence) > 0:\n            # Otherwise, call review_to_wordlist to get a list of words\n            sentences.append( nltk.word_tokenize(raw_sentence))\n    #\n    # Return the list of sentences (each sentence is a list of words,\n    # so this returns a list of lists\n    return sentences\n\ndef load_and_process(pretext_proc=True):\n\n    global max_vocab\n    global subsampling\n    global upsampling\n    \n    tqdm.pandas()\n    if debug:\n        train_df = pd.read_csv(\"../input/train.csv\")[:10000]\n        test_df = pd.read_csv(\"../input/test.csv\")[:2000]\n    else:\n        train_df = pd.read_csv(\"../input/train.csv\")\n        test_df = pd.read_csv(\"../input/test.csv\")\n        \n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    if (subsampling > 0): \n        train_sincere=train_df[  (train_df['target'] == 0) ]\n        train_insincere=train_df[  (train_df['target'] == 1) ]\n        sincere= int(train_sincere['target'].count())\n        insincere= int(train_insincere['target'].count())\n        remove_n = int(sincere * subsampling /100)\n        if (debug):\n            print (\"Sincere:\",sincere)\n            print (\"Insincere:\",insincere)\n        drop_indices = np.random.choice(train_sincere.index, remove_n, replace=False)\n        train_sincere= train_sincere.drop(drop_indices)     \n        train_df = pd.concat([train_sincere,train_insincere])\n    if (upsampling):\n        ### \n        train_sincere=train_df[  (train_df['target'] == 0) ]\n        train_insincere=train_df[  (train_df['target'] == 1) ]\n        sincere= int(train_sincere['target'].count())\n        insincere= int(train_insincere['target'].count())\n        if (debug):\n            remove_sincere = int(sincere)-300\n            remove_insincere = int(insincere)-50\n        else:\n            remove_sincere = int(sincere)-298530\n            remove_insincere = int(insincere)-22470\n        drop_indices = np.random.choice(train_sincere.index, remove_sincere, replace=False)\n        train_sincere= train_sincere.drop(drop_indices)     \n        drop_indices = np.random.choice(train_insincere.index, remove_insincere, replace=False)\n        train_insincere= train_insincere.drop(drop_indices) \n        add_df = pd.concat([train_sincere,train_insincere])\n        add_df.drop(['target'], axis=1)\n        test_df= pd.concat([test_df,add_df])\n        \n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n      \n    if (pretext_proc==True):\n        # lower\n        train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: x.lower())\n        test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: x.lower())\n\n        # Clean the text\n        train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n        test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n\n        # Clean numbers\n        train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n        test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n\n        # Clean speelings\n        train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n        test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n\n         # Clean Contractions\n        #train[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: clean_contractions(x,contraction_mapping))\n        #test[\"question_text\"] = test[\"question_text\"].progress_apply(lambda x: clean_contractions(x,contraction_mapping))\n    \n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n    if debug:\n        max_vocab=10000\n\n    if load_Quora:\n\n        #\n        # Calculate Quoras Word2vrc\n        #\n        all_questions = list(train_X) + list(test_X) # Initialize an empty list of sentences\n        questions=[]\n        print (\"Parsing questions from All Questions\")\n        for review in all_questions:\n            questions += review_to_sentences(review)\n\n        # Import the built-in logging module and configure it so that Word2Vec \n        # creates nice output messages\n        import multiprocessing\n        import logging\n\n        logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',\\\n            level=logging.INFO)\n\n        # Set values for various parameters\n        #sg defines the training algorithm. By default (sg=0), CBOW is used. Otherwise (sg=1), skip-gram is employed\n        dimension = 300   # Word vector dimensionality                      \n        min_word_count = 1    # Minimum word count                        \n        num_workers = 2    # Number of threads to run in parallel\n        context = 5       # Context window size                                                                                    \n        downsampling = 1e-2   # Downsample setting for frequent words\n\n        # Initialize and train the model (this will take some time)\n        from gensim.models import word2vec\n        print (\"Training model...\")\n        model = word2vec.Word2Vec(questions, \\\n                                  sg=1, \\\n                                  size=dimension, \\\n                                  window = context,   \\\n                                  workers=num_workers, \\\n                                  min_count = min_word_count, \\\n                                  sample = downsampling)\n\n        model_name = \"../quora_clean_300d.bin\"\n        model.wv.save_word2vec_format(model_name, binary=True)     \n        \n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words= max_vocab )\n    \n    vocab_list= list(train_X)+list(test_X)\n    if (build_vocab==\"train_only\"):\n        vocab_list = list(train_X)\n    if (build_vocab==\"test_only\"):\n        vocab_list = list(test_X)\n\n    tokenizer.fit_on_texts(vocab_list)\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    \n\n\n    if (include_extra_features):\n            ###################### Add Features ###############################\n            train = add_features(train_df)\n            test = add_features(test_df)\n            \n            ###\n            ### Just Two Features are selected\n            ###\n            features = train[['caps_vs_length', 'words_vs_unique']].fillna(0)\n            test_features = test[['caps_vs_length', 'words_vs_unique']].fillna(0)\n\n            ss = StandardScaler()\n            ss.fit(np.vstack((features, test_features)))\n            features = ss.transform(features)\n            test_features = ss.transform(test_features)\n            if debug:\n                print (features.shape)\n                print (test_features.shape)\n    else:\n        features=None\n        test_features=None    \n    \n    #shuffling the data\n    np.random.seed(SEED)\n    trn_idx = np.random.permutation(len(train_X))\n\n    train_X = train_X[trn_idx]\n    train_y = train_y[trn_idx]\n    if (include_extra_features):\n        features=features[trn_idx]\n        \n    if (max_vocab==None):\n        max_vocab = len(tokenizer.word_index) +1\n    print (\"Vocab Size:\", max_vocab)\n    return  train_X, test_X, train_y, test_df, tokenizer.word_index,  features, test_features","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ff74f9776e7692fe0f483b27e4f03cce7bf7f3c"},"cell_type":"markdown","source":"### Cyclic CLR\nCode taken from https://www.kaggle.com/dannykliu/lstm-with-attention-clr-in-pytorch"},{"metadata":{"trusted":true,"_uuid":"613e87fa74cabd3bd61c338511d39839c54a7320","_kg_hide-input":true},"cell_type":"code","source":"# code inspired from: https://github.com/anandsaha/pytorch.cyclic.learning.rate/blob/master/cls.py\nclass CyclicLR(object):\n    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n                 step_size=2000, mode='triangular', gamma=1.,\n                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n            if len(base_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} base_lr, got {}\".format(\n                    len(optimizer.param_groups), len(base_lr)))\n            self.base_lrs = list(base_lr)\n        else:\n            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n\n        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n            if len(max_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} max_lr, got {}\".format(\n                    len(optimizer.param_groups), len(max_lr)))\n            self.max_lrs = list(max_lr)\n        else:\n            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n\n        self.step_size = step_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n\n        self.batch_step(last_batch_iteration + 1)\n        self.last_batch_iteration = last_batch_iteration\n\n    def batch_step(self, batch_iteration=None):\n        if batch_iteration is None:\n            batch_iteration = self.last_batch_iteration + 1\n        self.last_batch_iteration = batch_iteration\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr\n\n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        step_size = float(self.step_size)\n        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n\n        lrs = []\n        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n        for param_group, base_lr, max_lr in param_lrs:\n            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n            lrs.append(lr)\n        return lrs\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"def809d84841df8e99a28f432f355a84ae0f6351"},"cell_type":"markdown","source":"** Model Layers**"},{"metadata":{"trusted":true,"_uuid":"233c6160c3290066b1d2b7b482837f98effea39e","_kg_hide-input":true},"cell_type":"code","source":"Routings = 4 #5\nNum_capsule = 5\nDim_capsule = 5#16\ndropout_p = 0.25\nLR = 0.001\nT_epsilon = 1e-7\nnum_classes = 30\n\n\nclass Embed_Layer(nn.Module):\n    def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=300):\n        super(Embed_Layer, self).__init__()\n        self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)\n        if use_pretrained_embedding:\n            # self.encoder.weight.data.copy_(t.from_numpy(np.load(embedding_path))) # 方法一，加载np.save的npy文件\n            self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))  # 方法二\n\n    def forward(self, x, dropout_p=0.25):\n        return nn.Dropout(p=dropout_p)(self.encoder(x))\n\n\nclass GRU_Layer(nn.Module):\n    def __init__(self):\n        super(GRU_Layer, self).__init__()\n        self.gru = nn.GRU(input_size=300,\n                          hidden_size=hidden_size,\n                          bidirectional=True)\n        '''\n        自己修改GRU里面的激活函数及加dropout和recurrent_dropout\n        如果要使用，把rnn_revised import进来，但好像是使用cpu跑的，比较慢\n       '''\n        # # if you uncomment /*from rnn_revised import * */, uncomment following code aswell\n        # self.gru = RNNHardSigmoid('GRU', input_size=300,\n        #                           hidden_size=hidden_size,\n        #                           bidirectional=True)\n\n    # 这步很关键，需要像keras一样用glorot_uniform和orthogonal_uniform初始化参数\n    def init_weights(self):\n        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n        for k in ih:\n            nn.init.xavier_uniform_(k)\n        for k in hh:\n            nn.init.orthogonal_(k)\n        for k in b:\n            nn.init.constant_(k, 0)\n\n    def forward(self, x):\n        return self.gru(x)\n\n\n# core caps_layer with squash func\nclass Caps_Layer(nn.Module):\n    global hidden_size \n    def __init__(self, input_dim_capsule=hidden_size * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \\\n                 routings=Routings, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Caps_Layer, self).__init__(**kwargs)\n\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size  # 暂时没用到\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = self.squash\n        else:\n            self.activation = nn.ReLU(inplace=True)\n\n        if self.share_weights:\n            self.W = nn.Parameter(\n                nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))\n        else:\n            self.W = nn.Parameter(\n                t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))  # 64即batch_size\n\n    def forward(self, x):\n\n        if self.share_weights:\n            u_hat_vecs = t.matmul(x, self.W)\n        else:\n            print('add later')\n\n        batch_size = x.size(0)\n        input_num_capsule = x.size(1)\n        u_hat_vecs = u_hat_vecs.view((batch_size, input_num_capsule,\n                                      self.num_capsule, self.dim_capsule))\n        u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)  # 转成(batch_size,num_capsule,input_num_capsule,dim_capsule)\n        b = t.zeros_like(u_hat_vecs[:, :, :, 0])  # (batch_size,num_capsule,input_num_capsule)\n\n        for i in range(self.routings):\n            b = b.permute(0, 2, 1)\n            c = F.softmax(b, dim=2)\n            c = c.permute(0, 2, 1)\n            b = b.permute(0, 2, 1)\n            outputs = self.activation(t.einsum('bij,bijk->bik', (c, u_hat_vecs)))  # batch matrix multiplication\n            # outputs shape (batch_size, num_capsule, dim_capsule)\n            if i < self.routings - 1:\n                b = t.einsum('bik,bijk->bij', (outputs, u_hat_vecs))  # batch matrix multiplication\n        return outputs  # (batch_size, num_capsule, dim_capsule)\n\n    # text version of squash, slight different from original one\n    def squash(self, x, axis=-1):\n        s_squared_norm = (x ** 2).sum(axis, keepdim=True)\n        scale = t.sqrt(s_squared_norm + T_epsilon)\n        return x / scale\n    \nclass Capsule_Main(nn.Module):\n    def __init__(self, embedding_matrix=None, vocab_size=None):\n        super(Capsule_Main, self).__init__()\n        self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)\n        self.gru_layer = GRU_Layer()\n        # 【重要】初始化GRU权重操作，这一步非常关键，acc上升到0.98，如果用默认的uniform初始化则acc一直在0.5左右\n        self.gru_layer.init_weights()\n        self.caps_layer = Caps_Layer()\n        self.dense_layer = Dense_Layer()\n\n    def forward(self, content):\n        content1 = self.embed_layer(content)\n        content2, _ = self.gru_layer(\n            content1)  # 这个输出是个tuple，一个output(seq_len, batch_size, num_directions * hidden_size)，一个hn\n        content3 = self.caps_layer(content2)\n        output = self.dense_layer(content3)\n        return output\n    \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"beb6daac65cf187661115680e32ec44d2fb5913e"},"cell_type":"markdown","source":"**Attention - Keras Layer**"},{"metadata":{"trusted":true,"_uuid":"c31838d836715b1fc32264218125b56add66e3d4","_kg_hide-input":true},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2e6cae395028e37f3fe38b85c41b593789f3633"},"cell_type":"markdown","source":"** Neural Net - Embeddings - LSTM, GRU, Attention **"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"353f297721962ffe8b6e8df5d92b9dbe1b04263d"},"cell_type":"code","source":"class NeuralNet(nn.Module):\n    global include_extra_features\n    def __init__(self):\n        \n        super(NeuralNet, self).__init__()\n  \n        self.embedding = nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1])\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(rate_drop_embedding)\n        self.lstm = nn.LSTM(embedding_matrix.shape[1], hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)\n        \n        self.lstm_attention = Attention(hidden_size*2, maxlen)\n        self.gru_attention = Attention(hidden_size*2, maxlen)\n\n        if (include_extra_features==True):\n            self.linear = nn.Linear(hidden_size*8+features.shape[1], linear_size) #643:80 - 483:60 - 323:40\n        else:\n            self.linear = nn.Linear(hidden_size*8, linear_size) #643:80 - 483:60 - 323:40\n\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(rate_drop_linear)\n        self.out = nn.Linear(linear_size, 1)\n        \n    def forward(self, x):\n        if (include_extra_features==True):\n            h_embedding = self.embedding(x[0])\n        else:\n            h_embedding = self.embedding(x) \n        #h_embedding = self.embedding(x)\n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        \n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        avg_pool = torch.mean(h_gru, 1)\n        max_pool, _ = torch.max(h_gru, 1)\n        if (include_extra_features==True):\n            f = torch.tensor(x[1], dtype=torch.float).cuda()\n            conc = torch.cat((h_lstm_atten, h_gru_atten, avg_pool, max_pool,f), 1)\n        else:\n            conc = torch.cat((h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)\n            \n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d612b1f7860d27e43181793eb4e803a50790bfa1"},"cell_type":"markdown","source":"** Neural Net 1 - Embedding, LSTM, GRU, Attention, Caps - Extra Features **"},{"metadata":{"trusted":true,"_uuid":"2e126f1345f474f4a0edae8fc12b26fa30a06b4f","_kg_hide-input":true},"cell_type":"code","source":"class NeuralNet1(nn.Module):\n    global include_extra_features\n    \n    def __init__(self):\n        super(NeuralNet1, self).__init__()\n        \n        self.embedding = nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1])\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = Trainable\n        \n        self.embedding_dropout = nn.Dropout2d(rate_drop_embedding)\n        self.lstm = nn.LSTM( embedding_matrix.shape[1], hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n\n        self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n\n        self.lstm_attention = Attention(hidden_size * 2, maxlen)\n        self.gru_attention = Attention(hidden_size * 2, maxlen)\n        \n        if (include_extra_features==True):\n            self.linear = nn.Linear(hidden_size*8+1+features.shape[1], linear_size) #643:80 - 483:60 - 323:40\n        else:\n            self.linear = nn.Linear(hidden_size*8+1, linear_size) #643:80 - 483:60 - 323:40\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(rate_drop_linear)\n        self.fc = nn.Linear(linear_size**2,linear_size)\n        self.out = nn.Linear(linear_size, 1)\n        self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)\n        self.caps_layer = Caps_Layer()\n    \n    def forward(self, x):\n        \n#         Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x)\n        if (include_extra_features==True):\n            h_embedding = self.embedding(x[0])\n        else:\n            h_embedding = self.embedding(x) \n            \n        h_embedding = torch.squeeze(\n            self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n\n        ##Attention Layer\n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        # global average pooling\n        avg_pool = torch.mean(h_gru, 1)\n        # global max pooling\n        max_pool, _ = torch.max(h_gru, 1)\n\n     \n        ##Capsule Layer        \n        content3 = self.caps_layer(h_gru)\n        content3 = self.dropout(content3)\n        batch_size1 = content3.size(0)\n        content3 = content3.view(batch_size1, -1)\n        content3 = self.relu(self.lincaps(content3))        \n        \n        if (include_extra_features==True):\n            f = torch.tensor(x[1], dtype=torch.float).cuda()\n            conc = torch.cat((h_lstm_atten, h_gru_atten,content3, avg_pool, max_pool,f), 1)\n        else:\n            conc = torch.cat((h_lstm_atten, h_gru_atten,content3, avg_pool, max_pool), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"120e09765367b5a0da4324c435f6001cd8b69500"},"cell_type":"markdown","source":"**Neural Net 2 - Class CNN**"},{"metadata":{"trusted":true,"_uuid":"4aee5df3cf3b664b44a8b105544f1cf5284ebb7c","_kg_hide-input":true},"cell_type":"code","source":"class NeuralNet2(nn.Module):\n    global include_extra_features\n    def __init__(self, n_filters=100, filter_sizes=[3,4,5]):\n        super(NeuralNet2,self).__init__()\n        \n        self.embedding = nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1])\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = Trainable\n        self.embedding_dropout = nn.Dropout2d(rate_drop_embedding)\n        self.convs = nn.ModuleList([nn.Conv2d(in_channels=1, out_channels=n_filters, kernel_size=(fs,embedding_matrix.shape[1])) for fs in filter_sizes])\n        if (include_extra_features==True):\n            self.linear = nn.Linear(len(filter_sizes)*n_filters+ features.shape[1], 1)\n        else:\n            self.linear = nn.Linear(len(filter_sizes)*n_filters, 1)\n        self.dropout = nn.Dropout(rate_drop_linear)\n        \n    def forward(self, x):\n        \n        #x = [sent len, batch size]\n        \n        #x = x.permute(1, 0)\n                \n        #x = [batch size, sent len]\n        \n        if (include_extra_features==True):\n            embedded = self.embedding(x[0])\n        else:\n            embedded = self.embedding(x) \n                \n        #embedded = [batch size, sent len, emb dim]\n        \n        embedded = embedded.unsqueeze(1)\n        \n        #embedded = [batch size, 1, sent len, emb dim]\n        \n        conved = [F.relu(conv(embedded)).squeeze(3) for conv in self.convs]\n            \n        #conv_n = [batch size, n_filters, sent len - filter_sizes[n]]\n        \n        pooled = [F.max_pool1d(conv, conv.shape[2]).squeeze(2) for conv in conved]\n        \n        #pooled_n = [batch size, n_filters]\n        if (include_extra_features==True):\n            f = torch.tensor(x[1], dtype=torch.float).cuda()\n            conc = torch.cat(pooled, dim=1)\n            conc = torch.cat((conc,f), 1)\n        else:\n            conc = torch.cat(pooled, dim=1)\n            \n        cat = self.dropout(conc)\n\n        #cat = [batch size, n_filters * len(filter_sizes)]\n            \n        return self.linear(cat)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e95e23f95857ef10ca302e2e5447f721d8ec8f46"},"cell_type":"markdown","source":"** Neural Net 3 - Embedding, LSTM, GRUs - Extra Features **"},{"metadata":{"trusted":true,"_uuid":"e448f0bde4ae30b983c70d8a44ce9ec294141f16","_kg_hide-input":true},"cell_type":"code","source":"class NeuralNet3(nn.Module):\n    global include_extra_features\n    def __init__(self):\n        super(NeuralNet3, self).__init__()\n\n        self.embedding = nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1])\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = Trainable\n              \n        self.embedding_dropout = nn.Dropout2d(rate_drop_embedding)\n        self.lstm = nn.LSTM(embedding_matrix.shape[1], hidden_size, bidirectional=True, batch_first=True)\n\n        for name, param in self.lstm.named_parameters():\n            if 'bias' in name:\n                 nn.init.constant_(param, 0.0)\n            elif 'weight_ih' in name:\n                 nn.init.kaiming_normal_(param)\n            elif 'weight_hh' in name:\n                 nn.init.orthogonal_(param)\n\n        self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)\n\n        for name, param in self.gru.named_parameters():\n            if 'bias' in name:\n                 nn.init.constant_(param, 0.0)\n            elif 'weight_ih' in name:\n                 nn.init.kaiming_normal_(param)\n            elif 'weight_hh' in name:\n                 nn.init.orthogonal_(param)\n        if (include_extra_features==True):\n            self.linear = nn.Linear(hidden_size*6 + features.shape[1], linear_size)\n        else:\n            self.linear = nn.Linear(hidden_size*6 , linear_size)\n        \n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(rate_drop_linear)\n        self.out = nn.Linear(linear_size, 1)\n\n    def forward(self, x):\n        if (include_extra_features==True):\n            h_embedding = self.embedding(x[0])\n        else:\n            h_embedding = self.embedding(x) \n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n\n        h_lstm, _ = self.lstm(h_embedding)\n\n        h_gru, hh_gru = self.gru(h_lstm)\n        hh_gru = hh_gru.view(-1, 2* hidden_size )\n\n        avg_pool = torch.mean(h_gru, 1)\n        max_pool, _ = torch.max(h_gru, 1)\n\n        if (include_extra_features==True):\n            f = torch.tensor(x[1], dtype=torch.float).cuda()\n            conc = torch.cat(( hh_gru, avg_pool, max_pool,f), 1)\n        else:\n            conc = torch.cat(( hh_gru, avg_pool, max_pool), 1)\n\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        return out","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d79c96572349efbe7d95e4bdd61ad0ba7ba8c5e2"},"cell_type":"markdown","source":"** Neural Net 4 - Embedding, LSTM - Extra Features **"},{"metadata":{"_uuid":"49758f15e955263e46b11998a8e008290fd1f393","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"class NeuralNet4(nn.Module):\n    global include_extra_features\n    def __init__(self, lstm_layer=2):\n        super(NeuralNet4, self).__init__()\n        \n        self.embedding = nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1])\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = Trainable             \n        self.embedding_dropout = nn.Dropout2d(rate_drop_embedding)\n\n        self.lstm = nn.LSTM(embedding_matrix.shape[1],\n                            hidden_size,\n                            num_layers=lstm_layer, \n                            dropout = rate_drop_hidden,\n                            bidirectional=True,\n                            batch_first=True)\n        \n\n        if (include_extra_features==True):\n            self.linear = nn.Linear(hidden_size*lstm_layer*2+ features.shape[1], linear_size)\n        else:\n            self.linear = nn.Linear(hidden_size*lstm_layer*2 , linear_size)\n         \n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(rate_drop_linear)\n        self.out = nn.Linear(linear_size, 1)\n        \n\n        \n    def forward(self, x):\n\n        if (include_extra_features==True):\n            h_embedding = self.embedding(x[0])\n        else:\n            h_embedding = self.embedding(x) \n\n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n\n        lstm_out, (h_n, c_n) = self.lstm(h_embedding)\n\n        conc= torch.cat([c_n[i,:, :] for i in range(c_n.shape[0])], dim=1)\n        if (include_extra_features==True):\n            f = torch.tensor(x[1], dtype=torch.float).cuda()\n            conc = torch.cat((conc,f), dim=1)\n           \n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        return out\n    \n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"21efc846bbf965261f507f2411c617d7dcf10b48"},"cell_type":"markdown","source":"** Neural Net 5 - Embedding, LSTM, GRUs - Extra Features **"},{"metadata":{"trusted":true,"_uuid":"c83c070cb2e214fa6a63a2a784a25f4cdc3c7a37","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class NeuralNet5(nn.Module):\n    def __init__(self):\n        super(NeuralNet5, self).__init__()\n\n        self.embedding = nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1])\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = Trainable             \n        self.embedding_dropout = nn.Dropout2d(rate_drop_embedding)\n        \n        self.conv = nn.Conv1d(2*hidden_size, linear_size, 2)\n        self.rnn = nn.GRU(embedding_matrix.shape[1], hidden_size, num_layers=1, bidirectional=True)\n        if (include_extra_features==True):\n            self.linear = nn.Linear(linear_size+ features.shape[1], 1)\n        else:\n            self.linear = nn.Linear(linear_size, 1)\n        self.drop = nn.Dropout(p=rate_drop_linear)\n        self.activation = nn.ReLU()\n\n    def forward(self, x):\n        \n        if (include_extra_features==True):\n            h_embedding = self.embedding(x[0])\n        else:\n            h_embedding = self.embedding(x) \n        h_embedding = self.embedding_dropout(h_embedding)\n        h_embedding, hid = self.rnn(h_embedding)\n        h_embedding = h_embedding.permute(0, 2, 1)\n\n        cnn = self.conv(h_embedding)\n        cnn = self.activation(cnn)\n        cnn = cnn.permute(0, 2, 1)\n        cnn = cnn.max(1)[0]\n        \n        if (include_extra_features==True):\n            f = torch.tensor(x[1], dtype=torch.float).cuda()\n            cnn = torch.cat((cnn,f), dim=1)\n        x = self.linear(cnn)\n        return x\n\n    def get_trainable_parameters(self):\n        return (param for param in self.parameters() if param.requires_grad)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46f9bdd16e1b5d9965b723609b418f62e15f4efe"},"cell_type":"code","source":"class NeuralNet6(nn.Module):\n    global include_extra_features\n    \n    def __init__(self):\n        super(NeuralNet6, self).__init__()\n        \n        self.embedding = nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1])\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = Trainable\n        \n        self.embedding_dropout = nn.Dropout2d(rate_drop_embedding)\n        self.lstm = nn.LSTM( embedding_matrix.shape[1], hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n\n        self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n\n        self.lstm_attention = Attention(hidden_size * 2, maxlen)\n        self.gru_attention = Attention(hidden_size * 2, maxlen)\n        self.bn = nn.BatchNorm1d(16, momentum=0.5)\n        if (include_extra_features==True):\n            self.linear = nn.Linear(hidden_size*8+1+features.shape[1], linear_size) #643:80 - 483:60 - 323:40\n        else:\n            self.linear = nn.Linear(hidden_size*8+1, linear_size) #643:80 - 483:60 - 323:40\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(rate_drop_linear)\n        self.fc = nn.Linear(linear_size**2,linear_size)\n        self.out = nn.Linear(linear_size, 1)\n        self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)\n        self.caps_layer = Caps_Layer()\n    \n    def forward(self, x):\n        \n#         Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x)\n        if (include_extra_features==True):\n            h_embedding = self.embedding(x[0])\n        else:\n            h_embedding = self.embedding(x) \n            \n        h_embedding = torch.squeeze(\n            self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n\n        ##Attention Layer\n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        # global average pooling\n        avg_pool = torch.mean(h_gru, 1)\n        # global max pooling\n        max_pool, _ = torch.max(h_gru, 1)\n\n     \n        ##Capsule Layer        \n        content3 = self.caps_layer(h_gru)\n        content3 = self.dropout(content3)\n        batch_size1 = content3.size(0)\n        content3 = content3.view(batch_size1, -1)\n        content3 = self.relu(self.lincaps(content3))        \n        \n        if (include_extra_features==True):\n            f = torch.tensor(x[1], dtype=torch.float).cuda()\n            conc = torch.cat((h_lstm_atten, h_gru_atten,content3, avg_pool, max_pool,f), 1)\n        else:\n            conc = torch.cat((h_lstm_atten, h_gru_atten,content3, avg_pool, max_pool), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.bn(conc)\n        conc = self.dropout(conc)\n\n        out = self.out(conc)\n        \n        return out\n################\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55b136e3a1362c22fca6639cac2ae05da235565d"},"cell_type":"markdown","source":"** Auxiliary Function to create Models **"},{"metadata":{"trusted":true,"_uuid":"56e1dbe3c97c85b96f3d49422520d1c906b8567f","_kg_hide-input":true},"cell_type":"code","source":"def create_model (label):\n    \n            if (label==\"0\"):\n                return NeuralNet()\n            if (label==\"1\"):\n                return NeuralNet1()\n            if (label==\"2\"):\n                return NeuralNet2()\n            if (label==\"3\"):\n                return NeuralNet3()            \n            if (label==\"4\"):\n                return NeuralNet4()\n            if (label==\"5\"):\n                return NeuralNet5()\n            if (label==\"6\"):\n                return NeuralNet6()\n            if (label==\"7\"):\n                return create_model7()  \n            if (label==\"8\"):\n                return create_model8()\n            if (label==\"9\"):\n                return create_model9() \n            if (label==\"10\"):\n                return create_model10()            \n            if (label==\"11\"):\n                return create_model11()                \n            if (label==\"12\"):\n                return create_model12()                \n            if (label==\"13\"):\n                return create_model13()                \n            if (label==\"14\"):\n                return create_model14()                \n            if (label==\"15\"):\n                return create_model15()                \n            if (label==\"16\"):\n                return create_model16()   \n            if (label==\"17\"):\n                return create_model17() \n            if (label==\"18\"):\n                return create_model18() \n            if (label==\"19\"):\n                return create_model19()\n            if (label==\"20\"):\n                return create_model20() \n            return None,\"None\"\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88434c9a21a9a8aafd02cd70b8f7465eac3e34c0","_kg_hide-input":true},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b106bb39ec98cd9160ec6801f4c5c18c0260c55b"},"cell_type":"markdown","source":"** Best F1 - 2 ** Function to calculate F1 **"},{"metadata":{"trusted":true,"_uuid":"bcc3ce3483c48eb3e687675df4386615a8d92b7c","_kg_hide-input":true},"cell_type":"code","source":"from sklearn.metrics import roc_curve, precision_recall_curve\nimport matplotlib.pyplot as plt\n%matplotlib inline\ndef best_F12(y_true, y_proba, plot=False):\n    precision, recall, thresholds = precision_recall_curve(y_true, y_proba)\n    thresholds = np.append(thresholds, 1.001) \n    F = 2 / (1/precision + 1/recall)\n    best_score = np.max(F)\n    best_th = thresholds[np.argmax(F)]\n    if plot:\n        plt.plot(thresholds, F, '-b')\n        plt.plot([best_th], [best_score], '*r')\n        plt.show()\n\n    search_result = {'threshold': best_th , 'f1': best_score}\n    return best_th, best_score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4058d7129cbecccc58484916f28a0697a5c2ded1"},"cell_type":"markdown","source":"** Best F1 - ** Function to calculate F1 **"},{"metadata":{"trusted":true,"_uuid":"8e460169d3fd30186ea6c7d5f64d5c0fe2477155","_kg_hide-input":true},"cell_type":"code","source":"from scikitplot.metrics import plot_confusion_matrix\nfrom sklearn.metrics import confusion_matrix\ndef score_model (ypred ,ytrue,show_plot=False):\n    bs=0\n    bt=0\n    for thresh in tqdm(np.arange(0.2, 0.402, 0.001)):\n    #for thresh in tqdm([i * 0.01 for i in range(100)]):\n        score = metrics.f1_score(ypred,(ytrue>=thresh).astype(int))\n        if score >= bs:\n            bt = thresh\n            bs = score  \n    rocauc= roc_auc_score(ypred,ytrue)\n    precision, recall, thresholds = precision_recall_curve(ypred,ytrue)\n    pr_auc = auc(recall, precision)\n    if (show_plot):        \n        print(\"\\nBest F1 score at threshold %2.4f is %2.5f \\n\" % (bt, bs))\n  \n        print(\"\\nROC AUC score is %2.5f \\n\" % rocauc)  \n\n        print(\"\\nPR AUC score is %2.5f \\n\" % pr_auc)\n        plot_confusion_matrix(ypred, np.array(pd.Series(ytrue.reshape(-1,)).map(lambda x:1 if x>=bt else 0)))\n        plt.show()\n        d=confusion_matrix(ypred, (ytrue>=bt).astype(int))\n        true_sincere=   d[0,:].sum()\n        true_insincere= d[1,:].sum()\n        pred_sincere=   d[:,0].sum()\n        pred_insincere= d[:,1].sum()\n        errors_insincere=d[0,1]\n        errors_sincere=d[1,0]\n        total_errors = errors_insincere+errors_sincere\n        \n        print (\"Total True Label    :{:8d}\".format( d.sum()),\" Insincere:{:8d}\".format(true_insincere),\" Sincere:{:8d}\".format(true_sincere))\n        print (\"Total Predicted     :{:8d}\".format( d.sum()),\" Insincere:{:8d}\".format(pred_insincere),\" Sincere:{:8d}\".format(pred_sincere))\n        print (\"Errors              :{:8d}\".format(total_errors),\" Insincere:{:8d}\".format( errors_insincere),\" Sincere:{:8d}\".format(errors_sincere))\n        print (\"Up/Down Diff        :          Insincere:{:8d}\".format(true_insincere-pred_insincere),\" Sincere:{:8d}\".format(true_sincere-pred_sincere))\n        print (\"                               Insincere:({:5.2f}%)\".format((errors_insincere/pred_insincere)*100),\" Sincere:({:5.2f}%)\".format((errors_sincere/pred_sincere) *100))\n    return bt, bs , rocauc, pr_auc, precision.mean(), recall.mean()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"865dec499efde9a86d378f4ef3a5e31a1b382f07"},"cell_type":"markdown","source":"'**Function to record results in a pandas dataframe**"},{"metadata":{"trusted":true,"_uuid":"9bb2a5c13181918e2c4ecee4c637b30bfef7e766","_kg_hide-input":true},"cell_type":"code","source":"def rec_results (Model,Embedding,Thresh,Score,rocauc,prauc,prec,recall,StartTime):\n    global resultsdf\n    resultsdf = resultsdf.append({'Model':Model,\n                                  'Method':method,\n                                  'Embedding':Embedding,\n                                  'Pretext':pretext_proc,\n                                  'Weigth':wgt,\n                                  'F1':Score,\n                                  'Roc Auc':rocauc, \n                                  'PR Auc':prauc,\n                                  'prec':prec,\n                                  'recall':recall,\n                                  \"Threshold\":Thresh,\n                                  \"Duration\":str(round((time.time()-StartTime)/60,0)),\n                                  'Epochs':epochs,\n                                  'EpochsUsed':epoc_used,\n                                  'Trainable':Trainable,\n                                  'Opt':opt,\n                                  'MaxLength':maxlen,\n                                  'MaxVocab':max_vocab,                                  \n                                  'batch_size':batch_size,\n                                  'patience':patience,\n                                  'hidden_size':hidden_size,\n                                  'rate_drop_hidden':rate_drop_hidden,\n                                  'linear_size':linear_size,\n                                  'rate_drop_linear':rate_drop_linear,\n                                  'rate_drop_embedding':rate_drop_embedding,\n                                  'opt':opt,\n                                  'sch':sch,\n                                  'es_mon':es_mon,\n                                  'es_mode':es_mode,\n                                  'Date':datetime.now().strftime(\"%d-%m-%Y %H:%M\")}, ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"de8e14a924f40fb55dfc7fafc5eeae48861f47e2"},"cell_type":"markdown","source":"**Function to load Embedding files (Glove, Paragram, Wiki and Google)**"},{"metadata":{"trusted":true,"_uuid":"6d0fdb7ceeb9923609ee5c7934fb21373a20cc5c","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def load_embedding (emb,word_index) :\n\n    estart = time.time()\n    if debug:\n            embedding_matrix = np.random.randn(10000,300)\n            return embedding_matrix\n    else:\n            print('Indexing '+emb+' vectors')\n            if (emb==\"Glove\"):\n                EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n                def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n                embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n            if (emb==\"Google\"):\n                EMBEDDING_FILE = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\n                embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)\n            if (emb==\"Paragram\"):\n                EMBEDDING_FILE =  '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt' \n                def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n                embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n            if (emb==\"Wiki\"):\n                EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'    \n                embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=False)\n            if (emb==\"Quora\"):\n                EMBEDDING_FILE = \"../quora_clean_300d.bin\"    \n                embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)\n\n            print(\"Vector\",EMBEDDING_FILE )\n            print(\"End Indexing:\",(str(timedelta(seconds=(time.time()-estart)))) )\n            estart = time.time()\n            print('Preparing embedding matrix')\n\n            out_of_features=0\n            out_of_embedding=0\n            in_embedding=0\n            if ( emb==\"Glove\" or emb==\"Paragram\"):\n                if (emb_out_of_vocab==\"mean\"):\n                    all_embs = np.stack(embeddings_index.values())\n                    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n                    embed_size = all_embs.shape[1]\n \n                if (emb_out_of_vocab==\"specific\"):\n                    if (emb==\"Glove\"):\n                        all_embs = np.stack(embeddings_index.values())\n                        emb_mean,emb_std = -0.005838499,0.48782197\n                        embed_size = all_embs.shape[1]\n                    if (emb==\"Paragram\"):\n                        all_embs = np.stack(embeddings_index.values())\n                        emb_mean,emb_std = -0.0053247833,0.49346462\n                        embed_size = all_embs.shape[1]\n                if (emb_out_of_vocab==\"hybrid\"):\n                    if (emb==\"Glove\"):\n                            all_embs = np.stack(embeddings_index.values())\n                            emb_mean,emb_std = all_embs.mean(), all_embs.std()\n                            embed_size = all_embs.shape[1]\n                    if (emb==\"Paragram\"):\n                        all_embs = np.stack(embeddings_index.values())\n                        emb_mean,emb_std = -0.0053247833,0.49346462\n                        embed_size = all_embs.shape[1]\n\n                #word_index = tokenizer.word_index\n                nb_words = min(max_vocab, len(word_index)+1)\n                embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n                for word, i in word_index.items():\n                    if i >= max_vocab: \n                        out_of_features+=1\n                        continue\n                    embedding_vector = embeddings_index.get(word)\n                    if embedding_vector is not None: \n                        embedding_matrix[i] = embedding_vector\n                        in_embedding+=1\n                    else:\n                        out_of_embedding+=1\n            else: #\"Google\"\n                embed_size=300\n                #word_index = tokenizer.word_index\n                nb_words = min(max_vocab, len(word_index)+1)\n                embedding_matrix= (np.random.rand(nb_words, embed_size) - 0.5) / 5.0\n                #embedding_matrix= (np.random.normal(nb_words, embed_size) - 0.5) / 5.0\n                for word, i in word_index.items():\n                    if i >= max_vocab:\n                        out_of_features+=1\n                        continue\n                    if word in embeddings_index:\n                        embedding_vector = embeddings_index.get_vector(word)\n                        embedding_matrix[i] = embedding_vector\n                        in_embedding+=1\n                    else:\n                        out_of_embedding+=1\n            print(\"Total Vocabulary:\",len(word_index))\n            print(\"Out of Vocabulariy:\", out_of_features)\n            print (\"Mapped Embeddings:\",in_embedding)\n            print (\"Out of Embeddings:\",out_of_embedding)\n            print(\"End Preparing embedding matrix:\",(str(timedelta(seconds=(time.time()-estart)))) )\n            print (\"-------------------------------------------\")\n            del embeddings_index\n            return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db77ee4414e5ff7114c02da3610457b042b120dd"},"cell_type":"markdown","source":"** Evaluate Model 0  - No Special Parameters**"},{"metadata":{"_uuid":"960868b3efac279e28ab68a28871fd425c003630"},"cell_type":"markdown","source":"** Evaluate Model 1 - with CyclicLR **"},{"metadata":{"trusted":true,"_uuid":"395cbb32afc9f5c942e15ffe5e7e59d73682da50","_kg_hide-input":true},"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2ebdd00769afcae270a511f7609711015febf52b"},"cell_type":"markdown","source":"** Torch - Binary Acuracy **"},{"metadata":{"trusted":true,"_uuid":"7b7cd06716f293116cf3add0b957ddd2abd2ed33","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def binary_accuracy(preds, y):\n    \"\"\"\n    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n    \"\"\"\n\n    #round predictions to the closest integer\n    rounded_preds = torch.round(torch.sigmoid(preds))\n    correct = (rounded_preds == y).float() #convert into float for division \n    acc = correct.sum()/len(correct)\n    return acc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b261b3779edbdf6b59bb3c27ce7d3926b33e0ed"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"f5150b0c014d3735e9b0d9f2eaaca51d880a070d"},"cell_type":"markdown","source":"** Torch Train **"},{"metadata":{"trusted":true,"_uuid":"63540736e9d6a62836de4aeef3a53f5abee70c9d","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"\ndef torch_train(model, iterator, optimizer,criterion,scheduler, metric):\n    \n    epoch_loss = 0\n    epoch_met = 0\n    probs = []\n    tgts = []\n    model.train()\n    \n    #for batch in iterator:\n    for i, (x_batch, y_batch) in enumerate(iterator):    \n        optimizer.zero_grad()\n\n        if (include_extra_features==True):\n            f = kfold_X_features[i * batch_size:(i+1) * batch_size]\n            predictions = model([x_batch,f])\n        else:\n            predictions = model(x_batch)\n            \n        if \"CyclicLR\" in str(scheduler) :\n            scheduler.batch_step()\n        \n        loss = criterion(predictions, y_batch)\n        if (metric==\"val_acc\"):\n            emet = binary_accuracy(predictions, y_batch)\n            epoch_met += emet.item()\n\n        optimizer.zero_grad()           \n        loss.backward()\n        optimizer.step()\n        proba= F.sigmoid(predictions).data.cpu().numpy()\n        epoch_loss += loss.item()\n\n        tgts.append(y_batch.data.cpu().numpy())\n        probs.append(proba)\n    tgts = np.vstack(tgts)\n    probs = np.vstack(probs)\n    if (metric==\"val_acc\"):\n        met_result=epoch_met / len(iterator)\n    if (metric==\"val_auc\"):\n        met_result = np.mean(roc_auc_score(tgts, probs)) \n    if (metric==\"val_f1\"):\n        bt ,met_result = best_F12(tgts, probs) \n    if (metric==\"val_loss\"):\n        met_result=epoch_loss / len(iterator)\n    return epoch_loss / len(iterator), met_result","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b6ca7c0818403ea345e3d9e750c95a5d4a0e6be"},"cell_type":"markdown","source":"** Torch Evaluate **"},{"metadata":{"trusted":true,"_uuid":"9cafa65ddc9074f34ebfce65f2e83500d5dad42a","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def torch_evaluate(model, iterator, criterion, metric, results):\n    \n    epoch_loss = 0\n    epoch_met = 0\n    probs = []\n    tgts = []\n    model.eval()\n    \n    with torch.no_grad():\n           \n        for i, (x_batch, y_batch) in enumerate(iterator):\n            if (include_extra_features):\n                f =  kfold_X_valid_features[i * batch_size:(i+1) * batch_size]\n                predictions = model([x_batch,f]).detach() \n            else:\n                predictions = model(x_batch).detach()\n                \n            results[i * batch_size:(i+1) * batch_size] = sigmoid(predictions.cpu().numpy())[:, 0]  \n            \n            loss = criterion(predictions, y_batch)\n\n            if (metric==\"val_acc\"):\n                emet = binary_accuracy(predictions, y_batch)\n                epoch_met += emet.item()\n            epoch_loss += loss.item()\n            proba= F.sigmoid(predictions).data.cpu().numpy()\n            tgts.append(y_batch.data.cpu().numpy())\n            probs.append(proba)\n        tgts = np.vstack(tgts)\n        probs = np.vstack(probs)\n        if (metric==\"val_acc\"):\n            met_result=epoch_met / len(iterator)\n        if (metric==\"val_auc\"):\n            met_result = np.mean(roc_auc_score(tgts, probs)) \n        if (metric==\"val_f1\"):\n            bt ,met_result = best_F12(tgts, probs) \n        if (metric==\"val_loss\"):\n            met_result=epoch_loss / len(iterator)\n    return epoch_loss / len(iterator), met_result, probs","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"317207f4adf33c17cc0897b2f9a67110f29366fa"},"cell_type":"markdown","source":"** Torch Predict **"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"6b39f21df44b8b4ec0667022da625e350746d98a","_kg_hide-output":true},"cell_type":"code","source":"def torch_predict(model, iterator, criterion, extra_features, results):\n      \n    model.eval()\n    \n    with torch.no_grad():\n    \n        for i, (x_batch, ) in enumerate(iterator):\n            if (include_extra_features):\n                f = extra_features[i * batch_size:(i+1) * batch_size]            \n                predictions = model([x_batch,f]).detach()               \n            else:\n                predictions = model(x_batch).detach()\n\n            results[i * batch_size:(i+1) * batch_size] = sigmoid(predictions.cpu().numpy())[:, 0]\n            \n    return ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7fea8b6836c147e9bde8489919962b37d6764927"},"cell_type":"markdown","source":"** Evaluate Model 2 - Utiliza TorchTrain, TorchEval, TorchPredic acima **"},{"metadata":{"trusted":true,"_uuid":"eec0e2d606a4e4abb007dace372c905dbc8ea753","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def save_torch(m, info):\n    torch.save(info, '../best_model.chck')\n    torch.save(m, '../best_model.m')\n    \ndef load_torch():\n    m = torch.load('../best_model.m')\n    info = torch.load('../best_model.chck')\n    return m, info","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6e0f77791e967182c79b7055720187d3d7eb36f5"},"cell_type":"markdown","source":"** Evaluate Model  - Utiliza TorchTrain, TorchEval, TorchPredic acima **"},{"metadata":{"trusted":true,"_uuid":"2059a86187a96f1f31a3a5122866a7b49109b57e","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def evaluate_model (label,emb, embedding_matrix,train_X,train_y, t_X, t_y, v_X, v_y, epochs, batch_size, patience=0):\n    \n        global resultsdf \n        global met\n        global clr\n        global epoc_used \n        global kfold_X_features\n        global kfold_X_valid_features\n        \n        best_thresh = 0.0\n        best_score = 0.0\n        epoc_used=0\n        best_epoch=0\n        mstart = time.time()        \n\n        model=None\n        model = create_model(label)\n        mdln=\"Neural Net\"+label\n        print (\"\")\n        print (mdln+'-'+emb)\n\n        model.cuda()\n        metric=es_mon\n        \n        step_size = 300\n        base_lr, max_lr = 0.001, 0.003  \n        if (opt==\"rmsprop\"):\n            optimizer = torch.optim.RMSprop(model.parameters())\n\n        if (opt==\"adam2\"):\n            optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=max_lr)\n            \n        if (opt==\"adam1\"):\n            optimizer = torch.optim.Adam(model.parameters())\n        \n        if (opt==\"adam\"):\n            optimizer = optim.Adam(model.parameters(), lr=0.001,\n                       betas=(0.9, 0.98), eps=1e-09)\n        criterion = torch.nn.BCEWithLogitsLoss(reduction=\"sum\")\n        \n        scheduler=None\n        rscheduler=None\n        \n        if \"lron\" in sch:\n            rscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.2, patience=patience, verbose=1)\n        \n        if \"clr\" in sch:\n\n            ################################################################################################\n            scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr, step_size=step_size, mode='exp_range', gamma=0.999999999)\n            ###############################################################################################\n\n        x_test_cuda = torch.tensor(test_X, dtype=torch.long).cuda()\n        X_train_cuda= torch.tensor(train_X, dtype=torch.long).cuda()\n\n        test = torch.utils.data.TensorDataset(x_test_cuda)\n        train = torch.utils.data.TensorDataset(t_X, t_y)\n        alltrain =torch.utils.data.TensorDataset(X_train_cuda)\n        valid = torch.utils.data.TensorDataset(v_X, v_y)\n\n        \n        test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n        train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n        alltrain_loader = torch.utils.data.DataLoader(alltrain, batch_size=batch_size, shuffle=False)\n        valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n        \n        train_preds_fold = np.zeros(len(train_X))\n        valid_preds_fold = np.zeros((v_X.size(0)))\n        test_preds_fold = np.zeros(len(test_X))\n        \n        if (es_mode==\"max\"):\n            best_metric=0\n        else:\n            best_metric =  float(\"inf\") \n        valid_metrics=[0.]\n        pt=0\n        for epoch in range(epochs):\n            \n            start_time = time.time()\n            train_loss, train_metric = torch_train(model, train_loader, optimizer, criterion, scheduler, metric)\n            valid_loss, valid_metric , valid_proba= torch_evaluate(model, valid_loader, criterion, metric, valid_preds_fold)\n\n            print(f'| Epoch: {epoch+1:02} | Train Loss: {train_loss:.3f} |','Train '+es_mon+':',f' {train_metric*100:.2f}%',f' | Val. Loss: {valid_loss:.3f}',' | Val. '+es_mon+f': {valid_metric*100:.2f} |')\n                \n            valid_metrics += [valid_metric]\n\n            if \"lron\" in sch:  ## Reduce on Plateu selected\n                rscheduler.step(valid_loss)\n\n            if \"est\" in sch:   ## Early Stop Selectec \n                if (es_mode==\"max\"):          \n                    if (valid_metric > best_metric):\n                        improved=True\n                    else:\n                        improved=False          \n                else:      \n                    if (valid_metric < best_metric):\n                        improved=True\n                    else:\n                        improved=False\n                epoc_used=epoch+1\n                if (improved):       \n                    \n                    print('New best '+metric+':', valid_metric)\n                    best_epoch=epoc_used\n                    model_state_dict = model.state_dict()\n                    checkpoint = {\n                        'model': model_state_dict,\n                        'optimizer': optimizer.state_dict(),\n                        'epoch': epoch,\n                         metric: valid_metric}\n                    best_metric=valid_metric\n                    save_torch (model,checkpoint )\n                    #torch.save(checkpoint, '../'+model_name)\n                    print('    - [Info] The checkpoint file has been updated.')\n                else:\n                    print(\"Metric not improved\")\n                    pt=pt+1\n                    if (pt>patience):\n                        break\n        if \"est\" in sch:\n            print (\"Getting the Best Model - epoch \",best_epoch )\n            model, checkpoint = load_torch()\n        \n        torch_predict(model, test_loader, criterion, test_features, test_preds_fold)\n\n        ret_thresh,ret_f1,ret_rocauc, ret_prauc, ret_prec, ret_recall=score_model(v_y.cpu().numpy()[:, 0],valid_preds_fold, False)\n        rec_results (\"ST\"+mdln,emb,ret_thresh,ret_f1,ret_rocauc, ret_prauc, ret_prec, ret_recall,mstart)\n        if (Full_Train_Validation):\n            torch_predict(model, alltrain_loader, criterion, features, train_preds_fold)\n            \n        import gc;           \n        del model\n        gc.collect()\n        time.sleep(10)\n        return  train_preds_fold,valid_preds_fold,test_preds_fold,ret_thresh,ret_f1,ret_rocauc, ret_prauc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1fa355ec004708e7606f69c4bfb0f34d39a96b5c"},"cell_type":"markdown","source":"** Stratified Kfold ** "},{"metadata":{"trusted":true,"_uuid":"560cc612975c915ecb2b602a3cc92045c2b8c2b7","_kg_hide-input":true},"cell_type":"code","source":"def evaluate_stratifiedkfold (label,mt,emb,train_X,train_y, test_X, n_splits, stop_split, epochs, batch_size,patience, specific_split=None):\n    \n    global resultsdf \n    global mn\n    global method\n    global embedding_matrix\n    global features\n    global kfold_X_features\n    global kfold_X_valid_features\n    \n#   train_preds = np.zeros((len(train_X)))\n#    test_preds = np.zeros((len(test_X)))\n#    features = np.array(features)\n \n    method=mt\n    train_stratified= np.zeros([len(train_X),(stop_split)])\n    test_stratified = np.zeros([len(test_X),(stop_split)])\n    val_stratified  = np.zeros([len(train_X)])\n    \n    msstart = time.time()\n    \n    embedding_matrix=None\n    if  (emb==\"Glove\"):\n        embedding_matrix=embedding_matrix_1\n    if  (emb==\"Wiki\"):\n        embedding_matrix=embedding_matrix_2\n    if  (emb==\"Paragram\"):\n        embedding_matrix=embedding_matrix_3\n    if  (emb==\"Google\"):\n        embedding_matrix=embedding_matrix_4\n    if  (emb==\"Quora\"):\n        embedding_matrix=embedding_matrix_5  \n    if  (emb==\"Combined\"):\n        embedding_matrix=embedding_matrix_combined\n    if  (emb==\"Combined2\"):\n        embedding_matrix=embedding_matrix_combined2    \n    if  (emb==\"Combined3\"):\n        embedding_matrix=embedding_matrix_combined3\n    if  (emb==\"Combined4\"):\n        embedding_matrix=embedding_matrix_combined4\n    if  (emb==\"Combined5\"):\n        embedding_matrix=embedding_matrix_combined5\n    if  (emb==\"Concatenated\"):\n        embedding_matrix=embedding_matrix_concatenated \n    if  (emb==\"All\"):\n        embedding_matrix=embedding_matrix_all\n\n    splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED).split(train_X, train_y))\n    mn=mn+1\n    bf1s=np.zeros([stop_split])\n    bt=np.zeros([stop_split])\n    rocaucs= np.zeros([(stop_split)])\n    praucs = np.zeros([(stop_split)])\n    seed_torch(SEED) \n    for idx, (train_idx, valid_idx) in enumerate(splits):\n        \n            if (split_to_run > 0):\n                if (idx != (split_to_run -1) ):\n                    continue\n                else:\n                    idx=0\n            if (include_extra_features):\n                kfold_X_features = features[train_idx.astype(int)]\n                kfold_X_valid_features = features[valid_idx.astype(int)]\n            X_split = torch.tensor(train_X[train_idx], dtype=torch.long).cuda()\n            y_split = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32).cuda()\n            X_val = torch.tensor(train_X[valid_idx], dtype=torch.long).cuda()\n            y_val = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n\n            print(f'Fold {idx + 1}')\n\n            pred_train_y, pred_val_y, pred_test_y, bt[idx] , bf1s[idx],rocaucs[idx], praucs[idx]=evaluate_model(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val, epochs, batch_size ,patience)\n              \n            val_stratified[valid_idx] =  pred_val_y\n            train_stratified[:,idx]   = pred_train_y\n            test_stratified[:,idx]    = pred_test_y\n            if ((idx+1)==stop_split):\n                break\n    val_pred   =  val_stratified\n    train_pred =  train_stratified.sum(axis=1)/stop_split\n    test_pred  =  test_stratified.sum(axis=1)/stop_split\n                         \n    if (Full_Train_Validation==False):\n        ret_thresh,ret_f1,ret_rocauc, ret_prauc, ret_prec, ret_recall=score_model(train_y,val_stratified, True)\n        rec_results (\"Model Kfold:\"+label+\"-skfold- \"+str(n_splits)+\"Stop: \"+str(stop_split),emb,ret_thresh,ret_f1,ret_rocauc, ret_prauc, ret_prec, ret_recall,msstart)\n        return train_pred,val_pred,test_pred,ret_thresh,ret_f1,ret_rocauc, ret_prauc\n    else:\n        ret_thresh, ret_f1,ret_rocauc, ret_prauc, ret_prec, ret_recall=score_model(train_y,train_pred, True)\n        if (stop_split > 1):\n            rec_results (\"Model Mean:\"+label+\"-skfold- \"+str(n_splits)+\"Stop: \"+str(stop_split),emb,bt.mean(),bf1s.mean(),rocaucs.mean(), praucs.mean(), ret_prec, ret_recall,msstart)\n        return train_pred,val_pred,test_pred,bt.mean(),bf1s.mean(),rocaucs.mean(), praucs.mean()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"651790ca899b69fd7495f0e209d9112d0ea14796"},"cell_type":"markdown","source":"** Defines which Embbedings to LOAD **"},{"metadata":{"trusted":true,"_uuid":"9ec1bf349a335e12009c4febcbd1ce4d02d2fb3f","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"load_Glove=False\nload_Paragram=False\nload_Quora=False\nload_Wiki=False\nload_Google=False\nload_Combined=False\nload_Combined2=False\nload_Combined3=False\nload_Combined4=False\nload_Combined5=False\nload_Concatenated=False\nload_All=False\nfor ne,item in  enumerate(model_list): \n    emb=item[3] # Embeddings\n    if emb==\"Glove\":\n        load_Glove=True\n    if emb==\"Paragram\":\n        load_Paragram=True\n    if emb==\"Quora\":\n        load_Quora=True\n    if emb==\"Wiki\":\n        load_Wiki=True\n    if emb==\"Google\":\n        load_Google=True\n    if emb==\"Combined\":\n        load_Glove=True\n        load_Paragram=True\n        load_Combined=True\n    if emb==\"Combined2\":\n        load_Wiki=True\n        load_Quora=True\n        load_Combined2=True\n    if emb==\"Combined3\":\n        load_Glove=True\n        load_Paragram=True\n        load_Wiki=True\n        load_Combined3=True\n    if emb==\"Combined4\":\n        load_Glove=True\n        load_Paragram=True\n        load_Quora=True\n        load_Combined4=True\n    if emb==\"Combined5\":\n        load_Glove=True\n        load_Paragram=True\n        load_Wiki=True\n        load_Google=True\n        load_Combined5=True\n\n    if emb==\"Concatenated\":\n        load_Glove=True\n        load_Paragram=True\n        load_Concatenated=True\n    if emb==\"All\":\n        #load_Glove=True\n        #load_Paragram=True\n        #load_Google=True\n        load_Wiki=True\n        load_Quora=True\n        load_All=True","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b44b0ddbe6a84cd89631772d592f1fdc787fbf9a"},"cell_type":"markdown","source":"** Load and Process Train,Val and Test  ** "},{"metadata":{"trusted":true,"_uuid":"8fdca1e249fdcaf89fdd8d4f07d3f3252435bdec","_kg_hide-input":true,"_kg_hide-output":true,"scrolled":false},"cell_type":"code","source":"###\n###  Load Train,Val and Test \n###\ntqdm_notebook().pandas()\ntrain_X, test_X, train_y, test_df, word_index , features, test_features = load_and_process(pretext_proc)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aebe46c0a10c2c9f76f5a8fb98551435a6099a4e"},"cell_type":"markdown","source":"** Dimensionality Reductions - Embeddings **"},{"metadata":{"trusted":true,"_uuid":"22a19d465020780bdbe9aaba0c60a21c225b3c3b","_kg_hide-input":true},"cell_type":"code","source":"# Build a Latent Semantic Indexing Model\nfrom sklearn.decomposition import TruncatedSVD\n\ndef reduce_matrix (emb_mat,n=10):\n  svd_model = TruncatedSVD(n_components=n)\n  reduced = svd_model.fit_transform(emb_mat)\n  return reduced ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"84f8c82c9701283366085208e6c033c96c317369"},"cell_type":"markdown","source":"**Load Embedding Matrix from Glove,Wki,Paragram and Google. **\n** Create Combination or Concatenatedf Matrix **"},{"metadata":{"trusted":true,"_uuid":"ff7b8b27a1f97cbdff16476c0c99712ac6d9efc4","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"###\n### Load Embedding Matrix\n###\nseed_torch(SEED)\nif load_Glove:\n    embedding_matrix_1 = load_embedding(\"Glove\",word_index)\nif load_Wiki:\n    embedding_matrix_2 = load_embedding(\"Wiki\",word_index)\nif load_Paragram:\n    embedding_matrix_3 = load_embedding(\"Paragram\",word_index)\nif load_Google:\n    embedding_matrix_4 = load_embedding(\"Google\",word_index)\nif load_Quora:\n    embedding_matrix_5 = load_embedding(\"Quora\",word_index)\n## Simple average: http://aclweb.org/anthology/N18-2031\n\n# We have presented an argument for averaging as\n# a valid meta-embedding technique, and found experimental\n# performance to be close to, or in some cases \n# better than that of concatenation, with the\n# additional benefit of reduced dimensionality  \n\n\n## Unweighted DME in https://arxiv.org/pdf/1804.07983.pdf\n\n# “The downside of concatenating embeddings and \n#  giving that as input to an RNN encoder, however,\n#  is that the network then quickly becomes inefficient\n#  as we combine more and more embeddings.”\n  \n#embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3], axis = 0)\nif load_Combined: \n    embedding_matrix_combined = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)\n    print (\"Combined Matrix Dimension:\",np.shape(embedding_matrix_combined))\n\nif load_Combined2: \n    embedding_matrix_combined2 = np.mean([embedding_matrix_2, embedding_matrix_5], axis = 0)\n    print (\"Combined Matrix Dimension:\",np.shape(embedding_matrix_combined2))    \n    \nif load_Combined3: \n    embedding_matrix_combined3 = np.mean([embedding_matrix_1, embedding_matrix_2,embedding_matrix_3], axis = 0)\n    print (\"Combined Matrix Dimension:\",np.shape(embedding_matrix_combined3))\n    \nif load_Combined4: \n    embedding_matrix_combined4 = np.mean([embedding_matrix_1, embedding_matrix_3,embedding_matrix_5], axis = 0)\n    print (\"Combined Matrix Dimension:\",np.shape(embedding_matrix_combined4))\n    \nif load_Combined5: \n    embedding_matrix_combined5 = np.mean([embedding_matrix_1, embedding_matrix_2,embedding_matrix_3,embedding_matrix_4], axis = 0)\n    print (\"Combined Matrix Dimension:\",np.shape(embedding_matrix_combined5))\n    \nif load_Concatenated:\n    embedding_matrix_concatenated = np.concatenate((embedding_matrix_1, embedding_matrix_3), axis=1)  \n    #del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4\n    #gc.collect()\n    print (\"Concatenates Matrix dimensions\",np.shape(embedding_matrix_concatenated))\n    \nif load_All:\n    #e1=reduce_matrix (embedding_matrix_1,n=150)\n    e2=reduce_matrix (embedding_matrix_2,n=100)\n    #e3=reduce_matrix (embedding_matrix_3,n=150)\n    #e4=reduce_matrix (embedding_matrix_4,n=50)\n    e5=reduce_matrix (embedding_matrix_5,n=100)\n    embedding_matrix_all = np.concatenate((e2,e5), axis=1) \n    #del embedding_matrix_1\n    del embedding_matrix_2\n    #del embedding_matrix_3\n    #del embedding_matrix_4\n    del embedding_matrix_5\n    print (\"All SVD Matrix dimensions\",np.shape(embedding_matrix_all))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3374663b48aee4450c1602fce1f460ed451cb91c"},"cell_type":"markdown","source":"**Run Models\n**"},{"metadata":{"trusted":true,"_uuid":"16dffb7853c3e94281d5a9e38bff5494bd322e49","_kg_hide-input":true,"_kg_hide-output":true,"scrolled":false},"cell_type":"code","source":"###\n### Run All Models\n###\nresultsdf = pd.DataFrame(columns=['Model','F1','Roc Auc',\"PR Auc\",'Embedding','Pretext','Weigth','Duration','Method',\n                                  'Epochs','EpochsUsed','patience','batch_size'\n                                  ,'MaxLength','MaxVocab' ])\n\n\n### Matrix of Predictions - Validation and Test\n### Stores All Predicted Values\n\nallpred_train_y=np.zeros([len(train_X),(len(model_list))])\nallpred_val_y=np.zeros([len(train_X),(len(model_list))])\nallpred_test_y=np.zeros([len(test_X),(len(model_list))])\nallbf1=np.zeros([(len(model_list))])\nallrocauc=np.zeros([(len(model_list))])\nallprauc=np.zeros([(len(model_list))])\nallthresh=np.zeros([(len(model_list))])\n\nepoc_used=0\nmn=-1 \nprint (\"Max Vocab\",max_vocab)\n#model_list.append([\"18\",1,\"5\",\"Combine\",8,pat,bs,splits,stop,nh,nd,ndh,ndd,nds,loss_bin,opt_rmsprop,met_acc,mon_acc,mode_acc])\nfor ne,item in  enumerate(model_list): \n    md=item[0]  # Model to Run\n    wgt=item[1]    # Model weigth for essembling\n    mt=item[2]   # Model Evaluation Method\n    emb=item[3] # Embeddings\n    epochs=item[4] # Number of Epochs\n    patience=item[5] # Patience - Wait patience epochs if score does not improve\n    batch_size=item[6] # Batch Size to Fit Model\n    nsplits=item[7]   # Number of Splits for Stratified k-fold\n    stop_split=item[8]    # Number of Splits to run in Stratified k-fold\n    split_to_run=item[9]\n    hidden_size=item[10]  # Size of hidden layers in models (lstm and gru)\n    linear_size=item[11] # Size of hidden layers in model - dense\n    rate_drop_hidden = item[12] # lstm and gru - drop-out rates\n    rate_drop_linear = item[13] # dense drop-out rates\n    rate_drop_embedding=item[14]  # spatial drop-out rate\n    loss=item[15] # Model Loss\n    opt=item[16]  #Models Optmizer\n    sch=item[17]  #Models Metric\n    es_mon=item[18] # Early stop monitor \n    es_mode=item[19]  # Early stop mode\n    if (nsplits != stop_split):\n        Full_Train_Validation=True\n    else:\n        Full_Train_Validataion=False\n\n    print (\"******Evaluating Model:\"+md+\" Evaluation Method:\",mt)\n    trainp,valp,testp,bt,bf1,brocauc,bprauc=evaluate_stratifiedkfold (md,mt,emb,train_X,train_y, test_X, nsplits, stop_split, epochs, batch_size,patience)\n    allbf1[mn]=bf1\n    allthresh[mn]=bt\n    allrocauc[mn]=brocauc\n    allprauc[mn]=bprauc\n    allpred_train_y[:,mn] = trainp\n    allpred_val_y[:,mn] = valp\n    allpred_test_y[:,mn] = testp\n    print (\"****************************************************\")\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad19564cfeb45f4bfcf520af253ae239bdd82735"},"cell_type":"markdown","source":"**Calculate weights and apply to results****Calculate weights and apply to results**"},{"metadata":{"trusted":true,"_uuid":"22cd300ddf59e25266a21bddbc752b0908331302","_kg_hide-input":true,"scrolled":false},"cell_type":"code","source":"#### Applying Weights to results\n#### Manual/Score\ntwgt=0\nwgt_test_y=0\nwgt_train_y=0\nwgt_val_y=0\nessemblet=\"Essemble: \"\nfor ne,item in  enumerate(model_list):\n    if (Weight_Method==\"Manual\"):\n        wg=item[1]\n    if (Weight_Method==\"f1\"):\n        wg=allbf1[ne]\n    if (Weight_Method==\"auc\"):\n        wg=allrocauc[ne]\n    twgt=twgt+wg\n    wgt_val_y=wgt_val_y+(allpred_val_y[:,ne]*wg)\n    wgt_train_y=wgt_train_y+(allpred_train_y[:,ne]*wg)\n    wgt_test_y =wgt_test_y +(allpred_test_y[:,ne]*wg)\n    essemblet=essemblet+item[0]+\" \"\nwgt_val_y  =wgt_val_y/twgt\nwgt_train_y=wgt_train_y/twgt\nwgt_test_y =wgt_test_y/twgt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e735e916b119a4148aaec87772f76130a000310f"},"cell_type":"markdown","source":"**Write Output to submission file**"},{"metadata":{"trusted":true,"_uuid":"ed7c755da2a2f96f4f41d58722c4d1f1af78a168","_kg_hide-input":true,"scrolled":true},"cell_type":"code","source":"def bestThresshold(y_train,train_preds):\n    tmp = [0,0,0] # idx, cur, max\n    delta = 0\n    for tmp[0] in tqdm(np.arange(0.2, 0.402, 0.001)):\n        tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])\n        if tmp[1] > tmp[2]:\n            delta = tmp[0]\n            tmp[2] = tmp[1]\n    print('best threshold is {:.4f} with F1 score: {:.5f}'.format(delta, tmp[2]))\n    return delta\n\nif (Full_Train_Validation):\n        print (\"Full Validation\")\n        best_thresh,best_score=best_F12(train_y,wgt_train_y,True )\n        print (\"Threshold Valid F12:\",best_thresh)\n        delta = bestThresshold(train_y,wgt_train_y)\n        print (\"Threshold Valid Delta:\",delta)\n        best_thresh,best_score,rauc,prauc,mp,mr=score_model(train_y,wgt_train_y,True )\n        rec_results (essemblet+\"FULL Validation \",\"-\",best_thresh,best_score,rauc,prauc,mp,mr,bstart) \n        print (\"Threshold Full Valid:\",best_thresh)\nelse:\n        print (\"Stratified Cross Validation\")\n        best_thresh,best_score=best_F12(train_y,wgt_val_y,True )\n        print (\"Threshold Valid F12:\",best_thresh)\n        delta = bestThresshold(train_y,wgt_val_y)\n        print (\"Threshold Valid Delta:\",delta)\n        best_thresh,best_score,rauc,prauc,mp,mr=score_model(train_y,wgt_val_y,True )\n        rec_results (essemblet+\"CV Validation \",\"-\",best_thresh,best_score,rauc,prauc,mp,mr,bstart)\n        print (\"Threshold Valid F1:\",best_thresh)\n\narray_test_y=np.array(wgt_test_y)\n\nprint (\"\")\nprint (allthresh)\nprint (\"Threshold Minimo:\",allthresh.min(),(np.count_nonzero(array_test_y >= allthresh.min())))\nprint (\"Threshold Médio:\",allthresh.mean(),(np.count_nonzero(array_test_y >= allthresh.mean())))\nprint (\"Threshold Maximo:\",allthresh.max(),(np.count_nonzero(array_test_y >= allthresh.max())))\nif (debug):\n    tg=allthresh.min()\n    itg=tg\n    ni=(np.count_nonzero(array_test_y >= tg))\n    ii=ni\n    print (\"tg i\",itg,\" Insincere i:\",ii)\n    while  (ni >= 3488) :\n            tg=tg+0.001\n            ni=(np.count_nonzero(array_test_y >= tg) )\n    print (\"tg\",tg,tg-itg,\" Insincere:\",ni, ii-ni)\n\n# Write the output\n#best_thresh\n#y_te = (array_test_y >= 0.2940).astype(np.int)\ny_te = (array_test_y >= best_thresh).astype(np.int)\nsubmit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\nsubmit_df.to_csv(\"./submission.csv\",index=False)\nprint (\"Submit Rows:\",submit_df[\"qid\"].count())\n#rec_results ('Time Elapsed','',0,0,bstart)\ndisplay (resultsdf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"957b2a2d74fa542e5481fd9952413323e55259fb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}