{"cells":[{"metadata":{},"cell_type":"markdown","source":"### This notebook is an update of [this work](https://www.kaggle.com/nxrprime/jigsaw-google-q-a-eda-ii). Just added my own pretrained weights.\n### If it is useful, please consider your upvote."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"# CODE TAKEN FROM https://github.com/kpe/bert-for-tf2/\n# ALL CREDITS TO https://github.com/kpe\n# CODE COPIED TO LOCAL FOLDER DUE TO INTERNET RESTRICTIONS\n# NORMALLY THIS CODE WOULD BE AVAILABLE VIA pip install bert-for-tf2\n\n# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Tokenization classes.\"\"\"\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport gc\nimport os\nimport warnings\nimport operator\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.corpus import stopwords\nfrom nltk import ngrams\nfrom collections import Counter\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom gensim.parsing.preprocessing import STOPWORDS\nfrom nltk.stem import WordNetLemmatizer, SnowballStemmer\nfrom nltk.stem.porter import *\nimport nltk\nfrom gensim import corpora, models\nimport pyLDAvis\nimport pyLDAvis.gensim\nfrom keras.preprocessing.text import Tokenizer\n\npyLDAvis.enable_notebook()\n#np.random.seed(2018)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"sample = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')\nsample.head(3)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')\ntrain.head(3)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')\ntest.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"target_columns = sample.columns.values[1:].tolist()\ntarget_columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"Train and test shape: {} {}\".format(train.shape, test.shape))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"targets = [\n        'question_asker_intent_understanding',\n        'question_body_critical',\n        'question_conversational',\n        'question_expect_short_answer',\n        'question_fact_seeking',\n        'question_has_commonly_accepted_answer',\n        'question_interestingness_others',\n        'question_interestingness_self',\n        'question_multi_intent',\n        'question_not_really_a_question',\n        'question_opinion_seeking',\n        'question_type_choice',\n        'question_type_compare',\n        'question_type_consequence',\n        'question_type_definition',\n        'question_type_entity',\n        'question_type_instructions',\n        'question_type_procedure',\n        'question_type_reason_explanation',\n        'question_type_spelling',\n        'question_well_written',\n        'answer_helpful',\n        'answer_level_of_information',\n        'answer_plausible',\n        'answer_relevance',\n        'answer_satisfaction',\n        'answer_type_instructions',\n        'answer_type_procedure',\n        'answer_type_reason_explanation',\n        'answer_well_written'    \n    ]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"stopwords=stopwords.words('english')\ntrain['que_stopwords']=train['question_body'].apply(lambda x : [x for x in x.split() if x in stopwords])\ntrain['ans_stopwords']=train['answer'].apply(lambda x: [x for x in x.split() if x in stopwords])","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"\ndef common_ngrams(col,common=10):\n    corpus=[]\n    for question in train[col].values:\n        words=[str(x[0]+' '+x[1]) for x in ngrams(question.split(),2)]\n        corpus.append(words)\n    flatten=[x for one in corpus for x in one]\n    counter=Counter(flatten)\n    most_common=counter.most_common(common)\n    string,value=zip(*(most_common))\n    return string,value\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"\n\nimport collections\nimport re\nimport unicodedata\nimport six\nimport tensorflow as tf\n\n\ndef validate_case_matches_checkpoint(do_lower_case, init_checkpoint):\n    \"\"\"Checks whether the casing config is consistent with the checkpoint name.\"\"\"\n\n    # The casing has to be passed in by the user and there is no explicit check\n    # as to whether it matches the checkpoint. The casing information probably\n    # should have been stored in the bert_config.json file, but it's not, so\n    # we have to heuristically detect it to validate.\n\n    if not init_checkpoint:\n        return\n\n    m = re.match(\"^.*?([A-Za-z0-9_-]+)/bert_model.ckpt\", init_checkpoint)\n    if m is None:\n        return\n\n    model_name = m.group(1)\n\n    lower_models = [\n        \"uncased_L-24_H-1024_A-16\", \"uncased_L-12_H-768_A-12\",\n        \"multilingual_L-12_H-768_A-12\", \"chinese_L-12_H-768_A-12\"\n    ]\n\n    cased_models = [\n        \"cased_L-12_H-768_A-12\", \"cased_L-24_H-1024_A-16\",\n        \"multi_cased_L-12_H-768_A-12\"\n    ]\n\n    is_bad_config = False\n    if model_name in lower_models and not do_lower_case:\n        is_bad_config = True\n        actual_flag = \"False\"\n        case_name = \"lowercased\"\n        opposite_flag = \"True\"\n\n    if model_name in cased_models and do_lower_case:\n        is_bad_config = True\n        actual_flag = \"True\"\n        case_name = \"cased\"\n        opposite_flag = \"False\"\n\n    if is_bad_config:\n        raise ValueError(\n            \"You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. \"\n            \"However, `%s` seems to be a %s model, so you \"\n            \"should pass in `--do_lower_case=%s` so that the fine-tuning matches \"\n            \"how the model was pre-training. If this error is wrong, please \"\n            \"just comment out this check.\" % (actual_flag, init_checkpoint,\n                                              model_name, case_name, opposite_flag))\n\n\ndef convert_to_unicode(text):\n    \"\"\"Converts `text` to Unicode (if it's not already), assuming utf-8 input.\"\"\"\n    if six.PY3:\n        if isinstance(text, str):\n            return text\n        elif isinstance(text, bytes):\n            return text.decode(\"utf-8\", \"ignore\")\n        else:\n            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n    elif six.PY2:\n        if isinstance(text, str):\n            return text.decode(\"utf-8\", \"ignore\")\n        elif isinstance(text, unicode):\n            return text\n        else:\n            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n    else:\n        raise ValueError(\"Not running on Python2 or Python 3?\")\n\n\ndef printable_text(text):\n    \"\"\"Returns text encoded in a way suitable for print or `tf.logging`.\"\"\"\n\n    # These functions want `str` for both Python2 and Python3, but in one case\n    # it's a Unicode string and in the other it's a byte string.\n    if six.PY3:\n        if isinstance(text, str):\n            return text\n        elif isinstance(text, bytes):\n            return text.decode(\"utf-8\", \"ignore\")\n        else:\n            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n    elif six.PY2:\n        if isinstance(text, str):\n            return text\n        elif isinstance(text, unicode):\n            return text.encode(\"utf-8\")\n        else:\n            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n    else:\n        raise ValueError(\"Not running on Python2 or Python 3?\")\n\n\ndef load_vocab(vocab_file):\n    \"\"\"Loads a vocabulary file into a dictionary.\"\"\"\n    vocab = collections.OrderedDict()\n    index = 0\n    with tf.io.gfile.GFile(vocab_file, \"r\") as reader:\n        while True:\n            token = convert_to_unicode(reader.readline())\n            if not token:\n                break\n            token = token.strip()\n            vocab[token] = index\n            index += 1\n    return vocab\n\n\ndef convert_by_vocab(vocab, items):\n    \"\"\"Converts a sequence of [tokens|ids] using the vocab.\"\"\"\n    output = []\n    for item in items:\n        output.append(vocab[item])\n    return output\n\n\ndef convert_tokens_to_ids(vocab, tokens):\n    return convert_by_vocab(vocab, tokens)\n\n\ndef convert_ids_to_tokens(inv_vocab, ids):\n    return convert_by_vocab(inv_vocab, ids)\n\n\ndef whitespace_tokenize(text):\n    \"\"\"Runs basic whitespace cleaning and splitting on a piece of text.\"\"\"\n    text = text.strip()\n    if not text:\n        return []\n    tokens = text.split()\n    return tokens\n\n\nclass FullTokenizer(object):\n    \"\"\"Runs end-to-end tokenziation.\"\"\"\n\n    def __init__(self, vocab_file, do_lower_case=True):\n        self.vocab = load_vocab(vocab_file)\n        self.inv_vocab = {v: k for k, v in self.vocab.items()}\n        self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)\n        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)\n\n    def tokenize(self, text):\n        split_tokens = []\n        for token in self.basic_tokenizer.tokenize(text):\n            for sub_token in self.wordpiece_tokenizer.tokenize(token):\n                split_tokens.append(sub_token)\n\n        return split_tokens\n\n    def convert_tokens_to_ids(self, tokens):\n        return convert_by_vocab(self.vocab, tokens)\n\n    def convert_ids_to_tokens(self, ids):\n        return convert_by_vocab(self.inv_vocab, ids)\n\n\nclass BasicTokenizer(object):\n    \"\"\"Runs basic tokenization (punctuation splitting, lower casing, etc.).\"\"\"\n\n    def __init__(self, do_lower_case=True):\n        \"\"\"Constructs a BasicTokenizer.\n        Args:\n          do_lower_case: Whether to lower case the input.\n        \"\"\"\n        self.do_lower_case = do_lower_case\n\n    def tokenize(self, text):\n        \"\"\"Tokenizes a piece of text.\"\"\"\n        text = convert_to_unicode(text)\n        text = self._clean_text(text)\n\n        # This was added on November 1st, 2018 for the multilingual and Chinese\n        # models. This is also applied to the English models now, but it doesn't\n        # matter since the English models were not trained on any Chinese data\n        # and generally don't have any Chinese data in them (there are Chinese\n        # characters in the vocabulary because Wikipedia does have some Chinese\n        # words in the English Wikipedia.).\n        text = self._tokenize_chinese_chars(text)\n\n        orig_tokens = whitespace_tokenize(text)\n        split_tokens = []\n        for token in orig_tokens:\n            if self.do_lower_case:\n                token = token.lower()\n                token = self._run_strip_accents(token)\n            split_tokens.extend(self._run_split_on_punc(token))\n\n        output_tokens = whitespace_tokenize(\" \".join(split_tokens))\n        return output_tokens\n\n    def _run_strip_accents(self, text):\n        \"\"\"Strips accents from a piece of text.\"\"\"\n        text = unicodedata.normalize(\"NFD\", text)\n        output = []\n        for char in text:\n            cat = unicodedata.category(char)\n            if cat == \"Mn\":\n                continue\n            output.append(char)\n        return \"\".join(output)\n\n    def _run_split_on_punc(self, text):\n        \"\"\"Splits punctuation on a piece of text.\"\"\"\n        chars = list(text)\n        i = 0\n        start_new_word = True\n        output = []\n        while i < len(chars):\n            char = chars[i]\n            if _is_punctuation(char):\n                output.append([char])\n                start_new_word = True\n            else:\n                if start_new_word:\n                    output.append([])\n                start_new_word = False\n                output[-1].append(char)\n            i += 1\n\n        return [\"\".join(x) for x in output]\n\n    def _tokenize_chinese_chars(self, text):\n        \"\"\"Adds whitespace around any CJK character.\"\"\"\n        output = []\n        for char in text:\n            cp = ord(char)\n            if self._is_chinese_char(cp):\n                output.append(\" \")\n                output.append(char)\n                output.append(\" \")\n            else:\n                output.append(char)\n        return \"\".join(output)\n\n    def _is_chinese_char(self, cp):\n        \"\"\"Checks whether CP is the codepoint of a CJK character.\"\"\"\n        # This defines a \"chinese character\" as anything in the CJK Unicode block:\n        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)\n        #\n        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,\n        # despite its name. The modern Korean Hangul alphabet is a different block,\n        # as is Japanese Hiragana and Katakana. Those alphabets are used to write\n        # space-separated words, so they are not treated specially and handled\n        # like the all of the other languages.\n        if ((cp >= 0x4E00 and cp <= 0x9FFF) or  #\n                (cp >= 0x3400 and cp <= 0x4DBF) or  #\n                (cp >= 0x20000 and cp <= 0x2A6DF) or  #\n                (cp >= 0x2A700 and cp <= 0x2B73F) or  #\n                (cp >= 0x2B740 and cp <= 0x2B81F) or  #\n                (cp >= 0x2B820 and cp <= 0x2CEAF) or\n                (cp >= 0xF900 and cp <= 0xFAFF) or  #\n                (cp >= 0x2F800 and cp <= 0x2FA1F)):  #\n            return True\n\n        return False\n\n    def _clean_text(self, text):\n        \"\"\"Performs invalid character removal and whitespace cleanup on text.\"\"\"\n        output = []\n        for char in text:\n            cp = ord(char)\n            if cp == 0 or cp == 0xfffd or _is_control(char):\n                continue\n            if _is_whitespace(char):\n                output.append(\" \")\n            else:\n                output.append(char)\n        return \"\".join(output)\n\n\nclass WordpieceTokenizer(object):\n    \"\"\"Runs WordPiece tokenziation.\"\"\"\n\n    def __init__(self, vocab, unk_token=\"[UNK]\", max_input_chars_per_word=200):\n        self.vocab = vocab\n        self.unk_token = unk_token\n        self.max_input_chars_per_word = max_input_chars_per_word\n\n    def tokenize(self, text):\n        \"\"\"Tokenizes a piece of text into its word pieces.\n        This uses a greedy longest-match-first algorithm to perform tokenization\n        using the given vocabulary.\n        For example:\n          input = \"unaffable\"\n          output = [\"un\", \"##aff\", \"##able\"]\n        Args:\n          text: A single token or whitespace separated tokens. This should have\n            already been passed through `BasicTokenizer.\n        Returns:\n          A list of wordpiece tokens.\n        \"\"\"\n\n        text = convert_to_unicode(text)\n\n        output_tokens = []\n        for token in whitespace_tokenize(text):\n            chars = list(token)\n            if len(chars) > self.max_input_chars_per_word:\n                output_tokens.append(self.unk_token)\n                continue\n\n            is_bad = False\n            start = 0\n            sub_tokens = []\n            while start < len(chars):\n                end = len(chars)\n                cur_substr = None\n                while start < end:\n                    substr = \"\".join(chars[start:end])\n                    if start > 0:\n                        substr = \"##\" + substr\n                    if substr in self.vocab:\n                        cur_substr = substr\n                        break\n                    end -= 1\n                if cur_substr is None:\n                    is_bad = True\n                    break\n                sub_tokens.append(cur_substr)\n                start = end\n\n            if is_bad:\n                output_tokens.append(self.unk_token)\n            else:\n                output_tokens.extend(sub_tokens)\n        return output_tokens\n\n\ndef _is_whitespace(char):\n    \"\"\"Checks whether `chars` is a whitespace character.\"\"\"\n    # \\t, \\n, and \\r are technically contorl characters but we treat them\n    # as whitespace since they are generally considered as such.\n    if char == \" \" or char == \"\\t\" or char == \"\\n\" or char == \"\\r\":\n        return True\n    cat = unicodedata.category(char)\n    if cat == \"Zs\":\n        return True\n    return False\n\n\ndef _is_control(char):\n    \"\"\"Checks whether `chars` is a control character.\"\"\"\n    # These are technically control characters but we count them as whitespace\n    # characters.\n    if char == \"\\t\" or char == \"\\n\" or char == \"\\r\":\n        return False\n    cat = unicodedata.category(char)\n    if cat in (\"Cc\", \"Cf\"):\n        return True\n    return False\n\n\ndef _is_punctuation(char):\n    \"\"\"Checks whether `chars` is a punctuation character.\"\"\"\n    cp = ord(char)\n    # We treat all non-letter/number ASCII as punctuation.\n    # Characters such as \"^\", \"$\", and \"`\" are not in the Unicode\n    # Punctuation class but we treat them as punctuation anyways, for\n    # consistency.\n    if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or\n            (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):\n        return True\n    cat = unicodedata.category(char)\n    if cat.startswith(\"P\"):\n        return True\n    return False","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import GroupKFold\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nimport tensorflow_hub as hub\nimport tensorflow as tf\n#import bert_tokenization as tokenization\nimport tensorflow.keras.backend as K\nimport gc\nimport os\nfrom scipy.stats import spearmanr\nfrom math import floor, ceil\nfrom tensorflow.keras.models import load_model\n\nnp.set_printoptions(suppress=True)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"PATH = '../input/google-quest-challenge/'\nBERT_PATH = '../input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'\ntokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)\nMAX_SEQUENCE_LENGTH = 512\n\ndf_train = pd.read_csv(PATH+'train.csv')\ndf_test = pd.read_csv(PATH+'test.csv')\ndf_sub = pd.read_csv(PATH+'sample_submission.csv')\nprint('train shape =', df_train.shape)\nprint('test shape =', df_test.shape)\n\noutput_categories = list(df_train.columns[11:])\ninput_categories = list(df_train.columns[[1,2,5]])\nprint('\\noutput categories:\\n\\t', output_categories)\nprint('\\ninput categories:\\n\\t', input_categories)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"def _get_masks(tokens, max_seq_length):\n    \"\"\"Mask for padding\"\"\"\n    if len(tokens)>max_seq_length:\n        raise IndexError(\"Token length more than max seq length!\")\n    return [1]*len(tokens) + [0] * (max_seq_length - len(tokens))\n\ndef _get_segments(tokens, max_seq_length):\n    \"\"\"Segments: 0 for the first sequence, 1 for the second\"\"\"\n    if len(tokens)>max_seq_length:\n        raise IndexError(\"Token length more than max seq length!\")\n    segments = []\n    first_sep = True\n    current_segment_id = 0\n    for token in tokens:\n        segments.append(current_segment_id)\n        if token == \"[SEP]\":\n            if first_sep:\n                first_sep = False \n            else:\n                current_segment_id = 1\n    return segments + [0] * (max_seq_length - len(tokens))\n\ndef _get_ids(tokens, tokenizer, max_seq_length):\n    \"\"\"Token ids from Tokenizer vocab\"\"\"\n    token_ids = tokenizer.convert_tokens_to_ids(tokens)\n    input_ids = token_ids + [0] * (max_seq_length-len(token_ids))\n    return input_ids\n\ndef _trim_input(title, question, answer, max_sequence_length, \n                t_max_len=30, q_max_len=239, a_max_len=239):\n\n    t = tokenizer.tokenize(title)\n    q = tokenizer.tokenize(question)\n    a = tokenizer.tokenize(answer)\n    \n    t_len = len(t)\n    q_len = len(q)\n    a_len = len(a)\n\n    if (t_len+q_len+a_len+4) > max_sequence_length:\n        \n        if t_max_len > t_len:\n            t_new_len = t_len\n            a_max_len = a_max_len + floor((t_max_len - t_len)/2)\n            q_max_len = q_max_len + ceil((t_max_len - t_len)/2)\n        else:\n            t_new_len = t_max_len\n      \n        if a_max_len > a_len:\n            a_new_len = a_len \n            q_new_len = q_max_len + (a_max_len - a_len)\n        elif q_max_len > q_len:\n            a_new_len = a_max_len + (q_max_len - q_len)\n            q_new_len = q_len\n        else:\n            a_new_len = a_max_len\n            q_new_len = q_max_len\n            \n            \n        if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:\n            raise ValueError(\"New sequence length should be %d, but is %d\" \n                             % (max_sequence_length, (t_new_len+a_new_len+q_new_len+4)))\n        \n        t = t[:t_new_len]\n        q = q[:q_new_len]\n        a = a[:a_new_len]\n    \n    return t, q, a\n\ndef _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):\n    \"\"\"Converts tokenized input to ids, masks and segments for BERT\"\"\"\n    \n    stoken = [\"[CLS]\"] + title + [\"[SEP]\"] + question + [\"[SEP]\"] + answer + [\"[SEP]\"]\n\n    input_ids = _get_ids(stoken, tokenizer, max_sequence_length)\n    input_masks = _get_masks(stoken, max_sequence_length)\n    input_segments = _get_segments(stoken, max_sequence_length)\n\n    return [input_ids, input_masks, input_segments]\n\ndef compute_input_arays(df, columns, tokenizer, max_sequence_length):\n    input_ids, input_masks, input_segments = [], [], []\n    for _, instance in tqdm(df[columns].iterrows()):\n        t, q, a = instance.question_title, instance.question_body, instance.answer\n\n        t, q, a = _trim_input(t, q, a, max_sequence_length)\n\n        ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)\n        input_ids.append(ids)\n        input_masks.append(masks)\n        input_segments.append(segments)\n        \n    return [np.asarray(input_ids, dtype=np.int32), \n            np.asarray(input_masks, dtype=np.int32), \n            np.asarray(input_segments, dtype=np.int32)]\n\n\ndef compute_output_arrays(df, columns):\n    return np.asarray(df[columns])","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"def compute_spearmanr(trues, preds):\n    rhos = []\n    for col_trues, col_pred in zip(trues.T, preds.T):\n        rhos.append(\n            spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)\n    return np.nanmean(rhos)\n\n\nclass CustomCallback(tf.keras.callbacks.Callback):\n    \n    def __init__(self, valid_data, test_data, batch_size=16, fold=None):\n\n        self.valid_inputs = valid_data[0]\n        self.valid_outputs = valid_data[1]\n        self.test_inputs = test_data\n        \n        self.batch_size = batch_size\n        self.fold = fold\n        \n    def on_train_begin(self, logs={}):\n        self.valid_predictions = []\n        self.test_predictions = []\n        \n    def on_epoch_end(self, epoch, logs={}):\n        self.valid_predictions.append(\n            self.model.predict(self.valid_inputs, batch_size=self.batch_size))\n        \n        rho_val = compute_spearmanr(\n            self.valid_outputs, np.average(self.valid_predictions, axis=0))\n        \n        print(\"\\nvalidation rho: %.4f\" % rho_val)\n        \n        if self.fold is not None:\n            self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')\n        \n        self.test_predictions.append(\n            self.model.predict(self.test_inputs, batch_size=self.batch_size)\n        )\n\ndef bert_model():\n    \n    input_word_ids = tf.keras.layers.Input(\n        (MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')\n    input_masks = tf.keras.layers.Input(\n        (MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')\n    input_segments = tf.keras.layers.Input(\n        (MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')\n    \n    bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)\n    \n    _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])\n    \n    x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)\n    x = tf.keras.layers.Dropout(0.2)(x)\n    out = tf.keras.layers.Dense(30, activation=\"sigmoid\", name=\"dense_output\")(x)\n\n    model = tf.keras.models.Model(\n        inputs=[input_word_ids, input_masks, input_segments], outputs=out)\n    \n    return model    \n        \ndef train_and_predict(model, train_data, valid_data, test_data, \n                      learning_rate, epochs, batch_size, loss_function, fold):\n        \n    custom_callback = CustomCallback(\n        valid_data=(valid_data[0], valid_data[1]), \n        test_data=test_data,\n        batch_size=batch_size,\n        fold=None)\n\n    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n    model.compile(loss=loss_function, optimizer=optimizer)\n    model.fit(train_data[0], train_data[1], epochs=epochs, \n              batch_size=batch_size, callbacks=[custom_callback])\n    \n    return custom_callback","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"gkf = GroupKFold(n_splits=10).split(X=df_train.question_body, groups=df_train.question_body)\n\ntest_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"models = []\nfor i in range(5):\n    model_path = f'../input/bertuned-f{i}/bertuned_f{i}.h5'\n    model = bert_model()\n    model.load_weights(model_path)\n    models.append(model)\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"model_path = f'../input/bertf1e15/Full-0.h5'\nmodel = bert_model()\nmodel.load_weights(model_path)\nmodels.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model_path = f'../input/bertmodelpretrained/bert-0.h5'\nmodel = bert_model()\nmodel.load_weights(model_path)\nmodels.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model_path = f'../input/bertmodelpretrained/bert-1.h5'\nmodel = bert_model()\nmodel.load_weights(model_path)\nmodels.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"for i in range(7):\n    model_path = f'../input/pretrained-bert/bert-{i}.h5'\n    model = bert_model()\n    model.load_weights(model_path)\n    models.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"len(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test_predictions = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"for model in models:\n    test_predictions.append(model.predict(test_inputs, batch_size=8)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test_predictions[i].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"final_predictions = np.mean(test_predictions, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"final_predictions.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_sub.iloc[:, 1:] = final_predictions\ndf_sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_sub.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"5ecb732c8804402fb38e2068b6168502":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_c3f92031bb6148b89a11a76e3dc2f4fd","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_6ac6dc0a9c9747cb9e62f08491735538","value":1}},"6ac6dc0a9c9747cb9e62f08491735538":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"758ce28f73804e748f073368e708d216":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_9dad433972994770868c742f1b7b7e25","placeholder":"​","style":"IPY_MODEL_a6016efb97d843c6b1472a4fc132026c","value":" 476/? [00:05&lt;00:00, 82.31it/s]"}},"9dad433972994770868c742f1b7b7e25":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a6016efb97d843c6b1472a4fc132026c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"c3f92031bb6148b89a11a76e3dc2f4fd":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"cccd2a5481ce498ea442caacd1701359":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_5ecb732c8804402fb38e2068b6168502","IPY_MODEL_758ce28f73804e748f073368e708d216"],"layout":"IPY_MODEL_cf2087134d6a41a6ae683187b87d7a97"}},"cf2087134d6a41a6ae683187b87d7a97":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":1}