{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This kernel provides the required datasets and commands to setup Hugging Face Transformers setup in offline mode. You can find the required github codebases in the datasets.\n\n- sacremoses dependency - https://www.kaggle.com/axel81/sacremoses\n- transformers - https://www.kaggle.com/axel81/transformers\n- config - \n\nThanks to axel81 and sakami."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install ../input/sacremoses/sacremoses-master\n!pip install ../input/transformers/transformers-master","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nimport torch \nimport random\nimport numpy as np\n\nimport torch \nimport torch.optim as optim\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom pathlib import Path\n\nfrom torch.utils.data import DataLoader, TensorDataset, RandomSampler\nfrom transformers import BertTokenizer, BertConfig\n\nfrom collections import defaultdict\nfrom dataclasses import dataclass\nimport functools\nimport gc\nimport itertools\nimport json\nfrom multiprocessing import Pool\nimport os\nfrom pathlib import Path\nimport random\nimport re\nimport shutil\nimport subprocess\nimport time\nfrom typing import Callable, Dict, List, Generator, Tuple\n\nimport numpy as np\nimport pandas as pd\n\n# from pandas.io.json._json import JsonReader\nfrom pandas.io.json._json import JsonReader\nfrom sklearn.preprocessing import LabelEncoder\nfrom tqdm import tqdm\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import Dataset, Subset, DataLoader\n\nfrom sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_number_of_trainable_parameters(model):\n    model_parameters = filter(lambda p: p.requires_grad, model.parameters())\n    params = sum([np.prod(p.size()) for p in model_parameters])\n    print(\"Number of trainable parameters in the model are : {}\".format(params))\n    return\n\n\nclass Example(object):\n    def __init__(\n        self,\n        example_id,\n        doc_start,\n        question_len,\n        text_len,\n        input_text_ids,\n        input_question_ids,\n        start_position,\n        end_position,\n        class_label,\n        doc_position\n    ):\n        self.example_id = example_id\n        self.doc_start = doc_start\n        self.question_len = question_len\n        self.text_len = text_len\n        self.input_text_ids = input_text_ids\n        self.input_question_ids = input_question_ids\n        self.start_position = start_position\n        self.end_position = end_position\n        self.class_label = class_label\n        self.doc_position = doc_position\n\n        \ndef convert_data(\n    line: str,\n    tokenizer: object,\n    max_seq_len: int,\n    max_question_len: int,\n    doc_stride: int\n) -> List[Example]:\n    \"\"\"Convert dictionary data into list of training data.\n\n    Parameters\n    ----------\n    line : str\n        Training data.\n    tokenizer : transformers.BertTokenizer\n        Tokenizer for encoding texts into ids.\n    max_seq_len : int\n        Maximum input sequence length.\n    max_question_len : int\n        Maximum input question length.\n    doc_stride : int\n        When splitting up a long document into chunks, how much stride to take between chunks.\n    \"\"\"\n\n    def _find_short_range(short_answers: List[Dict]) -> Tuple[int, int]:\n        answers = pd.DataFrame(short_answers)\n        start_min = answers['start_token'].min()\n        end_max = answers['end_token'].max()\n        return start_min, end_max\n\n    # model input\n    data = json.loads(line)\n    doc_words = data['document_text'].split()\n    question_tokens = tokenizer.tokenize(data['question_text'])\n    question_len = len(question_tokens)\n    if len(question_tokens) > max_question_len:\n        question_tokens = question_tokens[:max_question_len]\n        question_len = max_question_len\n    else:\n        question_tokens = question_tokens + [tokenizer.pad_token]*(max_question_len - question_len)\n        \n\n    # tokenized index of i-th original token corresponds to original_to_tokenized_index[i]\n    # if a token in original text is removed, its tokenized index indicates next token\n    original_to_tokenized_index = []\n    tokenized_to_original_index = []\n    all_doc_tokens = []  # tokenized document text\n    for i, word in enumerate(doc_words):\n        original_to_tokenized_index.append(len(all_doc_tokens))\n        if re.match(r'<.+>', word):  # remove paragraph tag\n            continue\n        sub_tokens = tokenizer.tokenize(word)\n        for sub_token in sub_tokens:\n            tokenized_to_original_index.append(i)\n            all_doc_tokens.append(sub_token)\n\n    # model output: (class_label, start_position, end_position)\n    annotations = data['annotations'][0]\n    if annotations['yes_no_answer'] in ['YES', 'NO']:\n        class_label = annotations['yes_no_answer'].lower()\n        start_position = annotations['long_answer']['start_token']\n        end_position = annotations['long_answer']['end_token']\n    elif annotations['short_answers']:\n        class_label = 'short'\n        start_position, end_position = _find_short_range(annotations['short_answers'])\n    elif annotations['long_answer']['candidate_index'] != -1:\n        class_label = 'long'\n        start_position = annotations['long_answer']['start_token']\n        end_position = annotations['long_answer']['end_token']\n    else:\n        class_label = 'no_answer'\n        start_position = -1\n        end_position = -1\n\n    # convert into tokenized index\n    if start_position != -1 and end_position != -1:\n        start_position = original_to_tokenized_index[start_position]\n        end_position = original_to_tokenized_index[end_position]\n\n    # make sure at least one object in `examples`\n    examples = []\n    \n    # take chunks with a stride of `doc_stride`\n    for doc_idx, doc_start in enumerate(range(0, len(all_doc_tokens), doc_stride)):\n        doc_end = doc_start + max_seq_len\n        # if truncated document does not contain annotated range\n        if not (doc_start <= start_position and end_position <= doc_end):\n            start, end, label = -1, -1, 'no_answer'\n        else:\n            start = start_position - doc_start \n            end = end_position - doc_start\n            label = class_label\n\n        assert -1 <= start <= max_seq_len, f'start position is out of range: {start}'\n        assert -1 <= end <= max_seq_len, f'end position is out of range: {end}'\n\n        text_tokens = all_doc_tokens[doc_start:doc_end]\n        text_len = len(text_tokens)\n        if text_len > max_seq_len:\n            text_tokens = text_tokens[:max_seq_len]\n            text_len = max_seq_len\n        else:\n            text_tokens = text_tokens + [tokenizer.pad_token]*(max_seq_len - text_len)\n\n        examples.append(\n            Example(\n                example_id=data['example_id'],\n                doc_start=doc_start,\n                question_len=question_len,\n                text_len = text_len,\n                input_text_ids=tokenizer.convert_tokens_to_ids(text_tokens),\n                input_question_ids=tokenizer.convert_tokens_to_ids(question_tokens),\n                start_position=start,\n                end_position=end,\n                class_label=label,\n                doc_position=doc_idx / (len(all_doc_tokens) / doc_stride)\n            )\n        )\n    return examples\n\n\nclass JsonChunkReader(JsonReader):\n    \"\"\"JsonReader provides an interface for reading in a JSON file.\n    \"\"\"\n    \n    def __init__(\n        self,\n        filepath_or_buffer: str,\n        convert_data: Callable[[str], List[Example]],\n        orient: str = None,\n        typ: str = 'frame',\n        dtype: bool = None,\n        convert_axes: bool = None,\n        convert_dates: bool = True,\n        keep_default_dates: bool = True,\n        numpy: bool = False,\n        precise_float: bool = False,\n        date_unit: str = None,\n        encoding: str = None,\n        lines: bool = True,\n        chunksize: int = 2000,\n        compression: str = None,\n    ):\n        super(JsonChunkReader, self).__init__(\n            str(filepath_or_buffer),\n            orient=orient, typ=typ, dtype=dtype,\n            convert_axes=convert_axes,\n            convert_dates=convert_dates,\n            keep_default_dates=keep_default_dates,\n            numpy=numpy, precise_float=precise_float,\n            date_unit=date_unit, encoding=encoding,\n            lines=lines, chunksize=chunksize,\n            compression=compression\n        )\n        self.convert_data = convert_data\n        \n    def __next__(self):\n        lines = list(itertools.islice(self.data, self.chunksize))\n        if lines:\n            with Pool(2) as p:\n                obj = p.map(self.convert_data, lines)\n            return obj\n\n        self.close()\n        raise StopIteration\n        \n\nclass TextDataset(Dataset):\n    \"\"\"Dataset for [TensorFlow 2.0 Question Answering](https://www.kaggle.com/c/tensorflow2-question-answering).\n    \n    Parameters\n    ----------\n    examples : list of Example\n        The whole Dataset.\n    \"\"\"\n    \n    def __init__(self, examples: List[Example]):\n        self.examples = examples\n        \n    def __len__(self) -> int:\n        return len(self.examples)\n    \n    def __getitem__(self, index):\n        annotated = list(\n            filter(lambda example: example.class_label != 'no_answer', self.examples[index]))\n        if len(annotated) == 0:\n            return random.choice(self.examples[index])\n        return random.choice(annotated)\n\n    \ndef collate_fn(examples: List[Example]) -> List[List[torch.Tensor]]:\n    # input tokens\n    max_len_text = max([len(example.input_text_ids) for example in examples])\n    max_len_ques = max([len(example.input_question_ids) for example in examples])\n    \n    # lengths \n    question_lengths = np.array([example.question_len for example in examples])\n    text_lengths = np.array([example.text_len for example in examples])\n    doc_positions = np.array([example.doc_position for example in examples])\n    \n    # tokens\n    text_tokens = np.zeros((len(examples), max_len_text), dtype=np.int64)\n    question_tokens = np.zeros((len(examples), max_len_ques), dtype=np.int64)\n    for i, example in enumerate(examples):\n        # text tokens \n        row = example.input_text_ids\n        text_tokens[i, :len(row)] = row\n        # question tokens \n        row2 = example.input_question_ids\n        question_tokens[i, :len(row2)] = row2\n         \n    # output labels\n    start_positions = np.array([example.start_position for example in examples])\n    end_positions = np.array([example.end_position for example in examples])\n    start_positions = np.where(start_positions >= max_len_text, -1, start_positions)\n    end_positions = np.where(end_positions >= max_len_text, -1, end_positions)\n    \n    all_labels = ['long', 'no', 'short', 'no_answer', 'yes']\n    class_labels = [all_labels.index(example.class_label) for example in examples]\n\n    input_and_labels = [\n        torch.from_numpy(text_tokens),\n        torch.from_numpy(question_tokens),\n        torch.from_numpy(text_lengths),\n        torch.from_numpy(question_lengths),\n        torch.from_numpy(doc_positions),\n        torch.LongTensor(start_positions),\n        torch.LongTensor(end_positions),\n        torch.LongTensor(class_labels)\n    ]\n\n    return input_and_labels\n\ndef triple_loss_function(\n    preds, labels, st_crtierion,\n    en_criterion, class_criterion\n):\n    start_preds, end_preds, class_preds = preds\n    start_labels, end_labels, class_labels = labels\n    \n    start_loss = st_crtierion(start_preds, start_labels)\n    end_loss = en_criterion(end_preds, end_labels)\n    class_loss = class_criterion(class_preds, class_labels)\n    return start_loss + end_loss + class_loss\n\ndef get_results_dict(preds, labels):\n    start_preds, end_preds, class_preds = preds\n    start_labels, end_labels, class_labels = labels\n    results = {\n#         \"f1_start_tokens\": f1_score(start_labels, start_preds),\n#         \"f1_end_tokens\": f1_score(end_labels, end_preds),\n        \"f1_class_labels\": f1_score(class_labels, class_preds, average='macro'),\n        \"acc_s_tok\": accuracy_score(start_labels, start_preds),\n        \"acc_e_tok\": accuracy_score(end_labels, end_preds),\n        \"acc_c_lab\": accuracy_score(class_labels, class_preds)\n    }\n    return results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Attention(nn.Module):\n    \"\"\"\n    Computes a weighted average of channels across timesteps (1 parameter pr. channel).\n    \"\"\"\n    def __init__(\n        self, attention_size,\n        pad_token_id,\n        device\n    ):\n        super(Attention, self).__init__()\n        self.attention_size = attention_size\n        self.pad_token_id = pad_token_id\n        self.device = device\n\n        self.attention = nn.Parameter(torch.rand(attention_size))\n\n    def forward(self, inputs, input_lengths):\n        # inputs = [batch_size, max_seq_length, attention_size]\n        # input_lengths = [batch_size]\n\n        max_seq_length = inputs.shape[1]\n\n        attn = torch.matmul(inputs, self.attention)\n        # attn = [batch_size, max_seq_len]\n\n        idxes = torch.arange(0, max_seq_length, out=torch.LongTensor(max_seq_length)).unsqueeze(0).to(self.device)\n        mask = torch.autograd.Variable((idxes < input_lengths.unsqueeze(1)).float()).to(self.device)\n        # mask = [batch_size, max_seq_length]\n        # idxes = [batch_size, max_seq_length]\n\n        attn_masked = attn.masked_fill(mask == self.pad_token_id, -1e10)\n        attention_weights = F.softmax(attn_masked, dim=1)\n        # attention_weights = [batch_size, max_seq_length]\n\n        # apply attention weights\n        weighted = torch.bmm(attention_weights.unsqueeze(1), inputs)\n        # weighted = [batch_size, 1, attention_size]\n\n        weighted = weighted.squeeze(1)\n        # weighted_outputs = [batch_size, attention_size]\n\n        return (weighted, attention_weights)\n    \n\nclass AttentiveBilstm(nn.Module):\n    def __init__(\n        self, max_seq_length, embedding_dim,\n        attention_size, pad_token_id,\n        hidden_size, num_layers,\n        output_dim, bidirectional,\n        dropout_ratio=0.2,\n        common_embedding_layer=None,\n        device=torch.device(\"cpu\")\n    ):\n        super(AttentiveBilstm, self).__init__()\n        self.max_seq_length = max_seq_length\n        self.embedding_dim = embedding_dim\n        self.hidden_size = hidden_size\n        self.attention_size = attention_size\n        self.pad_token_id = pad_token_id\n        self.num_layers = num_layers\n        self.output_dim = output_dim\n        self.bidirectional = bidirectional\n        self.dropout_ratio = dropout_ratio\n        self.device = device\n\n        self.lstm_layer = nn.LSTM(\n            input_size=embedding_dim,\n            num_layers=num_layers,\n            hidden_size=hidden_size,\n            bidirectional=bidirectional,\n            dropout=dropout_ratio\n        )\n            \n        if bidirectional is True:\n            self.hidden2attention = nn.Linear(\n                2*hidden_size, attention_size\n            )\n            self.attention_layer = Attention(\n                attention_size=attention_size,\n                pad_token_id=pad_token_id,\n                device=self.device\n            )\n        else:\n            self.hidden2attention = nn.Linear(\n                1*hidden_size, attention_size\n            )\n            self.attention_layer = Attention(\n                attention_size=attention_size,\n                pad_token_id=pad_token_id,\n                device=self.device\n            )\n\n        self.attention2output = nn.Linear(\n            attention_size, output_dim\n        )\n        self.dropout_layer = nn.Dropout(dropout_ratio)\n    \n    def forward(self, ques_embedded, seq_lengths):\n        # question = [batch_size, max_seq_length, embedding_dim]\n        # seq_lengths = [batch_size]\n\n        # permuting for pad packed easiness\n        embedded = ques_embedded.permute(1, 0, 2)\n        # embedded = [max_seq_length, batch_size, embedding_dim]\n\n        packed_embedded = nn.utils.rnn.pack_padded_sequence(\n            embedded, seq_lengths, enforce_sorted=False\n        )\n    \n        packed_outputs, (_, _) = self.lstm_layer(packed_embedded)\n\n        outputs, _ = nn.utils.rnn.pad_packed_sequence(packed_outputs, total_length=self.max_seq_length)\n        # outputs = [max_seq_length, batch_size, num_directions*hidden_size]\n\n        # using hidden2attention\n        outputs = self.hidden2attention(outputs.permute(1, 0, 2))\n        # outputs = [batch_size, max_seq_length, attention_size]\n\n        # outputs are permuted again because attention layer needs batch_first\n        (weighted_outputs, attention_weights) = self.attention_layer(outputs, seq_lengths)\n        # weighted_outputs = [batch_size, attention_size]\n\n        outputs = self.attention2output(weighted_outputs)\n\n        return (outputs, attention_weights)\n    \n\n# Stanford Attentive Reader\n# will be used in start and end token prediction\nclass StanfordAttentiveReader(nn.Module):\n    def __init__(\n        self, max_seq_length, embedding_dim,\n        question_encoding_dim, pad_token_id,\n        hidden_size, num_layers,\n        bidirectional, dropout_ratio=0.2,\n        common_embedding_layer=None,\n        device=torch.device(\"cpu\")\n    ):\n        super(StanfordAttentiveReader, self).__init__()\n        self.max_seq_length = max_seq_length        \n        self.embedding_dim = embedding_dim\n        self.question_encoding_dim = question_encoding_dim\n        self.pad_token_id = pad_token_id\n        self.hidden_size = hidden_size\n        self.num_layers = num_layers\n        self.bidirectional = bidirectional\n        self.dropout_ratio = dropout_ratio\n        self.device = device\n        \n        self.lstm_layer = nn.LSTM(\n            input_size=embedding_dim,\n            num_layers=num_layers,\n            hidden_size=hidden_size,\n            bidirectional=bidirectional,\n            dropout=dropout_ratio\n        )\n\n        if bidirectional:\n            self.bilinear_attention_start = nn.Parameter(\n                torch.rand(\n                    question_encoding_dim,\n                    2*hidden_size\n                )\n            )\n            self.bilinear_attention_end = nn.Parameter(\n                torch.rand(\n                    question_encoding_dim,\n                    2*hidden_size\n                )\n            )\n        else:\n            self.bilinear_attention_start = nn.Parameter(\n                torch.rand(\n                    question_encoding_dim,\n                    hidden_size\n                )\n            )\n            self.bilinear_attention_end = nn.Parameter(\n                torch.rand(\n                    question_encoding_dim,\n                    hidden_size\n                )\n            )\n        \n        self.dropout_layer = nn.Dropout(dropout_ratio)\n\n    def forward(\n        self, text_embedded=None, ques_encoded=None,\n        text_lengths=None\n    ):\n        # text_embedded = [batch_size, max_text_seq_length, embedding_dim] \n        # ques_encoded = [batch_size, question_encoding_dim]\n        # text_lengths = [batch_size]\n\n        # permuting for pad packed easiness\n        embedded = text_embedded.permute(1, 0, 2)\n        # embedded = [max_text_seq_length, batch_size, embedding_dim]\n\n        packed_embedded = nn.utils.rnn.pack_padded_sequence(\n            embedded, text_lengths, enforce_sorted=False\n        )\n    \n        packed_outputs, (_, _) = self.lstm_layer(packed_embedded)\n\n        outputs, _ = nn.utils.rnn.pad_packed_sequence(packed_outputs, total_length=self.max_seq_length)\n        # outputs = [max_text_seq_length, batch_size, num_directions*hidden_size]\n\n        outputs = outputs.permute(1, 2, 0)\n        # outputs = [batch_size, num_directions*hidden_size, max_text_seq_length]\n\n        # calculating bilinear attentions\n        # for starts\n        start_outputs = self.calculate_bilinear_attention(\n            self.bilinear_attention_start, outputs,\n            ques_encoded, text_lengths\n        )\n        # for ends\n        end_outputs = self.calculate_bilinear_attention(\n            self.bilinear_attention_end, outputs,\n            ques_encoded, text_lengths\n        )\n        # start_outputs = [batch_size, max_text_seq_len]\n        # end_outputs = [batch_size, max_text_seq_len]\n        return [start_outputs, end_outputs]\n    \n    def calculate_bilinear_attention(\n        self, bilinear_attention_weights, lstm_outputs,\n        ques_encoding, text_lengths\n    ):  \n        ques_encoding = ques_encoding.unsqueeze(1)\n        # ques_encoding = []\n        \n        energy = torch.matmul(ques_encoding, bilinear_attention_weights)\n        # energy = [batch_size, 1, 2*hidden_size]\n\n        attended = torch.matmul(energy, lstm_outputs).squeeze(1)\n        # attended = [batch_size, max_seq_text_length]\n\n        # preparing masks\n        idxes = torch.arange(0, self.max_seq_length, out=torch.LongTensor(self.max_seq_length)).unsqueeze(0).to(self.device)\n        mask = torch.autograd.Variable((idxes < text_lengths.unsqueeze(1)).float()).to(self.device)\n        # mask = [batch_size, max_seq_length]\n        # idxes = [batch_size, max_seq_length]\n\n        attention_masked = attended.masked_fill(mask == self.pad_token_id, -1e10)\n        # attention_masked = [batch_size, max_text_seq_length]\n\n        attention_out = F.softmax(attention_masked, dim=1)\n        # attention_out = [batch_size, max_text_seq_length]\n        return attention_out\n    \n\nclass QALstmJointModel(nn.Module):\n    def __init__(\n        self,\n        vocab_size, embedding_dim, num_labels,\n        linear_layer_out, dropout_ratio,\n\n        max_ques_seq_length, ques_pad_token_id,\n        ques_encoding_attention_size,\n        ques_encoding_dim,\n        ques_enc_hidden_size, ques_enc_num_layers,\n        ques_enc_bidirectional, ques_enc_dropout_ratio,\n\n        max_text_seq_length,\n        text_encoding_attention_size, text_encoding_dim,\n        text_pad_token_id, \n        text_enc_hidden_size, text_enc_num_layers,\n        text_enc_bidirectional, text_enc_dropout_ratio,\n        text_reader_hidden_size, text_reader_num_layers,\n        text_reader_bidirectional, text_reader_dropout_ratio,\n        device=torch.device(\"cpu\")\n    ):\n        super(QALstmJointModel, self).__init__()\n        self.max_ques_seq_length = max_ques_seq_length\n        self.max_text_seq_length = max_text_seq_length\n        self.text_encoding_dim = text_encoding_dim\n        self.ques_encoding_dim = ques_encoding_dim\n        self.device = device\n\n        # common embedding layer    \n        self.embedding_layer = nn.Embedding(vocab_size, embedding_dim)\n        # self.embedding_layer_norm = nn.LayerNorm(embedding_dim)\n\n        # question encoder\n        self.question_encoder = AttentiveBilstm(\n            max_seq_length=max_ques_seq_length,\n            embedding_dim=embedding_dim,\n            attention_size=ques_encoding_attention_size,\n            pad_token_id=ques_pad_token_id,\n            hidden_size=ques_enc_hidden_size, num_layers=ques_enc_num_layers,\n            output_dim=ques_encoding_dim,\n            bidirectional=ques_enc_bidirectional, dropout_ratio=ques_enc_dropout_ratio,\n            common_embedding_layer=self.embedding_layer,\n            device=self.device\n        )\n        \n        # text encoder\n        self.text_encoder = AttentiveBilstm(\n            max_seq_length=max_text_seq_length,\n            embedding_dim=embedding_dim,\n            attention_size=text_encoding_attention_size,\n            pad_token_id=text_pad_token_id,\n            hidden_size=text_enc_hidden_size,\n            num_layers=text_enc_num_layers,\n            output_dim=text_encoding_dim,\n            bidirectional=text_enc_bidirectional,\n            dropout_ratio=text_enc_dropout_ratio,\n            common_embedding_layer=self.embedding_layer,\n            device=self.device\n        )\n\n        # attentive reader\n        self.reader = StanfordAttentiveReader(\n            max_seq_length=max_text_seq_length,\n            embedding_dim=embedding_dim,\n            question_encoding_dim=ques_encoding_dim,\n            pad_token_id=text_pad_token_id,\n            hidden_size=text_reader_hidden_size,\n            num_layers=text_reader_num_layers,\n            bidirectional=text_reader_bidirectional,\n            dropout_ratio=text_reader_dropout_ratio,\n            common_embedding_layer=self.embedding_layer,\n            device=self.device\n        )\n\n        # linear layer before classifier\n        self.linear1 = nn.Linear(\n            ques_encoding_dim + text_encoding_dim,\n            linear_layer_out\n        )\n        # label classifier\n        self.label_classifier_layer = nn.Linear(\n            linear_layer_out,\n            num_labels\n        )\n        # dropout layer\n        self.dropout_layer = nn.Dropout(dropout_ratio)\n    \n    def forward(\n        self,\n        text_input_ids, ques_input_ids,\n        text_lengths, ques_lengths\n    ):\n        # text_input_ids = [batch_size, max_text_seq_len]\n        # ques_input_ids = [batch_size, max_ques_seq_len]\n        # text_lengths = [batch_size]\n        # ques_lengths = [batch_size]\n\n        # embedding\n        text_embedded = self.dropout_layer(\n            self.embedding_layer(text_input_ids)\n        )\n        ques_embedded = self.dropout_layer(\n            self.embedding_layer(ques_input_ids)\n        )\n        text_encoding, _ = self.text_encoder(\n            text_embedded, text_lengths\n        )\n        # text_encoding = [batch_size, text_encoding_dim]\n\n        ques_encoding, _ = self.question_encoder(\n            ques_embedded, ques_lengths\n        )\n        # ques_encoding = [batch_size, ques_encoding_dim]\n\n        ques_and_text_encoding = torch.cat(\n            (ques_encoding, text_encoding),\n            dim=1\n        )\n        # ques_and_text_encoding = [batch_size, ques_encoding_dim + text_encoding_dim]\n\n        linear_1_out = self.linear1(self.dropout_layer(ques_and_text_encoding))\n        # linear_1_out = [batch_size, linear_layer_out]\n\n        classifier_out = self.label_classifier_layer(linear_1_out)\n        # classifier_out = [batch_size, num_labels]\n\n        # attentive outputs \n        start_outputs, end_outputs = self.reader(\n            text_embedded, ques_encoding,\n            text_lengths\n        )\n        # start_outputs = [batch_size, max_text_seq_len]\n        # end_outputs = [batch_size, max_text_seq_len]\n\n        return classifier_out, start_outputs, end_outputs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = Path('../input/tensorflow2-question-answering/')\nDATA_PATH = DATA_DIR / 'simplified-nq-train.jsonl'\n\n\nSEED = 1029\nvalid_size = 0\ntrain_size = 307373 - valid_size\n\nCHUNKSIZE = 1000\nMAX_SEQ_LEN = 500\nMAX_QUESTION_LEN = 64\nDOC_STRIDE = 128\n\nNUM_LABELS = 5\nTRAIN_BATCH_SIZE = 64\nDATALOADER_NUM_WORKERS = 4\n\nbert_model = 'bert-base-uncased'\ndo_lower_case = 'uncased' in bert_model\n# device = torch.device('cuda')\n\nTOKENIZER = BertTokenizer.from_pretrained(\"../input/bert-config/vocab.txt\", do_lower_case=do_lower_case)\nCONFIG = BertConfig.from_pretrained(\"../input/bert-config/bert_config.json\")\n\n# output_model_file = 'bert_pytorch.bin'\n# output_optimizer_file = 'bert_pytorch_optimizer.bin'\n# output_amp_file = 'bert_pytorch_amp.bin'\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\n# torch.cuda.manual_seed(seed)\n# torch.backends.cudnn.deterministic = True\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(DEVICE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = QALstmJointModel(\n    vocab_size=TOKENIZER.vocab_size, embedding_dim=300, num_labels=NUM_LABELS,\n    linear_layer_out=100, dropout_ratio=0.2,\n\n    max_ques_seq_length=MAX_QUESTION_LEN, ques_pad_token_id=TOKENIZER.pad_token_id,\n    ques_encoding_attention_size=64,\n    ques_encoding_dim=64,\n    ques_enc_hidden_size=32, ques_enc_num_layers=2,\n    ques_enc_bidirectional=True, ques_enc_dropout_ratio=0.2,\n\n    max_text_seq_length=MAX_SEQ_LEN,\n    text_encoding_attention_size=256, text_encoding_dim=256,\n    text_pad_token_id=TOKENIZER.pad_token_id, \n    text_enc_hidden_size=256, text_enc_num_layers=2,\n    text_enc_bidirectional=True, text_enc_dropout_ratio=0.2,\n\n    text_reader_hidden_size=256, text_reader_num_layers=2,\n    text_reader_bidirectional=True, text_reader_dropout_ratio=0.2,\n    device=DEVICE\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_number_of_trainable_parameters(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"st_criterion = nn.CrossEntropyLoss(ignore_index=-1)\nen_criterion = nn.CrossEntropyLoss(ignore_index=-1)\nclass_criterion = nn.CrossEntropyLoss()\n\noptimizer = optim.Adam(model.parameters())\nscheduler = optim.lr_scheduler.StepLR(optimizer, step_size=100, gamma=0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = model.to(DEVICE)\nst_criterion = st_criterion.to(DEVICE)\nen_criterion = en_criterion.to(DEVICE)\nclass_criterion = class_criterion.to(DEVICE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_stats_at_step = 10\naccumulation_steps = 2\ntr_loss = 0.0\navg_tr_loss = 0.0\n\nstart_preds = None\nstart_out_label_ids = None\n\nend_preds = None\nend_out_label_ids = None\n\nclass_preds = None\nclass_out_label_ids = None\n\nconvert_func = functools.partial(\n    convert_data,\n    tokenizer=TOKENIZER,\n    max_seq_len=MAX_SEQ_LEN,\n    max_question_len=MAX_QUESTION_LEN,\n    doc_stride=DOC_STRIDE\n)\n\ndata_reader = JsonChunkReader(DATA_PATH, convert_func, chunksize=CHUNKSIZE)\n\nstep = 0\nmodel.train()\nfor epoch in range(1):\n    print(f'Epoch going on is {epoch}')\n    for examples in data_reader:\n        train_dataset = TextDataset(examples)\n        train_loader = DataLoader(\n            train_dataset, batch_size=TRAIN_BATCH_SIZE,\n            shuffle=True, collate_fn=collate_fn, \n            num_workers=DATALOADER_NUM_WORKERS\n        )\n\n        generator_batch_iterator = tqdm(train_loader)\n        for batch_idx, batch_data in enumerate(generator_batch_iterator):\n            if batch_data[0].shape[0] == TRAIN_BATCH_SIZE:\n\n                batch_data = tuple(t.to(DEVICE) for t in batch_data)\n                inputs = {\n                    \"text_input_ids\": batch_data[0],\n                    \"ques_input_ids\": batch_data[1],\n                    \"text_lengths\": batch_data[2], \n                    \"ques_lengths\": batch_data[3]\n                }\n                true_start_positions = batch_data[5]\n                true_end_positions = batch_data[6]\n                true_class_labels = batch_data[7]\n\n                # getting outputs \n                class_outputs, start_outputs, end_outputs = model(**inputs)\n                # propagating loss backwards and scheduler and optimizer steps\n                loss = triple_loss_function(\n                    [start_outputs, end_outputs, class_outputs],\n                    [true_start_positions, true_end_positions, true_class_labels],\n                    st_criterion, en_criterion, class_criterion\n                )\n                loss.backward()\n                if (step + 1) % accumulation_steps == 0:\n                    optimizer.step()\n                    scheduler.step()\n                    model.zero_grad()\n\n                step_loss = loss.item()\n                tr_loss += step_loss\n                avg_tr_loss += step_loss\n\n                # for calculation of results matrix\n                temp_start_preds = torch.max(F.softmax(start_outputs, dim=-1), 1)[1]\n                start_mask_arr = temp_start_preds.ne(-1)\n                temp_start_preds = temp_start_preds.masked_select(start_mask_arr)\n                true_start_positions = true_start_positions.masked_select(start_mask_arr)\n\n                temp_end_preds = torch.max(F.softmax(start_outputs, dim=-1), 1)[1]\n                end_mask_arr = temp_end_preds.ne(-1)\n                temp_end_preds = temp_end_preds.masked_select(end_mask_arr)\n                true_end_positions = true_end_positions.masked_select(end_mask_arr)\n                if start_preds is None:\n                    start_preds = temp_start_preds.detach().cpu().numpy()\n                    start_out_label_ids = true_start_positions.detach().cpu().numpy()\n\n                    end_preds = temp_end_preds.detach().cpu().numpy()\n                    end_out_label_ids = true_end_positions.detach().cpu().numpy()\n\n                    class_preds = torch.max(F.softmax(class_outputs, dim=-1), 1)[1].detach().cpu().numpy()\n                    class_out_label_ids = true_class_labels.detach().cpu().numpy()\n                else:\n                    start_preds = np.append(\n                        start_preds,\n                        temp_start_preds.detach().cpu().numpy(),\n                        axis=0\n                    )\n                    start_out_label_ids = np.append(\n                        start_out_label_ids,\n                        true_start_positions.detach().cpu().numpy(),\n                        axis=0\n                    )\n                    end_preds = np.append(\n                        end_preds,\n                        temp_end_preds.detach().cpu().numpy(),\n                        axis=0\n                    )\n                    end_out_label_ids = np.append(\n                        end_out_label_ids,\n                        true_end_positions.detach().cpu().numpy(),\n                        axis=0\n                    )\n                    class_preds = np.append(\n                        class_preds,\n                        torch.max(F.softmax(class_outputs, dim=-1), 1)[1].detach().cpu().numpy(),\n                        axis=0\n                    )\n                    class_out_label_ids = np.append(\n                        class_out_label_ids,\n                        true_class_labels.detach().cpu().numpy(),\n                        axis=0\n                    )\n\n                if step % print_stats_at_step == 0:\n                    tr_loss = tr_loss / print_stats_at_step\n                    results = get_results_dict(\n                        [start_preds, end_preds, class_preds],\n                        [start_out_label_ids, end_out_label_ids, class_out_label_ids]\n                    )\n                    # writing on bar\n                    generator_batch_iterator.set_description(\n                        f'Tr Iter: {step}, avg_step_loss: {tr_loss:.4f}, avg_tr_loss: {(avg_tr_loss / (step + 1)):.4f}, f1_c_labels: {results[\"f1_class_labels\"]:.4f}, acc_s_tok: {results[\"acc_s_tok\"]:.4f}, acc_e_tok: {results[\"acc_e_tok\"]:.4f}, acc_c_lab: {results[\"acc_c_lab\"]:.4f}'\n                    )\n                    tr_loss = 0.0\n                    start_preds = None\n                    start_out_label_ids = None\n                    end_preds = None\n                    end_out_label_ids = None\n                    class_preds = None\n                    class_out_label_ids = None\n                step += 1\n        del examples, train_dataset, train_loader\n        break       \n#         print(f\"{generator_idx + 1} generator is completed.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}