{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install ../input/sacremoses/sacremoses-master","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/transformers/transformers-master","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nimport torch \nimport numpy as np\n\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\nfrom torch.utils.data import DataLoader, TensorDataset, RandomSampler\nfrom transformers import BertTokenizer, BertConfig\n\nfrom sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_FILE_NAME = \"../input/tensorflow2-question-answering/simplified-nq-train.jsonl\"\nTEST_FILE_NAME = \"../input/tensorflow2-question-answering/simplified-nq-test.jsonl\"\n\nVOCAB_SIZE = 30522\nTEXT_MAX_SEQUENCE_LENGTH = 500\nQUESTION_MAX_SEQUENCE_LENGTH = 25\n\nTRAIN_BATCH_SIZE = 256\nVAL_BATCH_SIZE = 64\n\nDATALOADER_NUM_WORKERS = 8\n\nTOKENIZER = BertTokenizer.from_pretrained(\"../input/bert-config/vocab.txt\")\nCONFIG = BertConfig.from_pretrained(\"../input/bert-config/bert_config.json\")\n# print(CONFIG)\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(DEVICE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\nimport random\nimport pandas as pd\nimport json\nfrom math import floor\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom transformers import BertTokenizer, BertConfig\n\nimport multiprocessing\nfrom functools import partial\n\nimport torch\nfrom torch.utils.data import DataLoader, TensorDataset, RandomSampler\nfrom sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n\n\ndef print_number_of_trainable_parameters(model):\n    model_parameters = filter(lambda p: p.requires_grad, model.parameters())\n    params = sum([np.prod(p.size()) for p in model_parameters])\n    print(\"Number of trainable parameters in the model are : {}\".format(params))\n    return\n\ndef get_results_dict(y_test, y_pred):\n    results = {\n        \"f1\": f1_score(y_test, y_pred, average=\"macro\"),\n        \"precision\": precision_score(y_test, y_pred, average=\"macro\"),\n        \"recall\": recall_score(y_test, y_pred, average=\"macro\"),\n        \"accuracy\": accuracy_score(y_test, y_pred)\n    }\n    return results\n    \nclass InputExample(object):\n    \"\"\"A single training/test example for question answering.\"\"\"\n    \n    def __init__(self, text, question, label, example_id, document_url):\n        self.text = text\n        self.question = question\n        self.label = label\n        self.example_id = example_id\n        self.document_url = document_url\n\n\ndef cleanhtml(raw_html):\n    cleanr = re.compile('<.*?>')\n    cleantext = re.sub(cleanr, '', raw_html)\n    cleantext = re.sub(\"\\s+\", \" \", cleantext)\n    return cleantext\n\n\ndef get_generator(file_name, read_batch_size):\n    with open(file_name, \"r\") as file_:\n        batch = []\n        \n        for idx, item in enumerate(file_):\n            if len(batch) == read_batch_size:\n                batch = []\n            \n            batch.append(json.loads(item))\n            \n            if len(batch) == read_batch_size:\n                yield batch\n                \n\ndef create_input_examples_from_rec(rec, negative_sampling_percent=0.5):\n    text = rec[\"document_text\"].split()\n    question_text = rec[\"question_text\"].strip()\n    question_text = cleanhtml(\" \".join(question_text)).strip().lower()\n    document_url = rec[\"document_url\"]\n    example_id = rec[\"example_id\"]\n\n    long_answer_start_token = rec[\"annotations\"][0][\"long_answer\"][\"start_token\"]\n    long_answer_end_token = rec[\"annotations\"][0][\"long_answer\"][\"end_token\"]\n    long_answer_candidate_idx = rec[\"annotations\"][0][\"long_answer\"]['candidate_index']\n\n    long_answer_candidates = rec[\"long_answer_candidates\"]\n\n    temp_input_examples_list = []\n\n    # removing true label \n    if long_answer_start_token != -1:\n        long_answer_candidates = long_answer_candidates[: long_answer_candidate_idx] \\\n        + long_answer_candidates[long_answer_candidate_idx + 1 :]\n\n        # adding true label\n        temp_text = text[long_answer_start_token:long_answer_end_token]\n        temp_text = cleanhtml(\" \".join(temp_text)).strip().lower()\n        \n        if len(temp_text.split()) > 0 and len(question_text.split()) > 0:\n            temp_input_examples_list.append(\n                InputExample(\n                    text=temp_text, \n                    question=question_text,\n                    label=1,\n                    example_id=example_id,\n                    document_url=document_url\n                )\n            )\n\n    num_negative_samples = floor(len(long_answer_candidates) * negative_sampling_percent)\n    sampled_negative_samples = random.sample(long_answer_candidates, num_negative_samples)\n\n    # adding negative samples\n    for candidate in sampled_negative_samples:\n        candidate_start = candidate[\"start_token\"]  \n        candidate_end = candidate[\"end_token\"]\n\n        temp_text = text[candidate_start:candidate_end]\n        temp_text = cleanhtml(\" \".join(temp_text)).strip().lower()\n        if len(temp_text.split()) > 0 and len(question_text.split()) > 0:\n            temp_input_examples_list.append(\n                InputExample(\n                    text=temp_text, \n                    question=question_text,\n                    label=0,\n                    example_id=example_id,\n                    document_url=document_url\n                )\n            )\n    return temp_input_examples_list\n\n\nclass InputFeature(object):\n    def __init__(\n        self, text_input_ids, ques_input_ids, \n        text_seq_length, ques_seq_length,label\n    ):\n        self.text_input_ids = text_input_ids\n        self.ques_input_ids = ques_input_ids\n        self.text_seq_length = text_seq_length\n        self.ques_seq_length = ques_seq_length\n        self.label = label\n\n\ndef convert_one_example_to_feature(\n    example, tokenizer,\n    text_max_seq_length=None,\n    ques_max_seq_length=None\n):\n    # featurinzing text\n    text_input_words = tokenizer.tokenize(example.text)\n    text_input_ids = tokenizer.convert_tokens_to_ids(text_input_words)\n    text_seq_length = len(text_input_words)\n    \n    if text_max_seq_length:\n        if text_seq_length > text_max_seq_length:\n            text_input_ids = text_input_ids[:text_max_seq_length]\n            text_seq_length = text_max_seq_length\n        else:\n            text_input_ids = text_input_ids + [tokenizer.pad_token_id]*(\n                text_max_seq_length - text_seq_length\n            )\n    \n    # featurizing question\n    ques_input_words = tokenizer.tokenize(example.question)\n    ques_input_ids = tokenizer.convert_tokens_to_ids(ques_input_words)\n    ques_seq_length = len(ques_input_words)\n    \n    if ques_max_seq_length:\n        if ques_seq_length > ques_max_seq_length:\n            ques_input_ids = ques_input_ids[:ques_max_seq_length]\n            ques_seq_length = ques_max_seq_length\n        else:\n            ques_input_ids = ques_input_ids + [tokenizer.pad_token_id]*(\n                ques_max_seq_length - ques_seq_length\n            )\n    \n    feature = InputFeature(\n        text_input_ids=text_input_ids,\n        ques_input_ids=ques_input_ids,\n        label = example.label,\n        text_seq_length=text_seq_length,\n        ques_seq_length=ques_seq_length\n    )\n    return feature\n    \n\ndef load_cache_examples_multiprocessing(\n    examples, tokenizer, text_max_seq_length, ques_max_seq_length\n):\n    pool = multiprocessing.Pool()\n    features = pool.map(\n        partial(\n            convert_one_example_to_feature,\n            tokenizer=tokenizer,\n            text_max_seq_length=text_max_seq_length,\n            ques_max_seq_length=ques_max_seq_length\n        ),\n        examples\n    )\n    pool.close()\n    \n    # convert to Tensors and build dataset\n    all_text_input_ids = torch.tensor(\n        [f.text_input_ids for f in features], dtype=torch.long\n    )\n    all_ques_input_ids = torch.tensor(\n        [f.ques_input_ids for f in features], dtype=torch.long\n    )\n    all_text_seq_lengths = torch.tensor(\n        [f.text_seq_length for f in features], dtype=torch.long\n    )\n    all_ques_seq_lengths = torch.tensor(\n        [f.ques_seq_length for f in features], dtype=torch.long\n    )\n    all_labels = torch.tensor(\n        [f.label for f in features], dtype=torch.float\n    )\n    dataset = TensorDataset(\n        all_text_input_ids, all_ques_input_ids, all_text_seq_lengths,\n        all_ques_seq_lengths, all_labels\n    )\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch \nimport torch.optim as optim\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\n# takes the last hidden_state as the question encoding\n\nclass BiLstmEncoder(nn.Module):\n    def __init__(\n        self, vocab_size, embedding_dim,\n        hidden_size, num_layers, output_dim,\n        bidirectional, dropout_ratio=0.2\n    ):\n        super(BiLstmEncoder, self).__init__()\n        self.vocab_size = vocab_size\n        self.embedding_dim = embedding_dim\n        self.hidden_size = hidden_size\n        self.num_layers = num_layers\n        self.bidirectional = bidirectional\n        self.dropout_ratio = dropout_ratio\n        self.output_dim = output_dim\n\n        self.embedding = nn.Embedding(\n            num_embeddings=vocab_size,\n            embedding_dim= embedding_dim\n        )\n        self.lstm_layer = nn.LSTM(\n            input_size=embedding_dim,\n            num_layers=num_layers,\n            hidden_size=hidden_size,\n            bidirectional=bidirectional,\n            dropout=dropout_ratio,\n            batch_first=True\n        )\n\n        if bidirectional is True: \n            self.linear_combiner = nn.Linear(num_layers*2*hidden_size, output_dim)\n        else:\n            self.linear_combiner = nn.Linear(num_layers*1*hidden_size, output_dim)\n\n        self.dropout_layer = nn.Dropout(dropout_ratio)\n    \n    def forward(self, question, ques_seq_lengths=None):\n        # question = [batch_size, sent_length]\n\n        batch_size = question.shape[0]\n    \n        embedded = self.embedding(question)\n        # embedded = [batch_size, sent_length, embedding_dim]\n\n        _, (hidden, _) = self.lstm_layer(embedded)\n        # hidden = [num_layers * num_directions, batch_size, hidden_size]\n\n        hidden = hidden.view(batch_size, -1)\n        # hidden = [batch_size, num_layers * num_directions * hidden_size]\n\n        combined_context = self.linear_combiner(self.dropout_layer(hidden))\n        # combined_context = [batch_size, output_dim]\n\n        return combined_context\n\n\nclass Attention(nn.Module):\n    \"\"\"\n    Computes a weighted average of channels across timesteps (1 parameter pr. channel).\n    \"\"\"\n    def __init__(\n        self, attention_size, device\n    ):\n        super(Attention, self).__init__()\n        self.attention_size = attention_size\n        self.device = device\n\n        self.attention = nn.Parameter(torch.rand(attention_size))\n\n    def forward(self, inputs, input_lengths):\n        # inputs = [batch_size, max_seq_length, attention_size]\n        # input_lengths = [batch_size]\n\n        max_seq_length = inputs.shape[1]\n\n        attn = torch.matmul(inputs, self.attention)\n        # attn = [batch_size, maz_seq_len]\n\n        idxes = torch.arange(0, max_seq_length, out=torch.LongTensor(max_seq_length)).unsqueeze(0).to(self.device)\n        mask = torch.autograd.Variable((idxes < input_lengths.unsqueeze(1)).float()).to(self.device)\n        # mask = [batch_size, max_seq_length]\n        # idxes = [batch_size, max_seq_length]\n\n        attn_masked = attn.masked_fill(mask == 0, -1e10)\n        attention_weights = F.softmax(attn_masked, dim=1)\n        # attention_weights = [batch_size, max_seq_length]\n\n        # apply attention weights\n        weighted = torch.bmm(attention_weights.unsqueeze(1), inputs)\n        # weighted = [batch_size, 1, attention_size]\n\n        weighted = weighted.squeeze(1)\n        # weighted_outputs = [batch_size, attention_size]\n\n        return (weighted, attention_weights)\n\n\nclass AttentiveBilstm(nn.Module):\n    def __init__(\n        self, max_seq_length,\n        vocab_size, embedding_dim,\n        hidden_size, num_layers, output_dim,\n        bidirectional, device, dropout_ratio=0.2\n    ):\n        super(AttentiveBilstm, self).__init__()\n        self.max_seq_length = max_seq_length\n        self.vocab_size = vocab_size\n        self.embedding_dim = embedding_dim\n        self.hidden_size = hidden_size\n        self.num_layers = num_layers\n        self.output_dim = output_dim\n        self.bidirectional = bidirectional\n        self.device = device\n        self.dropout_ratio = dropout_ratio\n\n        self.embedding = nn.Embedding(\n            num_embeddings=vocab_size,\n            embedding_dim=embedding_dim\n        )\n\n        self.lstm_layer = nn.LSTM(\n            input_size=embedding_dim,\n            num_layers=num_layers,\n            hidden_size=hidden_size,\n            bidirectional=bidirectional,\n            dropout=dropout_ratio\n        )\n        \n        if bidirectional is True:\n            self.attention_layer = Attention(\n                attention_size=hidden_size*2,\n                device=device\n            )\n            self.linear_combiner = nn.Linear(\n                hidden_size*2, output_dim\n            )\n        else:\n            self.attention_layer = Attention(\n                attention_size=hidden_size*1,\n                device=device\n            )\n            self.linear_combiner = nn.Linear(\n                hidden_size, output_dim\n            ) \n\n        self.dropout_layer = nn.Dropout(dropout_ratio)\n    \n    def forward(self, question, seq_lengths):\n        # question = [batch_size, max_seq_length]\n        # seq_lengths = [batch_size]\n\n        embedded = self.dropout_layer(self.embedding(question))\n        # embedded = [batch_size, max_seq_length, embedding_dim]\n        print(embedded.shape)\n\n        # permuting for pad packed easiness\n        embedded = embedded.permute(1, 0, 2)\n        # embedded = [max_seq_length, batch_size, embedding_dim]\n\n        packed_embedded = nn.utils.rnn.pack_padded_sequence(\n            embedded, seq_lengths, enforce_sorted=False\n        )\n    \n        packed_outputs, (_, _) = self.lstm_layer(packed_embedded)\n\n        outputs, _ = nn.utils.rnn.pad_packed_sequence(packed_outputs, total_length=self.max_seq_length)\n        # outputs = [max_seq_length, batch_size, num_directions*hidden_size]\n\n        # outputs are permuted again because attention layer needs batch_first\n        (weighted_outputs, attention_weights) = self.attention_layer(outputs.permute(1, 0, 2), seq_lengths)\n        # weighted_outputs = [batch_size, attention_size]\n        \n        weighted_outputs = self.linear_combiner(weighted_outputs)\n        # weighted_outputs = [batch_size, output_dim]\n\n        return (weighted_outputs, attention_weights)\n\n\nclass DeepMoji(nn.Module):\n    def __init__(\n        self, vocab_size, embedding_dim, hidden_state_size,\n        num_layers, output_dim, device,\n        dropout_ratio=0.5, bidirectional=True\n    ):\n\n        super(DeepMoji, self).__init__()\n        self.vocab_size = vocab_size\n        self.embedding_dim = embedding_dim\n        self.hidden_state_size = hidden_state_size\n        self.num_layers=num_layers\n        self.output_dim = output_dim\n        self.dropout_ratio = dropout_ratio\n        self.bidirectional = bidirectional\n        self.output_dim = output_dim\n        self.device = device\n        \n        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n        \n        self.bilstm_one = nn.LSTM(\n            input_size=embedding_dim,\n            hidden_size=hidden_state_size,\n            bidirectional=bidirectional,\n            num_layers=num_layers,\n            dropout=dropout_ratio\n        )\n        \n        self.lstm_one_context_combiner_layer = nn.Linear(\n            2*hidden_state_size, hidden_state_size\n        )\n        \n        self.bilstm_two = nn.LSTM(\n            input_size=hidden_state_size,\n            hidden_size=hidden_state_size,\n            bidirectional=bidirectional,\n            num_layers=num_layers,\n            dropout=dropout_ratio\n        )\n        self.lstm_two_context_combiner_layer = nn.Linear(\n            2*hidden_state_size, hidden_state_size\n        )\n        \n        self.attn_layer = Attention(hidden_state_size*2 + embedding_dim, device)\n\n        self.output_layer = nn.Linear(hidden_state_size*2 + embedding_dim, output_dim)\n        \n        self.dropout_layer = nn.Dropout(dropout_ratio)\n    \n    def forward(self, inp, src_len):\n        # inp = [batch_size, sent_length]\n        # src_len = [batch_size]\n        \n        embedded = self.dropout_layer(self.embedding(inp)).permute(1, 0, 2)\n        # embedded = [sent_length, batch_size, embedding_dim]\n        \n        embedded_packed = torch.nn.utils.rnn.pack_padded_sequence(\n            embedded, src_len, enforce_sorted=False\n        )\n        \n        bilstm_out_1_packed, (_, _) = self.bilstm_one(embedded_packed)\n        # bilstm_out_1 = [seq_len, batch_size, 2 * hidden_state_size] \n        \n        bilstm_out_1, _ = nn.utils.rnn.pad_packed_sequence(\n            bilstm_out_1_packed, total_length=embedded.shape[0])\n\n        bilstm_out_1_context_combined = self.lstm_one_context_combiner_layer(\n            bilstm_out_1\n        )\n        # bilstm_out_1_context_combined = [seq_len, batch_size, hidden_state_size]\n\n        bilstm_2_input_packed = torch.nn.utils.rnn.pack_padded_sequence(\n            bilstm_out_1_context_combined, src_len, enforce_sorted=False\n        )\n        bilstm_out_2_packed, (_, _) = self.bilstm_two(bilstm_2_input_packed) \n        # bilstm_out_2_packed = [seq_len, batch_size, 2 * hidden_state_size]\n        \n        bilstm_out_2, _ = nn.utils.rnn.pad_packed_sequence(\n            bilstm_out_2_packed, total_length=embedded.shape[0]\n        )\n        \n        bilstm_out_2_context_combined = self.lstm_two_context_combiner_layer(\n            bilstm_out_2\n        )\n        # bilstm_out_2_context_combined = [seq_len, batch_size, hidden_state_size]\n    \n        bilstm_stacked = torch.cat(\n            (\n                bilstm_out_1_context_combined,\n                bilstm_out_2_context_combined\n            ), \n            dim=2\n        )\n        # bilstm_stacked = [seq_len, batch_size, 2 * hidden_state_size]\n\n        # stacking embedded to bilstm_stacked\n        bilstm_and_embedded_stacked = torch.cat(\n            (\n                bilstm_stacked,\n                embedded\n            ), \n            dim=2\n        )    \n        # bilstm_and_embedded_stacked = [seq_len, batch_size, 2 * hidden_state_size + embedding_dim]\n        \n                \n        (weighted_outputs, attention_weights) = self.attn_layer(\n            bilstm_and_embedded_stacked.permute(1, 0, 2), src_len\n        )\n        # weighted_outputs = [batch_size, attention_size]\n\n        outputs = self.output_layer(weighted_outputs)\n        # outputs = [batch_size, output_dim]\n\n        return outputs\n\n\nclass QAModel(nn.Module):\n    def __init__(\n        self, ques_embedding_dim,\n        text_embedding_dim,\n        ques_embedder,\n        text_embedder, device,\n        first_combiner_size=100,\n        dropout_ratio=0.2\n    ):\n        super(QAModel, self).__init__()\n\n        self.text_embedding_dim = text_embedding_dim\n        self.ques_embedding_dim = ques_embedding_dim\n        self.text_embedder = text_embedder\n        self.ques_embedder = ques_embedder\n        self.device = device\n\n        self.linear_combiner_1 = nn.Linear(\n            text_embedding_dim + ques_embedding_dim, first_combiner_size\n        )\n        self.linear_combiner_2 = nn.Linear(\n            first_combiner_size, 1\n        )\n        self.dropout_layer = nn.Dropout(dropout_ratio)\n    \n    def forward(\n        self, text_input_ids, ques_input_ids,\n        text_seq_lengths, ques_seq_lengths\n    ):\n        # text_input_ids = [batch_size, text_max_seq_length]\n        # ques_input_ids = [batch_size, ques_max_seq_length]\n        # text_seq_lengths = [batch_size]\n        # quest_seq_lengths = [batch_size]\n        # labels = [batch_size]\n        \n        text_encoded = self.text_embedder(text_input_ids, text_seq_lengths)\n        # text_encoded = [batch_size, text_embedding_dim]\n        \n        ques_encoded = self.ques_embedder(ques_input_ids, ques_seq_lengths)\n        # ques_encoded = [batch_size, ques_embedding_dim]\n        \n        # stacking both\n        stacked_ques_text = torch.cat(\n            (\n                ques_encoded,\n                text_encoded\n            ), \n            dim=1\n        )\n        # stacked_ques_text = [batch_size, ques_embedding_dim + text_embedding_dim]\n        \n        fc_out_1 = self.linear_combiner_1(self.dropout_layer(stacked_ques_text))\n        # fc_out_1 = [batch_size, first_combiner_size]\n        \n        output = self.linear_combiner_2(fc_out_1)\n        # output = [batch_size, 1]\n        \n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ques_encoder = BiLstmEncoder(\n    vocab_size=VOCAB_SIZE, embedding_dim=50,\n    hidden_size=64, num_layers=1, output_dim=100,\n    bidirectional=True\n)\n\ntext_encoder = DeepMoji(\n    vocab_size=VOCAB_SIZE, embedding_dim=300, hidden_state_size=256,\n    num_layers=2, output_dim=500,\n    dropout_ratio=0.5, bidirectional=True, device=DEVICE\n)\n\nmodel = QAModel(\n    ques_embedding_dim=100,\n    text_embedding_dim=500,\n    ques_embedder=ques_encoder,\n    text_embedder=text_encoder,\n    first_combiner_size=200,\n    dropout_ratio=0.2, device=DEVICE\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(model.parameters())\nscheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1000, gamma=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = model.to(DEVICE)\ncriterion = criterion.to(DEVICE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch_from_generator(\n    model, train_generator, optimizer, criterion,\n    negative_sampling_percent, tokenizer,\n    scheduler, print_stats_at_step=20,\n    train_batch_size=TRAIN_BATCH_SIZE,\n    generator_num_workers=DATALOADER_NUM_WORKERS,\n    text_max_seq_length=TEXT_MAX_SEQUENCE_LENGTH,\n    ques_max_seq_length=QUESTION_MAX_SEQUENCE_LENGTH,\n    device=DEVICE\n):\n    tr_loss = 0.0\n    avg_tr_loss = 0.0\n    \n    preds = None\n    out_label_ids = None\n    \n    model.train()\n    step = 0\n    for generator_idx, generator_batch in enumerate(train_generator):\n        all_data = []\n\n        for rec in generator_batch:\n            all_data.extend(create_input_examples_from_rec(rec, negative_sampling_percent=0.2))\n\n        generator_train_dataset = load_cache_examples_multiprocessing(\n            all_data, tokenizer, text_max_seq_length=text_max_seq_length,\n            ques_max_seq_length=ques_max_seq_length\n        )\n\n        generator_random_sampler = RandomSampler(generator_train_dataset)\n        generator_data_loader = DataLoader(\n            generator_train_dataset, sampler=generator_random_sampler,\n            batch_size=train_batch_size,\n            num_workers=generator_num_workers\n        )\n        \n        generator_batch_iterator = tqdm(generator_data_loader)\n        for batch_idx, batch_data in enumerate(generator_batch_iterator):\n            if batch_data[0].shape[0] == train_batch_size:\n                model.zero_grad()\n                \n                batch_data = tuple(t.to(device) for t in batch_data)\n                inputs = {\n                    \"text_input_ids\": batch_data[0],\n                    \"ques_input_ids\": batch_data[1],\n                    \"text_seq_lengths\": batch_data[2], \n                    \"ques_seq_lengths\": batch_data[3]\n                }\n                true_labels = batch_data[4]\n                \n                # getting outputs \n                logits = model(**inputs).squeeze(1)\n                \n                # propagating loss backwards and scheduler and optimizer steps\n                loss = criterion(logits, true_labels)\n                loss.backward()\n                optimizer.step()\n                scheduler.step()\n                step_loss = loss.item()\n                \n                tr_loss += step_loss\n                avg_tr_loss += step_loss\n                \n                # for calculation of results matrix\n                if preds is None:\n                    preds = torch.round(F.sigmoid(logits)).detach().cpu().numpy()\n                    out_label_ids = true_labels.detach().cpu().numpy()\n                else:\n                    preds = np.append(\n                        preds,\n                        torch.round(F.sigmoid(logits)).detach().cpu().numpy(),\n                        axis=0\n                    )\n                    out_label_ids = np.append(\n                        out_label_ids,\n                        true_labels.detach().cpu().numpy(),\n                        axis=0\n                    )\n                if step % print_stats_at_step == 0:\n                    tr_loss = tr_loss / print_stats_at_step\n                    results = get_results_dict(out_label_ids, preds)\n                    # writing on bar\n                    generator_batch_iterator.set_description(\n                        f'Tr Iter: {step}, avg_step_loss: {tr_loss:.4f}, avg_tr_loss: {(avg_tr_loss / (step + 1)):.4f}, tr_f1: {results[\"f1\"]:.4f}, tr_prec: {results[\"precision\"]:.4f}, tr_rec: {results[\"recall\"]:.4f}, tr_acc: {results[\"accuracy\"]:.4f}'\n                    )\n                    tr_loss = 0.0\n                    preds = None\n                    out_label_ids = None\n                step += 1\n                \n        print(f\"{generator_idx + 1} generator is completed.\")\n        if generator_idx > 1:\n            break\n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_generator = get_generator(TRAIN_FILE_NAME, 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_epoch_from_generator(\n    model, data_generator, optimizer=optimizer, criterion=criterion,\n    negative_sampling_percent=0.1, tokenizer=TOKENIZER,\n    scheduler=scheduler, print_stats_at_step=20,\n    train_batch_size=TRAIN_BATCH_SIZE,\n    generator_num_workers=DATALOADER_NUM_WORKERS,\n    text_max_seq_length=TEXT_MAX_SEQUENCE_LENGTH,\n    ques_max_seq_length=QUESTION_MAX_SEQUENCE_LENGTH,\n    device=DEVICE\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}