{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-05T06:30:57.609827Z","iopub.execute_input":"2022-08-05T06:30:57.611101Z","iopub.status.idle":"2022-08-05T06:30:57.651368Z","shell.execute_reply.started":"2022-08-05T06:30:57.610962Z","shell.execute_reply":"2022-08-05T06:30:57.650039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ast import literal_eval\nfrom itertools import chain\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom sklearn.metrics import precision_recall_fscore_support\nfrom sklearn.model_selection import train_test_split\nfrom torch import optim\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset\nfrom tqdm.notebook import tqdm\nfrom transformers import AutoModel, AutoTokenizer, RobertaTokenizerFast, RobertaModel, RobertaConfig, AutoConfig\nimport pytorch_lightning as pl\nimport torch.nn.functional as F\nfrom torch.nn import Parameter\nimport torchmetrics\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import f1_score","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:30:57.963323Z","iopub.execute_input":"2022-08-05T06:30:57.964572Z","iopub.status.idle":"2022-08-05T06:31:03.535787Z","shell.execute_reply.started":"2022-08-05T06:30:57.964525Z","shell.execute_reply":"2022-08-05T06:31:03.534359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions\n### 1. Datasets Helper Function\nneed to merge `features.csv`, `patient_notes.csv` with `train.csv`","metadata":{}},{"cell_type":"code","source":"'''\nTo prepare the dataset\n'''\nBASE_URL = \"../input/nbme-score-clinical-patient-notes\"\n\n\ndef process_feature_text(text):\n    return text.replace(\"-OR-\", \";-\").replace(\"-\", \" \")\n\n\ndef prepare_datasets():\n    features = pd.read_csv(f\"{BASE_URL}/features.csv\")\n    notes = pd.read_csv(f\"{BASE_URL}/patient_notes.csv\")\n    df = pd.read_csv(f\"{BASE_URL}/train.csv\")\n    df[\"annotation_list\"] = [literal_eval(x) for x in df[\"annotation\"]]\n    df[\"location_list\"] = [literal_eval(x) for x in df[\"location\"]]\n\n    merged = df.merge(notes, how=\"left\")\n    merged = merged.merge(features, how=\"left\")\n\n    merged[\"feature_text\"] = [process_feature_text(x) for x in merged[\"feature_text\"]]\n    merged[\"feature_text\"] = merged[\"feature_text\"].apply(lambda x: x.lower())\n    merged[\"pn_history\"] = merged[\"pn_history\"].apply(lambda x: x.lower())\n\n    return merged","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:03.538424Z","iopub.execute_input":"2022-08-05T06:31:03.539360Z","iopub.status.idle":"2022-08-05T06:31:03.550440Z","shell.execute_reply.started":"2022-08-05T06:31:03.539320Z","shell.execute_reply":"2022-08-05T06:31:03.549258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = prepare_datasets()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:03.551954Z","iopub.execute_input":"2022-08-05T06:31:03.552374Z","iopub.status.idle":"2022-08-05T06:31:04.904393Z","shell.execute_reply.started":"2022-08-05T06:31:03.552340Z","shell.execute_reply":"2022-08-05T06:31:04.903311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:04.905666Z","iopub.execute_input":"2022-08-05T06:31:04.906024Z","iopub.status.idle":"2022-08-05T06:31:04.931187Z","shell.execute_reply.started":"2022-08-05T06:31:04.905982Z","shell.execute_reply":"2022-08-05T06:31:04.930092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nget the location predictions from get_location_predictions\nby passing prediction, offset and sequence id arrays\n\ncalculate_char_cv(predictions, offset_mapping, sequence_ids, labels)\nget the accuracy, precision, recall, f1-score.\n'''\n\nfrom sklearn.metrics import accuracy_score\n\ndef get_location_predictions(preds, offset_mapping, sequence_ids, test=False):\n    all_predictions = []\n    for pred, offsets, seq_ids in zip(preds, offset_mapping, sequence_ids):\n        pred = 1 / (1 + np.exp(-pred))\n        start_idx = None\n        end_idx = None\n        current_preds = []\n        for pred, offset, seq_id in zip(pred, offsets, seq_ids):\n            if seq_id is None or seq_id == 0:\n                continue\n\n            if pred > 0.5:\n                if start_idx is None:\n                    start_idx = offset[0]\n                end_idx = offset[1]\n            elif start_idx is not None:\n                if test:\n                    current_preds.append(f\"{start_idx} {end_idx}\")\n                else:\n                    current_preds.append((start_idx, end_idx))\n                start_idx = None\n        if test:\n            all_predictions.append(\"; \".join(current_preds))\n        else:\n            all_predictions.append(current_preds)\n            \n    return all_predictions\n\n\ndef calculate_char_cv(predictions, offset_mapping, sequence_ids, labels):\n    all_labels = []\n    all_preds = []\n    for preds, offsets, seq_ids, labels in zip(predictions, offset_mapping, sequence_ids, labels):\n\n        num_chars = max(list(chain(*offsets)))\n        char_labels = np.zeros(num_chars)\n\n        for o, s_id, label in zip(offsets, seq_ids, labels):\n            if s_id is None or s_id == 0:\n                continue\n            if int(label) == 1:\n                char_labels[o[0]:o[1]] = 1\n\n        char_preds = np.zeros(num_chars)\n\n        for start_idx, end_idx in preds:\n            char_preds[start_idx:end_idx] = 1\n\n        all_labels.extend(char_labels)\n        all_preds.extend(char_preds)\n\n    results = precision_recall_fscore_support(all_labels, all_preds, average=\"binary\", labels=np.unique(all_preds) ,warn_for=(\"precision\", \"recall\", \"f-score\"), zero_division= 0 )\n    accuracy = accuracy_score(all_labels, all_preds)\n    \n\n    return {\n        \"Accuracy\": accuracy,\n        \"precision\": results[0],\n        \"recall\": results[1],\n        \"f1\": results[2]\n    }","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:04.934040Z","iopub.execute_input":"2022-08-05T06:31:04.934355Z","iopub.status.idle":"2022-08-05T06:31:04.948391Z","shell.execute_reply.started":"2022-08-05T06:31:04.934326Z","shell.execute_reply":"2022-08-05T06:31:04.946997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nloc_list_to_ints converts the location into tuples of integers\ntokenize_and_add_labels tokenizes the feature_text and pn_history \nand adds the columns of tlocation_int, sequence_ids and labels\n'''\n\ndef loc_list_to_ints(loc_list):\n    to_return = []\n    for loc_str in loc_list:\n        loc_strs = loc_str.split(\";\")\n        for loc in loc_strs:\n            start, end = loc.split()\n            to_return.append((int(start), int(end)))\n    return to_return\n\n\ndef tokenize_and_add_labels(tokenizer, data, config):\n    out = tokenizer(\n        data[\"feature_text\"],\n        data[\"pn_history\"],\n        truncation=config['truncation'],\n        max_length=config['max_length'],\n        padding=config['padding'],\n        return_offsets_mapping=config['return_offsets_mapping']\n    )\n    labels = [0.0] * len(out[\"input_ids\"])\n    out[\"location_int\"] = loc_list_to_ints(data[\"location_list\"])\n    out[\"sequence_ids\"] = out.sequence_ids()\n\n    for idx, (seq_id, offsets) in enumerate(zip(out[\"sequence_ids\"], out[\"offset_mapping\"])):\n        if not seq_id or seq_id == 0:\n            labels[idx] = -1\n            continue\n\n        token_start, token_end = offsets\n        for feature_start, feature_end in out[\"location_int\"]:\n            if token_start >= feature_start and token_end <= feature_end:\n                labels[idx] = 1.0\n                break\n\n    out[\"labels\"] = labels\n\n    return out","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:04.950067Z","iopub.execute_input":"2022-08-05T06:31:04.950682Z","iopub.status.idle":"2022-08-05T06:31:04.962996Z","shell.execute_reply.started":"2022-08-05T06:31:04.950648Z","shell.execute_reply":"2022-08-05T06:31:04.962093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hyperparameters = {\n    \"max_length\": 416,\n    \"padding\": \"max_length\",\n    \"return_offsets_mapping\": True,\n    \"truncation\": \"only_second\",\n    \"model_name\": \"../input/roberta-base\",\n    \"dropout\": 0.2,\n    \"lr\": 1e-5,\n    \"test_size\": 0.2,\n    \"seed\": 1268,\n    \"batch_size\": 8\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:04.964527Z","iopub.execute_input":"2022-08-05T06:31:04.965579Z","iopub.status.idle":"2022-08-05T06:31:04.976891Z","shell.execute_reply.started":"2022-08-05T06:31:04.965522Z","shell.execute_reply":"2022-08-05T06:31:04.975869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nClass dataset where data, tokenizer and configuration is passed for initialization\n'''\nclass CustomDataset(Dataset):\n    def __init__(self, data, tokenizer, config):\n        self.data = data\n        self.tokenizer = tokenizer\n        self.config = config\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        data = self.data.iloc[idx]\n        tokens = tokenize_and_add_labels(self.tokenizer, data, self.config)\n\n        input_ids = np.array(tokens[\"input_ids\"])\n        attention_mask = np.array(tokens[\"attention_mask\"])\n#         token_type_ids = np.array(tokens[\"token_type_ids\"])\n\n        labels = np.array(tokens[\"labels\"])\n        offset_mapping = np.array(tokens['offset_mapping'])\n        sequence_ids = np.array(tokens['sequence_ids']).astype(\"float16\")\n        \n        return input_ids, attention_mask, labels, offset_mapping, sequence_ids","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:04.978335Z","iopub.execute_input":"2022-08-05T06:31:04.979287Z","iopub.status.idle":"2022-08-05T06:31:04.988513Z","shell.execute_reply.started":"2022-08-05T06:31:04.979238Z","shell.execute_reply":"2022-08-05T06:31:04.987527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nDefining our custom models and logits.\nOur model consists of 3 fully connected linear layers \n'''\n\n'''Flow of pytorch Lightning Module'''\n\n\nclass CustomModel(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n#         self.config = config\n        self.config = AutoConfig.from_pretrained(hyperparameters['model_name'], output_hidden_states=True)\n        self.bert = RobertaModel.from_pretrained(hyperparameters['model_name'], config = self.config)  # BERT model\n        self.dropout = nn.Dropout(p= hyperparameters['dropout'])\n        self.fc1 = nn.Linear(768, 512)\n        self.fc2 = nn.Linear(512, 512)\n        self.fc3 = nn.Linear(512, 1)\n        self.train_acc = torchmetrics.Accuracy()\n#         self.train_f1 = torchmetrics.functional.f1(average=\"micro\")\n        self.val_acc = torchmetrics.Accuracy()\n#         self.val_f1 = torchmetrics.functional.f1(average=\"micro\")\n        \n        \n        \n    #prediction function\n    def forward(self,input_ids,attention_mask):\n        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)\n        logits = self.fc1(outputs[0])\n        logits = self.fc2(self.dropout(logits))\n        logits = self.fc3(self.dropout(logits)).squeeze(-1)\n        return logits\n\n\n       \n    \n    \n    \n    def configure_optimizers(self):\n        optimizer = optim.AdamW(self.parameters(), lr=hyperparameters['lr'])\n        return optimizer\n    \n    \n    \n    def training_step(self, batch,batch_idx):\n        input_ids, attention_mask,y = batch[0],batch[1],batch[2] \n        y_pred = self.forward(input_ids,attention_mask)\n        acc = []\n#         print('y_preds',y_pred)\n#         print('y',y)\n        loss = nn.BCEWithLogitsLoss()(y_pred,y)\n#         acc = self.train_acc(y_pred.detach().cpu().numpy().round, torch.argmax(y.squeeze(), dim=1))\n#         acc = accuracy_score(y,np.round(y_pred))\n#         acc = self.train_acc(y_pred, torch.argmax(y.squeeze(), dim=1))\n        output = (y_pred>0.5).float()\n        correct = (output == y).float().sum()\n        acc.append(correct)\n            \n        accuracy = len(acc)/1144 *100\n\n\n#         f1 = self.train_f1(y_pred,y)\n        self.log('train_loss',loss, on_epoch =True)\n        self.log(\"train_accuracy\", accuracy, on_epoch = True,prog_bar = True,on_step = True, logger = True)\n#         self.log(\"train_f1\", f1)\n        return loss\n    \n    \n    def validation_step(self, batch, batch_idx):\n# #      It sets the model in evaluation mode \n        model.eval()\n        input_ids,attention_mask,y,offset_mapping, sequence_ids = batch[0],batch[1],batch[2],batch[3], batch[4]\n        valid_loss = []\n        preds = []\n        offsets = []\n        seq_ids = []\n        valid_labels = []\n        acc = []\n\n        y_preds = self.forward(input_ids, attention_mask)\n        loss = nn.BCEWithLogitsLoss()(y_preds,y)\n        loss = torch.masked_select(loss, y > -1.0).mean()\n#         valid_loss.append(loss.item() * input_ids.size(0))\n#         self.log('val_loss',loss, on_epoch= True)\n\n        preds.append(y_preds.detach().cpu().numpy())\n        offsets.append(offset_mapping.numpy())\n        seq_ids.append(sequence_ids.numpy())\n        valid_labels.append(y.detach().cpu().numpy())\n\n        preds = np.concatenate(preds, axis=0)\n#         print(np.shape(preds))\n        offsets = np.concatenate(offsets, axis=0)\n        seq_ids = np.concatenate(seq_ids, axis=0)\n        valid_labels = np.concatenate(valid_labels, axis=0)\n        location_preds = get_location_predictions(preds, offsets, seq_ids, test=False)\n    #         print(location_preds)\n        score = calculate_char_cv(location_preds, offsets, seq_ids, valid_labels)\n        loss = nn.BCEWithLogitsLoss()(y_preds,y)\n#         acc = self.val_acc(y_preds, torch.argmax(y.squeeze(), dim=1))\n        output = (y_preds>0.5).float()\n        correct = (output == y).float().sum()\n        acc.append(correct)\n        accuracy = len(acc)/286 *100\n        self.log('val_loss',loss, on_epoch =True)\n        self.log(\"val_accuracy\", accuracy, on_epoch = True, prog_bar = True, on_step = True, logger = True)\n        \n        return loss\n        \n        \n    def train_dataloader(self):\n        self.training_data = CustomDataset(X_train, tokenizer, hyperparameters)\n        loader = DataLoader(self.training_data, batch_size=hyperparameters['batch_size'], shuffle=True)\n        return loader\n    \n    def test_dataloader(self):\n        self.test_data = CustomDataset(X_test, tokenizer, hyperparameters)\n        loader = DataLoader(self.test_data, batch_size=hyperparameters['batch_size'], shuffle=False)\n        return loader\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:33:14.758169Z","iopub.execute_input":"2022-08-05T06:33:14.758643Z","iopub.status.idle":"2022-08-05T06:33:14.785195Z","shell.execute_reply.started":"2022-08-05T06:33:14.758608Z","shell.execute_reply":"2022-08-05T06:33:14.783852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# splitting the data into train and test df\n\ntrain_df = prepare_datasets()\n\nX_train, X_test = train_test_split(train_df, test_size=hyperparameters['test_size'],\n                                   random_state=hyperparameters['seed'])\n# Defining tokenizer, train dataloader and test dataloader for the model\n\n# tokenizer = RobertaTokenizerFast.from_pretrained(hyperparameters['model_name'])\n\n\ntokenizer = RobertaTokenizerFast.from_pretrained(hyperparameters['model_name'], do_lower_case=True)\nmodel_config = RobertaConfig().from_pretrained(hyperparameters['model_name'])\nmodel_config.output_hidden_states = True\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:33:18.582089Z","iopub.execute_input":"2022-08-05T06:33:18.582482Z","iopub.status.idle":"2022-08-05T06:33:19.306145Z","shell.execute_reply.started":"2022-08-05T06:33:18.582450Z","shell.execute_reply":"2022-08-05T06:33:19.305219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len(X_train))\n# print(len(X_test))","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:05.811419Z","iopub.execute_input":"2022-08-05T06:31:05.812188Z","iopub.status.idle":"2022-08-05T06:31:05.817115Z","shell.execute_reply.started":"2022-08-05T06:31:05.812136Z","shell.execute_reply":"2022-08-05T06:31:05.815891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:05.819033Z","iopub.execute_input":"2022-08-05T06:31:05.819765Z","iopub.status.idle":"2022-08-05T06:31:05.827702Z","shell.execute_reply.started":"2022-08-05T06:31:05.819720Z","shell.execute_reply":"2022-08-05T06:31:05.826779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Training out Roberta model using Pytorch'''\n\nmodel = CustomModel()\ntdl = model.train_dataloader()\ntdll = model.test_dataloader()\n\n    \ntrainer = pl.Trainer(max_epochs =1) \ntrainer.fit(model,tdl, tdll)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:33:20.916981Z","iopub.execute_input":"2022-08-05T06:33:20.917784Z","iopub.status.idle":"2022-08-05T06:34:29.055578Z","shell.execute_reply.started":"2022-08-05T06:33:20.917744Z","shell.execute_reply":"2022-08-05T06:34:29.054118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.train_dataloader()\n# model.test_dataloader()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.104791Z","iopub.status.idle":"2022-08-05T06:31:37.106272Z","shell.execute_reply.started":"2022-08-05T06:31:37.105943Z","shell.execute_reply":"2022-08-05T06:31:37.105974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.configure_optimizers()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.107689Z","iopub.status.idle":"2022-08-05T06:31:37.108879Z","shell.execute_reply.started":"2022-08-05T06:31:37.108557Z","shell.execute_reply":"2022-08-05T06:31:37.108589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Testing the model'''\n# trainer.test(model,tdll)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.111339Z","iopub.status.idle":"2022-08-05T06:31:37.112138Z","shell.execute_reply.started":"2022-08-05T06:31:37.111767Z","shell.execute_reply":"2022-08-05T06:31:37.111809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**SUBMISSION**","metadata":{}},{"cell_type":"code","source":"def create_test_df():\n    feats = pd.read_csv(f\"{BASE_URL}/features.csv\")\n    notes = pd.read_csv(f\"{BASE_URL}/patient_notes.csv\")\n    test = pd.read_csv(f\"{BASE_URL}/test.csv\")\n\n    merged = test.merge(notes, how = \"left\")\n    merged = merged.merge(feats, how = \"left\")\n\n    def process_feature_text(text):\n        return text.replace(\"-OR-\", \";-\").replace(\"-\", \" \")\n    \n    merged[\"feature_text\"] = [process_feature_text(x) for x in merged[\"feature_text\"]]\n    \n    return merged\n\n\nclass SubmissionDataset(Dataset):\n    def __init__(self, data, tokenizer, config):\n        self.data = data\n        self.tokenizer = tokenizer\n        self.config = config\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        example = self.data.loc[idx]\n        tokenized = self.tokenizer(\n            example[\"feature_text\"],\n            example[\"pn_history\"],\n            truncation = self.config['truncation'],\n            max_length = self.config['max_length'],\n            padding = self.config['padding'],\n            return_offsets_mapping = self.config['return_offsets_mapping']\n        )\n        tokenized[\"sequence_ids\"] = tokenized.sequence_ids()\n\n        input_ids = np.array(tokenized[\"input_ids\"])\n        attention_mask = np.array(tokenized[\"attention_mask\"])\n        offset_mapping = np.array(tokenized[\"offset_mapping\"])\n        sequence_ids = np.array(tokenized[\"sequence_ids\"]).astype(\"float16\")\n\n        return input_ids, attention_mask, offset_mapping, sequence_ids","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.114121Z","iopub.status.idle":"2022-08-05T06:31:37.114798Z","shell.execute_reply.started":"2022-08-05T06:31:37.114480Z","shell.execute_reply":"2022-08-05T06:31:37.114510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = create_test_df()\n\nsubmission_data = SubmissionDataset(test_df, tokenizer, hyperparameters)\nsubmission_dataloader = DataLoader(submission_data, batch_size=hyperparameters['batch_size'], shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.116565Z","iopub.status.idle":"2022-08-05T06:31:37.117168Z","shell.execute_reply.started":"2022-08-05T06:31:37.116839Z","shell.execute_reply":"2022-08-05T06:31:37.116868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Loading the GPU'''\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nmodel = CustomModel().to(DEVICE)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.118889Z","iopub.status.idle":"2022-08-05T06:31:37.119466Z","shell.execute_reply.started":"2022-08-05T06:31:37.119177Z","shell.execute_reply":"2022-08-05T06:31:37.119204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Making prediction from trained model'''\n\nmodel.eval()\npreds = []\noffsets = []\nseq_ids = []\n\nfor batch in tqdm(submission_dataloader):\n    input_ids = batch[0].to(DEVICE)\n    attention_mask = batch[1].to(DEVICE)\n    offset_mapping = batch[2]\n    sequence_ids = batch[3]\n\n    logits = model.forward(input_ids, attention_mask)\n    \n    preds.append(logits.detach().cpu().numpy())\n    offsets.append(offset_mapping.numpy())\n    seq_ids.append(sequence_ids.numpy())\n\npreds = np.concatenate(preds, axis=0)\noffsets = np.concatenate(offsets, axis=0)\nseq_ids = np.concatenate(seq_ids, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.121323Z","iopub.status.idle":"2022-08-05T06:31:37.122119Z","shell.execute_reply.started":"2022-08-05T06:31:37.121789Z","shell.execute_reply":"2022-08-05T06:31:37.121819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.shape(preds)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.123808Z","iopub.status.idle":"2022-08-05T06:31:37.124391Z","shell.execute_reply.started":"2022-08-05T06:31:37.124089Z","shell.execute_reply":"2022-08-05T06:31:37.124116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Applying function to get location of offset'''\n\nlocation_preds = get_location_predictions(preds, offsets, seq_ids, test=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.126316Z","iopub.status.idle":"2022-08-05T06:31:37.126890Z","shell.execute_reply.started":"2022-08-05T06:31:37.126587Z","shell.execute_reply":"2022-08-05T06:31:37.126614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = create_test_df()\ntest_df","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.128480Z","iopub.status.idle":"2022-08-05T06:31:37.129102Z","shell.execute_reply.started":"2022-08-05T06:31:37.128786Z","shell.execute_reply":"2022-08-05T06:31:37.128812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Submission'''\n\ntest_df[\"location\"] = location_preds\ntest_df[[\"id\", \"location\"]].to_csv(\"submission.csv\", index = False)\npd.read_csv(\"submission.csv\").head()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.130987Z","iopub.status.idle":"2022-08-05T06:31:37.131572Z","shell.execute_reply.started":"2022-08-05T06:31:37.131274Z","shell.execute_reply":"2022-08-05T06:31:37.131311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Testing the model'''\n# trainer.test(dataloaders  = tdll)\n# print(f'Accuracy: {test[0][\"accuracy\"]:.2f}')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:31:37.133153Z","iopub.status.idle":"2022-08-05T06:31:37.133735Z","shell.execute_reply.started":"2022-08-05T06:31:37.133424Z","shell.execute_reply":"2022-08-05T06:31:37.133450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}