{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip uninstall -q -y transformers","metadata":{"execution":{"iopub.status.busy":"2022-07-05T02:38:30.469148Z","iopub.execute_input":"2022-07-05T02:38:30.469863Z","iopub.status.idle":"2022-07-05T02:38:32.240694Z","shell.execute_reply.started":"2022-07-05T02:38:30.469822Z","shell.execute_reply":"2022-07-05T02:38:32.239519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/transformers/src\")\nimport transformers\n\nprint(f\"Transformers version: {transformers.__version__}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-05T02:38:32.244708Z","iopub.execute_input":"2022-07-05T02:38:32.245037Z","iopub.status.idle":"2022-07-05T02:38:32.251074Z","shell.execute_reply.started":"2022-07-05T02:38:32.244993Z","shell.execute_reply":"2022-07-05T02:38:32.249997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hyperparameters = {\n    \"max_length\": 416,\n    \"padding\": \"max_length\",\n    \"return_offsets_mapping\": True,\n    \"truncation\": \"only_second\",\n    \"model_name\": (\"../input/layoutlm/BiomedNLP-PubMedBERT\"\n                   \"-base-uncased-abstract-fulltext\"),\n    \"dropout\": 0.2,\n    \"lr\": 1e-5,\n    \"test_size\": 0.2,\n    \"seed\": 1268,\n    \"batch_size\": 8\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-05T02:38:32.252466Z","iopub.execute_input":"2022-07-05T02:38:32.252808Z","iopub.status.idle":"2022-07-05T02:38:32.260426Z","shell.execute_reply.started":"2022-07-05T02:38:32.252772Z","shell.execute_reply":"2022-07-05T02:38:32.259251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport os  # noqa\nimport re\nimport gc\nimport ast\nimport sys  # noqa\nimport pickle  # noqa\nimport random  # noqa\nimport itertools\nimport warnings\nimport itertools  # noqa\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\nfrom IPython import embed  # noqa\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\n\n\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\n\nwarnings.filterwarnings(\"ignore\")\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)\n\nif 'KAGGLE_URL_BASE' in set(os.environ.keys()):\n    OUTPUT_DIR = Path('../temp')\n    OUTPUT_DIR.mkdir(exist_ok=True)\nelse:\n    OUTPUT_DIR = Path('')\n\ndef get_tokenizer(dir_name):\n    \n    if dir_name == 'layoutlm/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext': # [Stephanie]\n        tokenizer = AutoTokenizer.from_pretrained(hyperparameters['model_name'])\n    else:\n        # original tokenizer\n        tokenizer = AutoTokenizer.from_pretrained(INPUT_DIR / dir_name / 'tokenizer', trim_offsets=False)\n    '''\n    [Stephanie]\n    Changed the tokenizer to the PubMedBert trained tokenizer\n    Update [July 1 @ 6:07 PM] - the updated line caused an error :D\n    Update [July 4 @ 12:32 PM] - turned on Internet to access the PubMedBert file path\n        - When I try Save Version, it says ../input/nbme-local-107/config.pth file path not found, but it's right there??\n    '''\n    # checkpointがない場合は、エラーになる  # TODO: どう修正するか検討する\n    return tokenizer\n\n\ndef get_char_probs(texts, predictions, tokenizer):\n    results = [np.zeros(len(t)) for t in texts]\n    for i, (text, prediction) in enumerate(zip(texts, predictions)):\n        encoded = tokenizer(text,\n                            add_special_tokens=True,\n                            return_offsets_mapping=True)\n        for idx, (offset_mapping, pred) in enumerate(zip(encoded['offset_mapping'], prediction)):\n            start = offset_mapping[0]\n            end = offset_mapping[1]\n            results[i][start:end] = pred\n    return results\n\n\ndef format_spans(indices):\n    segs = []\n    left = last = None\n    for i in indices:\n        if left is None:\n            left = last = i\n        elif last + 1 == i:\n            last = i\n        else:\n            # New segment\n            segs.append('%d %d' % (left, last + 1))\n            left = last = i\n\n    if last is not None:\n        segs.append('%d %d' % (left, last + 1))\n\n    return ';'.join(segs)\n\n\ndef get_results(char_probs, threshold=0.5):\n    locs = []\n    for prob in char_probs:\n        pn_history = prob['pn_history']\n        y_prob = prob['character_level_probs']\n\n        list_ = []\n        i_begin = i_last = None\n        last_space = False\n        for i, (x, p) in enumerate(zip(pn_history, y_prob)):\n            if p >= threshold:\n                if i_begin is None:  # 先頭\n                    if x not in (' ', '\\n', '\\r', '\\t'):  # 文字がスペースじゃない\n                        i_begin = i_last = i\n                        list_.append(i)\n                    else:\n                        pass\n                else:  # 先頭じゃない\n                    assert i_last + 1 == i  # i_last + 1 と　i　が同一じゃなかったらエラー\n                    i_last = i\n                    list_.append(i)\n                    if x in (' ', '\\n', '\\r', '\\t'):\n                        last_space = True\n                    else:\n                        last_space = False\n            else:\n                i_begin = i_last = None  # Negative; reset span\n                if last_space:\n                    list_.pop(-1)\n                    last_space = False\n        locs.append(format_spans(list_))\n\n    return locs\n\n\ndef get_results_with_threshold_list(char_probs, threshold_list):\n    locs = []\n    for prob, threshold in zip(char_probs, threshold_list):\n        pn_history = prob['pn_history']\n        y_prob = prob['character_level_probs']\n\n        list_ = []\n        i_begin = i_last = None\n        last_space = False\n        for i, (x, p) in enumerate(zip(pn_history, y_prob)):\n            if p >= threshold:\n                if i_begin is None:  # 先頭\n                    if x not in (' ', '\\n', '\\r', '\\t'):  # 文字がスペースじゃない\n                        i_begin = i_last = i\n                        list_.append(i)\n                    else:\n                        pass\n                else:  # 先頭じゃない\n                    assert i_last + 1 == i  # i_last + 1 と　i　が同一じゃなかったらエラー\n                    i_last = i\n                    list_.append(i)\n                    if x in (' ', '\\n', '\\r', '\\t'):\n                        last_space = True\n                    else:\n                        last_space = False\n            else:\n                i_begin = i_last = None  # Negative; reset span\n                if last_space:\n                    list_.pop(-1)\n                    last_space = False\n\n        locs.append(format_spans(list_))\n\n    return locs\n\n# ====================================================\n# Dataset\n# ====================================================\ndef create_label(tokenizer, pn_history, annotation_length, location_list, max_len):\n    encoded = tokenizer(pn_history,\n                        add_special_tokens=True,\n                        max_length=max_len,\n                        padding=\"max_length\",\n                        return_offsets_mapping=True)\n    offset_mapping = encoded['offset_mapping']\n    ignore_idxes = np.where(np.array(encoded.sequence_ids()) != 0)[0]\n    label = np.zeros(len(offset_mapping))\n    label[ignore_idxes] = -1\n    if annotation_length != 0:\n        for location in location_list:\n            for loc in [s.split() for s in location.split(';')]:\n                start_idx = -1\n                end_idx = -1\n                start, end = int(loc[0]), int(loc[1])\n                for idx in range(len(offset_mapping)):\n                    if (start_idx == -1) & (start < offset_mapping[idx][0]):\n                        start_idx = idx - 1\n                    if (end_idx == -1) & (end <= offset_mapping[idx][1]):\n                        end_idx = idx + 1\n                if start_idx == -1:\n                    start_idx = end_idx\n                if (start_idx != -1) & (end_idx != -1):\n                    label[start_idx:end_idx] = 1\n    return label\n\n\ndef df_merge(df, mode, preprocess_feature):\n    # マージ\n    patient_notes = pd.read_csv(INPUT_DIR / 'nbme-score-clinical-patient-notes' / \"patient_notes.csv\")\n    features = pd.read_csv(INPUT_DIR / 'nbme-score-clinical-patient-notes' / \"features.csv\")\n    if preprocess_feature:\n        features['feature_text'] = features['feature_text'].apply(process_feature_text)\n    df = df.merge(features, how='left', on=[\"case_num\", \"feature_num\"])\n    df = df.merge(patient_notes, how=\"left\", on=['case_num', 'pn_num'])\n\n    if mode != 'test':\n        # データ形式の変換\n        df['anno_list'] = df['annotation'].apply(ast.literal_eval)\n        df['loc_list'] = df['location'].apply(ast.literal_eval)\n        df['anno_length'] = df['anno_list'].apply(len)\n\n    return df\n\n\nimport torch.nn.functional as F\n\nclass CustomModel(nn.Module): # [Stephanie]\n    def __init__(self, config):\n        super().__init__()\n        self.bert = AutoModel.from_pretrained(config['model_name']) \n        self.dropout = nn.Dropout(p=config['dropout'])\n        self.config = config\n        self.fc1 = nn.Linear(768, 512)\n        self.fc2 = nn.Linear(512, 1)\n        \n\n    def forward(self, input_ids, attention_mask, token_type_ids):\n        outputs = self.bert(input_ids=input_ids,\n                            attention_mask=attention_mask,\n                            token_type_ids=token_type_ids)\n        logits = F.relu(self.fc1(outputs[0]))\n        logits = self.fc2(self.dropout(logits)).squeeze(-1)\n        return logits\n\n\nclass CustomModelTest(nn.Module):\n    def __init__(self, model_dir, config_path=None, pretrained=False, output_dim=1):\n        super().__init__()\n        if config_path is None:\n            self.config = AutoConfig.from_pretrained(model_dir, output_hidden_states=True)\n        else:\n            self.config = torch.load(config_path)\n        if pretrained:\n            self.model = AutoModel.from_pretrained(model_dir, config=self.config)\n        else:\n            self.model = AutoModel.from_config(self.config)\n        # self.fc_dropout = nn.Dropout(args.fc_dropout)\n        self.fc = nn.Linear(self.config.hidden_size, output_dim)\n\n    def forward(self, tokens, attention_mask, token_type_ids):\n        outputs = self.model(\n            tokens,\n            attention_mask=attention_mask,\n            token_type_ids=token_type_ids,\n        )\n        feature = outputs[0]  # last_hidden_states\n\n        output = self.fc(feature)\n        return output\n    \ndef create_labels_for_scoring_fix(df):\n    # example: ['0 1', '3 4'] -> ['0 1; 3 4']\n    # 'loc_list'から作成するように変更\n    df['location_for_create_labels'] = [ast.literal_eval('[]')] * len(df)\n    for i in range(len(df)):\n        lst = df.loc[i, 'loc_list']\n        if lst:\n            new_lst = ';'.join(lst)\n            df.loc[i, 'location_for_create_labels'] = ast.literal_eval(f'[[\"{new_lst}\"]]')\n    # create labels\n    truths = []\n    for location_list in df['location_for_create_labels'].values:\n        truth = []\n        if len(location_list) > 0:\n            location = location_list[0]\n            for loc in [s.split() for s in location.split(';')]:\n                start, end = int(loc[0]), int(loc[1])\n                truth.append([start, end])\n        truths.append(truth)\n    return truths\n\n\ndef prepare_input(tokenizer, pn_history, feature_text, return_pn_history=False,\n                  return_offsets_mapping=False, padding='max_length', max_len=500):\n    inputs = tokenizer(pn_history,\n                       feature_text,\n                       add_special_tokens=True,\n                       max_length=max_len,\n                       truncation=True,\n                       padding=padding,\n                       return_offsets_mapping=return_offsets_mapping)\n    if return_pn_history:\n        inputs['pn_history'] = pn_history\n    return inputs\n\n\nclass TestDataset(Dataset):\n    def __init__(self, tokenizer, df, max_len, return_label=False):\n        self.tokenizer = tokenizer\n        self.feature_texts = df['feature_text'].values\n        self.pn_historys = df['pn_history'].values\n        if return_label:\n            self.annotation_lengths = df['anno_length'].values\n            self.locations = df['loc_list'].values\n        self.return_label = return_label\n        self.max_len = max_len\n        if return_label:\n            self.labels = []\n            for i in range(len(df)):\n                label = create_label(\n                    tokenizer,\n                    df.loc[i, 'pn_history'],\n                    df.loc[i, 'anno_length'],\n                    df.loc[i, 'loc_list'],\n                    max_len\n                    )\n                self.labels.append(label)\n\n    def __len__(self):\n        return len(self.pn_historys)\n\n    def __getitem__(self, idx):\n        inputs = prepare_input(self.tokenizer,\n                               self.pn_historys[idx],\n                               self.feature_texts[idx],\n                               return_pn_history=True,\n                               return_offsets_mapping=True,\n                               padding=False,\n                               max_len=self.max_len\n                               )\n\n        if not self.return_label:\n            return inputs\n        else:\n            inputs['label'] = self.labels[idx]\n        return inputs\n\n\nclass Collate:\n    def __init__(self, tokenizer, return_label=False):\n        self.tokenizer = tokenizer\n        self.return_label = return_label\n\n    def __call__(self, batch, label=None):\n        output = dict()\n        output[\"offset_mapping\"] = [sample[\"offset_mapping\"] for sample in batch]\n        output[\"pn_history\"] = [sample[\"pn_history\"] for sample in batch]\n\n        output[\"input_ids\"] = [sample[\"input_ids\"] for sample in batch]\n        output[\"attention_mask\"] = [sample[\"attention_mask\"] for sample in batch]\n        output[\"token_type_ids\"] = [sample[\"token_type_ids\"] for sample in batch]\n\n        # calculate max token length of this batch\n        batch_max = max([len(token_id) for token_id in output[\"input_ids\"]])\n\n        # add padding\n        if self.tokenizer.padding_side == \"right\":\n            output[\"input_ids\"] = [s + (batch_max - len(s)) * [self.tokenizer.pad_token_id] for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [s + (batch_max - len(s)) * [0] for s in output[\"attention_mask\"]]\n            output[\"token_type_ids\"] = [s + (batch_max - len(s)) * [0] for s in output[\"token_type_ids\"]]\n        else:\n            output[\"input_ids\"] = [(batch_max - len(s)) * [self.tokenizer.pad_token_id] + s for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [(batch_max - len(s)) * [0] + s for s in output[\"attention_mask\"]]\n            output[\"token_type_ids\"] = [(batch_max - len(s)) * [0] + s for s in output[\"token_type_ids\"]]\n\n        # convert to tensors\n        output[\"input_ids\"] = torch.tensor(output[\"input_ids\"], dtype=torch.long)\n        output[\"attention_mask\"] = torch.tensor(output[\"attention_mask\"], dtype=torch.long)\n        output[\"token_type_ids\"] = torch.tensor(output[\"token_type_ids\"], dtype=torch.long)\n\n        if self.return_label:\n            labels = torch.tensor([sample['label'] for sample in batch], dtype=torch.float)\n            output['labels'] = labels[:, :batch_max]\n\n        return output\n\n\ndef make_oof(device, model_dir, max_len, num_fold, ckpt_path, output_dim):\n    \n    if model_dir == 'layoutlm/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext':\n        model = CustomModel(hyperparameters).to(device)\n    else:\n        model = CustomModelTest(model_dir, config_path=INPUT_DIR / model_dir / 'config.pth', pretrained=False,\n                            output_dim=output_dim)\n    results = []\n    for fold in range(num_fold):\n        if model_dir == 'layoutlm/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext': # [Stephanie]\n            model.load_state_dict(torch.load('../input/pytorch-pubmedbert-nbme/nbme_pubmed_bert.pth'))\n        else:\n            model.load_state_dict(torch.load(Path('../input') / model_dir / ckpt_path[fold]))\n\n        results_n = {\n            'id': [],\n            'token_mask': [],\n            'offset_mapping': [],\n            'probability': [],\n        }\n        model.eval()\n        model.to(device)\n\n        for inputs in tqdm(te_loader):\n            input_ids = inputs['input_ids'].to(device)\n            token_mask = inputs['attention_mask'].to(device)\n            token_type_ids = inputs['token_type_ids'].to(device)\n            with torch.no_grad():\n                y_preds = model(input_ids, token_mask, token_type_ids).sigmoid().cpu().numpy() # Shape of y_preds is wrong\n                \n                if model_dir == 'layoutlm/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext': # [Stephanie]\n                    y_preds = y_preds.reshape(y_preds.shape[0], y_preds.shape[1], 1)\n                \n                if y_preds.shape[1] > max_len:\n                    y_preds[:, :max_len]\n                else:\n                    y_preds = np.pad(y_preds, ((0, 0), (0, max_len - y_preds.shape[1]), (0, 0)), 'constant', constant_values=0)\n                results_n['probability'].append(y_preds[:, :, :1])\n                if fold == 0:\n                    results_n['offset_mapping'] += [x for x in inputs['offset_mapping']]\n\n        torch.cuda.empty_cache()\n\n        # -------\n        print('')\n        if fold == 0:\n            results.append({\n                'probability': np.concatenate(results_n['probability']),\n                'offset_mapping': np.array(results_n['offset_mapping'], object)\n            })\n        else:\n            results.append({\n                'probability': np.concatenate(results_n['probability'])\n            })\n        del y_preds, results_n\n\n    # ----\n    del model\n    gc.collect()\n    torch.cuda.empty_cache()\n\n    agg_results = []\n    num_samples = len(results[0]['probability'])\n    for i in range(num_samples):\n        preds = []\n        for n in range(num_fold):\n            preds.append(results[n]['probability'][i])\n        pred = np.mean(preds, axis=0)\n        agg_results.append((pred.squeeze(), (results[0]['offset_mapping'][i])))\n\n    return agg_results\n\n\ndef process_feature_text(text):\n    text = re.sub('I-year', '1-year', text)\n    text = re.sub('-OR-', \" or \", text)\n    text = re.sub('-', ' ', text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-07-05T02:38:32.264441Z","iopub.execute_input":"2022-07-05T02:38:32.264858Z","iopub.status.idle":"2022-07-05T02:38:32.348601Z","shell.execute_reply.started":"2022-07-05T02:38:32.26483Z","shell.execute_reply":"2022-07-05T02:38:32.347671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# main","metadata":{}},{"cell_type":"markdown","source":"## 設定ファイル読み込み","metadata":{}},{"cell_type":"code","source":"INPUT_DIR = Path('../input')\nNUM_JOBS = 4\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T02:38:32.349997Z","iopub.execute_input":"2022-07-05T02:38:32.350939Z","iopub.status.idle":"2022-07-05T02:38:32.356078Z","shell.execute_reply.started":"2022-07-05T02:38:32.350898Z","shell.execute_reply":"2022-07-05T02:38:32.354799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 設定","metadata":{}},{"cell_type":"code","source":"###\n# 設定\n###\nDEBUG = False  # DEBUGだと、それぞれのモデルで2foldのみ\n\n\nMODEL_WEIGHTS = [1.0, 0.83571, 0.69586, 0.79665, 0.81276, 0.71487]  # どの割合で足し合わせるか設定。足して1にする必要はない。\nUSE_THRESHOLD_LIST = None\nif USE_THRESHOLD_LIST:\n    case_num_weights = {\n    }    \n\nconfig1 = {'model_dir': 'nbme-local-107',  # LB 0.893\n      'max_len': 350,\n      'batch_size': 80,\n      'model_name': 'microsoft/deberta-v3-large',\n      'preprocess_feature': False,  # process_feature_textをしないならFalse、するならTrue\n      'ckpt_path': [# 'model_0.bin',\n                    'model_1.bin',\n                    'model_2.bin',\n                    'model_3.bin',\n                    'model_4.bin',\n                    'model_all.bin',\n                   ],\n       'output_dim': 1  # 最終層の出力\n      }\n\nconfig2 = {'model_dir': 'nbme-local-092',  # LB 0.893\n          'max_len': 355,\n          'batch_size': 60,\n          'model_name': 'microsoft/deberta-v2-xlarge',\n          'preprocess_feature': False,  # process_feature_textをしないならFalse、するならTrue\n          'ckpt_path': ['model_0.bin',\n                        # 'model_1.bin',\n                        # 'model_2.bin',\n                        'model_3.bin',\n                        'model_4.bin',\n                        'model_all.bin'\n                        \n                       ],\n           'output_dim': 1  # 最終層の出力\n          }\n\nconfig3 = {'model_dir': 'nbme-local-106',  # LB: 0.892\n          'max_len': 500,\n          'batch_size': 60,\n          'model_name': 'microsoft/deberta-large',\n          'preprocess_feature': False,  # process_feature_textをしないならFalse、するならTrue\n          'ckpt_path': ['model_0.bin',\n                        'model_1.bin',\n                        # 'model_2.bin',\n                        # 'model_3.bin',\n                        # 'model_4.bin',\n                        'model_all.bin'\n                       ],\n           'output_dim': 1  # 最終層の出力\n          }\n\nconfig4 = {'model_dir': 'nbme-local-102',  # LB 0.892\n          'max_len': 355,\n          'batch_size': 60,\n          'model_name': 'microsoft/deberta-v2-xlarge',\n          'preprocess_feature': False,  # process_feature_textをしないならFalse、するならTrue\n          'ckpt_path': ['model_0.bin',\n                        'model_1.bin',\n                        'model_2.bin',\n                        # 'model_3.bin',\n                        # 'model_4.bin',\n                        'model_all.bin'\n                        \n                       ],\n           'output_dim': 3  # 最終層の出力\n          }\n\nconfig5 = {'model_dir': 'nbme-local-101',\n          'max_len': 500,\n          'batch_size': 60,\n          'model_name': 'microsoft/deberta-large',\n          'preprocess_feature': False,  # process_feature_textをしないならFalse、するならTrue\n          'ckpt_path': ['model_0.bin',\n                        'model_1.bin',\n                        'model_2.bin',\n                        'model_3.bin',\n                        # 'model_4.bin',\n                        # 'model_all.bin'\n                       ],\n           'output_dim': 3  # 最終層の出力\n          }\n\nconfig6 = {'model_dir': 'layoutlm/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext',  # LB 0.893\n      'max_len': 416, \n      'batch_size': 8,\n      'model_name': 'microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract',\n      'preprocess_feature': False,  # process_feature_textをしないならFalse、するならTrue\n      'ckpt_path': ['pytorch_model.bin'\n                    # 'model_0.bin',\n                    # 'model_1.bin',\n                    # 'model_2.bin',\n                    # 'model_3.bin',\n                    # 'model_4.bin',\n                    # 'model_all.bin',\n                   ],\n       'output_dim': 1  # 最終層の出力\n      }\n'''\nconfig6 = {'model_dir': 'nbme-local-104',  # LB\n          'max_len': 355,\n          'batch_size': 60,\n          'model_name': 'microsoft/deberta-v2-xlarge',\n          'preprocess_feature': False,  # process_feature_textをしないならFalse、するならTrue\n          'ckpt_path': [# 'model_0.bin',\n                        'model_1.bin',\n                        'model_2.bin',\n                        # 'model_3.bin',\n                        # 'model_4.bin',\n                        'model_all.bin'\n                        \n                       ],\n           'output_dim': 1  # 最終層の出力\n          }\n'''\n# config_listにconfigを入れておく\nconfig_list = [config1, config2, config3, config4, config5, config6]\n\n###\n# 設定ここまで\n###\n\nassert len(config_list) == len(MODEL_WEIGHTS), 'MODEL_WEIGHTS length mismatch'","metadata":{"execution":{"iopub.status.busy":"2022-07-05T02:38:32.357833Z","iopub.execute_input":"2022-07-05T02:38:32.35914Z","iopub.status.idle":"2022-07-05T02:38:32.375315Z","shell.execute_reply.started":"2022-07-05T02:38:32.359056Z","shell.execute_reply":"2022-07-05T02:38:32.37436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    for i in range(len(config_list)):\n        config_list[i]['ckpt_path'] = config_list[i]['ckpt_path'][:2]\n\nmax_lengths = []\nfor i in range(len(config_list)):\n    max_lengths.append(config_list[i]['max_len'])\n\ndisplay(config_list)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T02:38:32.378185Z","iopub.execute_input":"2022-07-05T02:38:32.379385Z","iopub.status.idle":"2022-07-05T02:38:32.391707Z","shell.execute_reply.started":"2022-07-05T02:38:32.379348Z","shell.execute_reply":"2022-07-05T02:38:32.39063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## サブミットファイル作成","metadata":{}},{"cell_type":"code","source":"for num, config in enumerate(config_list):\n    model_dir = config['model_dir']\n    max_len = config['max_len']\n    batch_size = config['batch_size']\n    model_name = config['model_name']\n    ckpt_path = config['ckpt_path']\n    num_fold = len(ckpt_path)\n    preprocess_feature = config['preprocess_feature']\n    output_dim = config['output_dim']\n\n    tokenizer = get_tokenizer(model_dir)\n    test = pd.read_csv(INPUT_DIR / '../input/nbme-score-clinical-patient-notes/test.csv')\n    test = df_merge(test, mode='test', preprocess_feature=preprocess_feature)\n    print(test)\n    test_dataset = TestDataset(tokenizer, test, max_len, return_label=False)\n\n    collate = Collate(tokenizer=tokenizer, return_label=False)\n    loader_params = {'batch_size': batch_size,\n                     'num_workers': NUM_JOBS,\n                     'pin_memory': False,\n                     }\n    te_loader = DataLoader(test_dataset, shuffle=False, drop_last=False, collate_fn=collate, **loader_params)\n    results = make_oof(device, model_dir, max_len, num_fold, ckpt_path, output_dim)\n    with open(OUTPUT_DIR / f'oof_word_predict_{num}.pickle', 'wb') as f:\n        pickle.dump(results, f)\n    del results\n    gc.collect()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-05T02:38:32.393319Z","iopub.execute_input":"2022-07-05T02:38:32.394706Z","iopub.status.idle":"2022-07-05T02:39:19.849736Z","shell.execute_reply.started":"2022-07-05T02:38:32.394647Z","shell.execute_reply":"2022-07-05T02:39:19.847408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_word_predict_list = []\nfor num in range(len(config_list)):\n    with open(OUTPUT_DIR / f'oof_word_predict_{num}.pickle', 'rb') as f:\n        results = pickle.load(f)\n    oof_word_predict_list.append(results)\n\nchar_probs = []\nfor i in range(len(test)):\n    pn_history = test.loc[i, 'pn_history']\n    character_level_probs = []\n\n    for num in range(len(config_list)):\n        token_to_text_probability = np.full((len(pn_history)), 0, np.float32)\n        p, offset_mapping = oof_word_predict_list[num][i]\n        end_prev = 0\n        for t, (start, end) in enumerate(offset_mapping):\n            if t == max_lengths[num] - 1:\n                break\n            elif end_prev > 0 and start == 0 and end == 0:\n                break\n            else:\n                token_to_text_probability[end_prev:end] += p[t]\n                end_prev = end\n        character_level_probs.append(token_to_text_probability)\n\n    if len(MODEL_WEIGHTS) >= 2:\n        pred = np.average(character_level_probs, axis=0, weights=MODEL_WEIGHTS)\n    else:\n        pred = character_level_probs[0]\n    if i < 5:\n        print(np.max(pred))\n    char_probs.append({'pn_history': pn_history, 'character_level_probs': pred})\n\nif not USE_THRESHOLD_LIST:\n    result1 = get_results(char_probs, threshold=0.5)\n\nelse:\n    print(case_num_weights)\n    threshold_list = test['case_num'].map(case_num_weights).values\n    print(threshold_list[:5])\n    result1 = get_results_with_threshold_list(char_probs, threshold_list)\n\nsubmission = pd.read_csv('../input/nbme-score-clinical-patient-notes/sample_submission.csv')\nsubmission['location'] = result1\nprint(submission.head())\nsubmission[['id', 'location']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T02:39:19.851078Z","iopub.status.idle":"2022-07-05T02:39:19.851602Z","shell.execute_reply.started":"2022-07-05T02:39:19.851339Z","shell.execute_reply":"2022-07-05T02:39:19.851365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}