{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n#### This is an inference notebook I literally took from its training version: https://www.kaggle.com/code/nbroad/token-classification-approach-fpe\n\nAdded some little patches on how to extract predictions, which is no more than just working.\n\nPlease upvote the original notebook if you like :)","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold\nimport gc\nimport random\nimport warnings\nimport torch\nfrom transformers import Trainer, TrainingArguments, AutoModelForTokenClassification, DataCollatorForTokenClassification, AutoTokenizer, AutoConfig\nfrom itertools import chain\nfrom text_unidecode import unidecode\nfrom typing import Tuple\nimport codecs\nimport re\nfrom functools import partial\nimport datasets\n\n\nwarnings.filterwarnings(\"ignore\")\ngc.collect()\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nINPUT_DIR = \"../input/feedback-prize-effectiveness/\"\nmodel_path = '../input/feedback-token-deberta-v3-large-sep/'\n\nclass CFG:\n    model = \"deberta-v3-large\"\n    max_len = 2048\n    batch_size = 2\n    epochs = 4\n    n_fold = 5\n    trn_fold = [0, 1]\n    lr = 1e-5\n    weight_decay = 1e-2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n    return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n    return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\n# Register the encoding and decoding error handlers for `utf-8` and `cp1252`.\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n    \"\"\"Resolve the encoding problems and normalize the abnormal characters.\"\"\"\n    text = (\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n    )\n    text = unidecode(text)\n    return text\n\n\ndef get_essay_text(sample, data_dir):\n    id_ = sample[\"essay_id\"]\n    with open(data_dir + \"test/\" + f\"{id_}.txt\", \"r\") as fp:\n        sample[\"essay_text\"] = resolve_encodings_and_normalize(fp.read())\n    return sample","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(model_path + 'tokenizer')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disc_types = [\n    \"Claim\",\n    \"Concluding Statement\",\n    \"Counterclaim\",\n    \"Evidence\",\n    \"Lead\",\n    \"Position\",\n    \"Rebuttal\",\n]\ncls_tokens_map = {label: f\"[CLS_{label.upper()}]\" for label in disc_types}\nend_tokens_map = {label: f\"[END_{label.upper()}]\" for label in disc_types}\n\nlabel2id = {\n    \"Adequate\": 0,\n    \"Effective\": 1,\n    \"Ineffective\": 2,\n}\n\ncls_id_map = {\n    label: tokenizer.encode(tkn)[1] for label, tkn in cls_tokens_map.items()\n}\n\nid_cls_map = {v: k for k, v in cls_id_map.items()}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_positions(sample):\n    text = sample[\"essay_text\"][0]\n\n    # keeps track of what has already\n    # been located\n    min_idx = 0\n\n    # stores start and end indexes of discourse_texts\n    idxs = []\n\n    for dt in sample[\"discourse_text\"]:\n        # calling strip is essential\n        matches = list(re.finditer(re.escape(dt.strip()), text))\n\n        # If there are multiple matches, take the first one\n        # that is past the previous discourse texts.\n        if len(matches) > 1:\n            for m in matches:\n                if m.start() >= min_idx:\n                    break\n        # If no matches are found\n        elif len(matches) == 0:\n            idxs.append([-1])  # will filter out later\n            continue\n            # If one match is found\n        else:\n            m = matches[0]\n\n        idxs.append([m.start(), m.end()])\n\n        min_idx = m.start()\n\n    return idxs\n\ndef tokenize(sample):\n    sample[\"idxs\"] = find_positions(sample)\n\n    text = sample[\"essay_text\"][0]\n    chunks = []\n    prev = 0\n\n    zipped = zip(\n        sample[\"idxs\"],\n        sample[\"discourse_type\"],\n    )\n    for idxs, disc_type in zipped:\n        # when the discourse_text wasn't found\n        if idxs == [-1]:\n            continue\n\n        s, e = idxs\n\n        # if the start of the current discourse_text is not\n        # at the end of the previous one.\n        # (text in between discourse_texts)\n        if s != prev:\n            chunks.append(text[prev:s])\n            prev = s\n\n        # if the start of the current discourse_text is\n        # the same as the end of the previous discourse_text\n        if s == prev:\n            chunks.append(cls_tokens_map[disc_type])\n            chunks.append(text[s:e])\n            chunks.append(end_tokens_map[disc_type])\n\n        prev = e\n\n    tokenized = tokenizer(\n        \" \".join(chunks),\n        padding=False,\n        truncation=True,\n        max_length=CFG.max_len,\n        add_special_tokens=True,\n    )\n\n    return tokenized","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(INPUT_DIR + \"test.csv\")\n\nessay_text_ds = datasets.Dataset.from_dict({\"essay_id\": test_df.essay_id.unique()})\nessay_text_ds = essay_text_ds.map(\n        partial(get_essay_text, data_dir=INPUT_DIR),\n        num_proc=1,\n        batched=False,\n        desc=\"Loading text files\",\n)\nessay_text_df = essay_text_ds.to_pandas()\n\ntest_df[\"discourse_text\"] = [resolve_encodings_and_normalize(x) for x in test_df[\"discourse_text\"]]\ntest_df = test_df.merge(essay_text_df, on=\"essay_id\", how=\"left\")\ndel essay_text_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# track the matchings of each discourse in its essay, by the order given in the csv file\n\ndiscourse_text_values = test_df['discourse_text'].values\nessay_text_values = test_df['essay_text'].values\n\nmatches = []\nfor i, dt in enumerate(discourse_text_values):\n    if dt.strip() in essay_text_values[i]:\n        matches.append(1)\n    else:\n        matches.append(0)\ntest_df['match'] = matches","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_df = test_df.groupby([\"essay_id\"]).agg(list)\n\nds = datasets.Dataset.from_pandas(grouped_df)\nds = ds.map(\n        tokenize,\n        batched=False,\n        num_proc=1,\n        desc=\"Tokenizing\",\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bad_matches = []\ncls_ids = set(list(cls_id_map.values()))\nfor id_, ids, dt in zip(ds[\"essay_id\"], ds[\"input_ids\"], ds[\"discourse_id\"]):\n    # count number of cls ids\n    num_cls_id = sum([x in cls_ids for x in ids])\n    # true number of discourse_texts\n    num_dt = len(dt)\n\n    if num_cls_id != num_dt:\n        bad_matches.append((id_, ids, dt))\n\nprint(\"Num bad matches:\", len(bad_matches))\nprint()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"collator_fn = DataCollatorForTokenClassification(\n    tokenizer=tokenizer, pad_to_multiple_of=8, padding=True\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_config = AutoConfig.from_pretrained(model_path)\n\npreds = []\n\nfor fold in range(CFG.n_fold):\n    if fold in CFG.trn_fold:\n        print(f\"fold = {fold}\")\n        \n        checkpoint = model_path + f\"{CFG.model.replace('/', '-')}_fold{fold}_best.bin\"\n        model = AutoModelForTokenClassification.from_pretrained(checkpoint, config=model_config)\n        \n        keep_cols = {\"input_ids\", \"attention_mask\"}\n        test_dataset = ds.remove_columns([c for c in ds.column_names if c not in keep_cols])\n        \n        trainer = Trainer(\n            model=model,\n            tokenizer=tokenizer,\n            data_collator=collator_fn,\n        )\n        \n        pred = trainer.predict(test_dataset)\n        preds.append(pred[0])\n        \n        del model\n        gc.collect()\n        torch.cuda.empty_cache()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get predicted logits\n\npreds = np.array(preds)\nlogits = np.exp(preds) / np.expand_dims(np.sum(np.exp(preds), axis=-1), axis=-1)\n\nmodel_preds = np.mean(logits, axis=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract the predicted labels for all [discourse_type_CLS] \n\nhead_preds = []\nfor i, sample in enumerate(model_preds):\n    sample_pred = []\n    sample_ids = ds['input_ids'][i]\n    for j, tk_id in enumerate(sample_ids):\n        if tk_id in cls_ids:\n            sample_pred.append(sample[j])\n    head_preds.append(sample_pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# collect predictions of each discourse by the order given in the csv file\n\nessay_id_map = {v : k for k, v in enumerate(ds['essay_id'])}\n\nfinal_preds = []\n\nordered_essay_ids = test_df['essay_id'].values\ndisordered_essay_matches = grouped_df['match'].values\n\npre_essay_id = ''\nfor essay_id in ordered_essay_ids:\n    if essay_id == pre_essay_id:\n        continue\n    pre_essay_id = essay_id\n    essay_pred = head_preds[essay_id_map[essay_id]]\n    essay_macth = disordered_essay_matches[essay_id_map[essay_id]]\n    for i, discourse_match in enumerate(essay_macth):\n        if discourse_match == 1 and i < len(essay_pred):\n            final_preds.append(essay_pred[i])\n        else:\n            final_preds.append([0., 0., 0.])\n            \nfinal_preds = np.array(final_preds)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_Ineffective = final_preds[:, 2]\npreds_Adequate = final_preds[:, 0]\npreds_Effective = final_preds[:, 1]\n\nsample = pd.read_csv(INPUT_DIR + 'sample_submission.csv')\n\nsample['Ineffective'] = preds_Ineffective\nsample['Adequate'] = preds_Adequate\nsample['Effective'] = preds_Effective\n\n\nsample.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(sample)","metadata":{},"execution_count":null,"outputs":[]}]}