{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Intro\n\nThis inference notebook is the companion to [this Training Notebook](https://www.kaggle.com/code/lextoumbourou/feedback-prize-eda-and-model-training).\n\nHowever, note that I have to trained the model in Google Colab, as Kaggle doesn't provide a long enough run time.\n\nSee [this discussion](https://www.kaggle.com/competitions/feedback-prize-effectiveness/discussion/333277) for the changes made to achieve the score.\n\nI have also made the following changes just to the inference kernel:\n\n* Use fast tokeniser: `use_fast=True`\n* Add `resolve_encodings` from the [4th place solution](https://www.kaggle.com/competitions/feedback-prize-2021/discussion/313330).\n\nAlso using a Kaggle Dataset means this notebook won't break when I change the training code 🙂","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nfrom types import SimpleNamespace\nimport logging\nfrom text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nimport codecs\n\nimport torch\nfrom tqdm.notebook import tqdm\nfrom datasets import Dataset\nfrom transformers import AutoModel, AutoModelForSequenceClassification, AutoTokenizer, AutoConfig, DataCollatorWithPadding\nfrom transformers import TrainingArguments, Trainer\nimport numpy as np\nimport pandas as pd\nimport sklearn\nimport glob, pandas as pd, numpy as np, re\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom scipy.special import softmax\n\n# To work around the aggressive HuggingFace log spam.\nlogging.disable(logging.WARNING)\n\n# From this Gist: https://gist.github.com/ihoromi4/b681a9088f348942b01711f251e5f964\ndef seed_everything(seed: int):\n    import random, os\n    import numpy as np\n    import torch\n    \n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:57:55.540168Z","iopub.execute_input":"2022-07-09T08:57:55.540943Z","iopub.status.idle":"2022-07-09T08:58:02.157771Z","shell.execute_reply.started":"2022-07-09T08:57:55.540854Z","shell.execute_reply":"2022-07-09T08:58:02.157016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cpu'\nif torch.cuda.is_available:\n    device = 'cuda'","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:02.161493Z","iopub.execute_input":"2022-07-09T08:58:02.161703Z","iopub.status.idle":"2022-07-09T08:58:02.165713Z","shell.execute_reply.started":"2022-07-09T08:58:02.161676Z","shell.execute_reply":"2022-07-09T08:58:02.164621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:02.167687Z","iopub.execute_input":"2022-07-09T08:58:02.168063Z","iopub.status.idle":"2022-07-09T08:58:02.180687Z","shell.execute_reply.started":"2022-07-09T08:58:02.168018Z","shell.execute_reply":"2022-07-09T08:58:02.179973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{"execution":{"iopub.status.busy":"2022-06-06T22:11:23.363932Z","iopub.status.idle":"2022-06-06T22:11:23.364284Z","shell.execute_reply.started":"2022-06-06T22:11:23.364113Z","shell.execute_reply":"2022-06-06T22:11:23.36413Z"}}},{"cell_type":"code","source":"config = SimpleNamespace()\n\nconfig.n_folds = 5\nconfig.seed = 420\nconfig.max_len = 512\nconfig.lr = 1e-5\nconfig.weight_decay = 0.01\nconfig.epochs = 5\nconfig.batch_size = 16\nconfig.warm_up_ratio = 0.1\nconfig.hidden_dropout_prob = 0.1\nconfig.output_path = Path('./')\nconfig.input_path = Path('../input/feedback-prize-effectiveness')\nconfig.model_path = Path('../input/feedback-deberta-v3-large/colab_output/deberta-v3-large-20220623-090623')","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:02.182638Z","iopub.execute_input":"2022-07-09T08:58:02.182926Z","iopub.status.idle":"2022-07-09T08:58:02.190064Z","shell.execute_reply.started":"2022-07-09T08:58:02.182893Z","shell.execute_reply":"2022-07-09T08:58:02.189085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(config.input_path / 'test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:02.192846Z","iopub.execute_input":"2022-07-09T08:58:02.193232Z","iopub.status.idle":"2022-07-09T08:58:02.207405Z","shell.execute_reply.started":"2022-07-09T08:58:02.193126Z","shell.execute_reply":"2022-07-09T08:58:02.206737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate Topics","metadata":{}},{"cell_type":"markdown","source":"Load the topic model and libraries created in [this](https://www.kaggle.com/code/lextoumbourou/feedback-topics-identification) notebook.","metadata":{}},{"cell_type":"code","source":"topic_pred_df = pd.read_csv('../input/feedback-topics-identification/topic_model_feedback.csv')\ntopic_pred_df = topic_pred_df.drop(columns={'prob'})\ntopic_pred_df = topic_pred_df.rename(columns={'id': 'essay_id'})\n\ntopic_meta_df = pd.read_csv('../input/feedback-topics-identification/topic_model_metadata.csv')\ntopic_meta_df = topic_meta_df.rename(columns={'Topic': 'topic', 'Name': 'topic_name'}).drop(columns=['Count'])\ntopic_meta_df.topic_name = topic_meta_df.topic_name.apply(lambda n: ' '.join(n.split('_')[1:]))\n\ntopic_pred_df = topic_pred_df.merge(topic_meta_df, on='topic', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:02.208588Z","iopub.execute_input":"2022-07-09T08:58:02.208908Z","iopub.status.idle":"2022-07-09T08:58:02.247899Z","shell.execute_reply.started":"2022-07-09T08:58:02.208873Z","shell.execute_reply":"2022-07-09T08:58:02.247169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/feedback-topics-identification/site-packages')\nfrom bertopic import BERTopic\n\ntopic_model = BERTopic.load(\"../input/feedback-topics-identification/feedback_2021_topic_model\")\n\nsws = stopwords.words(\"english\") + [\"n't\",  \"'s\", \"'ve\"]\nfls = glob.glob(\"../input/feedback-prize-effectiveness/test/*.txt\")\ndocs = []\nfor fl in tqdm(fls):\n    with open(fl) as f:\n        txt = f.read()\n        word_tokens = word_tokenize(txt)\n        txt = \" \".join([w for w in word_tokens if not w.lower() in sws])\n    docs.append(txt)\n\ntopics, probs = topic_model.transform(docs)\n\npred_topics = pd.DataFrame()\ndids = list(map(lambda fl: fl.split(\"/\")[-1].split(\".\")[0], fls))\npred_topics[\"id\"] = dids\npred_topics[\"topic\"] = topics\npred_topics['prob'] = probs\npred_topics = pred_topics.drop(columns={'prob'})\npred_topics = pred_topics.rename(columns={'id': 'essay_id'})\npred_topics = pred_topics.merge(topic_meta_df, on='topic', how='left')\npred_topics","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:02.249116Z","iopub.execute_input":"2022-07-09T08:58:02.249484Z","iopub.status.idle":"2022-07-09T08:58:43.322901Z","shell.execute_reply.started":"2022-07-09T08:58:02.249449Z","shell.execute_reply":"2022-07-09T08:58:43.322174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df.merge(pred_topics, on='essay_id', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:43.324313Z","iopub.execute_input":"2022-07-09T08:58:43.324552Z","iopub.status.idle":"2022-07-09T08:58:43.332537Z","shell.execute_reply.started":"2022-07-09T08:58:43.324519Z","shell.execute_reply":"2022-07-09T08:58:43.331115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:43.334220Z","iopub.execute_input":"2022-07-09T08:58:43.334619Z","iopub.status.idle":"2022-07-09T08:58:43.348993Z","shell.execute_reply.started":"2022-07-09T08:58:43.334579Z","shell.execute_reply":"2022-07-09T08:58:43.348204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare Data","metadata":{}},{"cell_type":"code","source":"labels = ['Adequate', 'Effective', 'Ineffective']\n\ntokenizer = AutoTokenizer.from_pretrained(config.model_path / 'fold_0', use_fast=True)\ntokenizer.model_max_len = config.max_len","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:43.350347Z","iopub.execute_input":"2022-07-09T08:58:43.351006Z","iopub.status.idle":"2022-07-09T08:58:44.035741Z","shell.execute_reply.started":"2022-07-09T08:58:43.350969Z","shell.execute_reply":"2022-07-09T08:58:44.034976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# From https://www.kaggle.com/competitions/feedback-prize-2021/discussion/313330\n\ndef replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n    return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n    return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\n# Register the encoding and decoding error handlers for `utf-8` and `cp1252`.\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n    \"\"\"Resolve the encoding problems and normalize the abnormal characters.\"\"\"\n    text = (\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n    )\n    text = unidecode(text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:44.037084Z","iopub.execute_input":"2022-07-09T08:58:44.037393Z","iopub.status.idle":"2022-07-09T08:58:44.045423Z","shell.execute_reply.started":"2022-07-09T08:58:44.037344Z","shell.execute_reply":"2022-07-09T08:58:44.044703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_essay(essay_ids, dataset):\n    essay_cache = {}\n\n    output = []\n    for essay_id in essay_ids:\n        essay_txt = open(config.input_path / dataset / f'{essay_id}.txt').read()\n        essay_txt = resolve_encodings_and_normalize(essay_txt).strip().lower()\n        output.append(essay_txt)\n\n    return output\n\n\ndef add_inputs(df):\n    encoded_text = df.discourse_text.apply(resolve_encodings_and_normalize)\n    df['inputs'] = (df.discourse_type + ' ' + tokenizer.sep_token + ' ' + df.topic_name + ' ' + tokenizer.sep_token + ' ' + encoded_text).str.lower()\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:44.046644Z","iopub.execute_input":"2022-07-09T08:58:44.047450Z","iopub.status.idle":"2022-07-09T08:58:44.058918Z","shell.execute_reply.started":"2022-07-09T08:58:44.047410Z","shell.execute_reply":"2022-07-09T08:58:44.058197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenizer_func(x):\n    return tokenizer(x[\"inputs\"], get_essay(x['essay_id'], 'test'), truncation=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:44.059893Z","iopub.execute_input":"2022-07-09T08:58:44.062320Z","iopub.status.idle":"2022-07-09T08:58:44.068722Z","shell.execute_reply.started":"2022-07-09T08:58:44.062289Z","shell.execute_reply":"2022-07-09T08:58:44.068055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = add_inputs(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:44.072128Z","iopub.execute_input":"2022-07-09T08:58:44.073418Z","iopub.status.idle":"2022-07-09T08:58:44.082634Z","shell.execute_reply.started":"2022-07-09T08:58:44.073373Z","shell.execute_reply":"2022-07-09T08:58:44.081925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:44.084208Z","iopub.execute_input":"2022-07-09T08:58:44.084479Z","iopub.status.idle":"2022-07-09T08:58:44.099520Z","shell.execute_reply.started":"2022-07-09T08:58:44.084443Z","shell.execute_reply":"2022-07-09T08:58:44.098831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom transformers import AutoConfig, AutoModelForSequenceClassification\nfrom transformers.models.deberta_v2.modeling_deberta_v2 import ContextPooler\nfrom transformers.models.deberta_v2.modeling_deberta_v2 import StableDropout\nfrom transformers.modeling_outputs import TokenClassifierOutput\nfrom transformers import DebertaV2ForSequenceClassification\n\ndef get_dropouts(num, start_prob, increment):\n    return [StableDropout(start_prob + (increment * i)) for i in range(num)]  \n\nclass MeanPooling(nn.Module):\n    def __init__(self):\n        super(MeanPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n        sum_mask = input_mask_expanded.sum(1)\n        sum_mask = torch.clamp(sum_mask, min=1e-9)\n        mean_embeddings = sum_embeddings / sum_mask\n        return mean_embeddings\n\nclass CustomModel(nn.Module):\n    def __init__(self, backbone):\n        super(CustomModel, self).__init__()\n        \n        self.model = backbone\n        self.config = self.model.config\n        self.num_labels = self.config.num_labels\n\n        # self.pooler = ContextPooler(self.config)\n        self.pooler = MeanPooling()\n        \n        self.classifier = nn.Linear(self.config.hidden_size, self.num_labels)\n    \n        self.dropouts = get_dropouts(num=5, start_prob=config.hidden_dropout_prob - 0.02, increment=0.01)\n    \n    def forward(\n        self,\n        input_ids=None,\n        attention_mask=None,\n        token_type_ids=None,\n        position_ids=None,\n        inputs_embeds=None,\n        labels=None,\n        output_attentions=None,\n        output_hidden_states=None,\n        return_dict=None\n    ):\n        outputs = self.model.deberta(\n            input_ids,\n            token_type_ids=token_type_ids,\n            attention_mask=attention_mask,\n            position_ids=position_ids,\n            inputs_embeds=inputs_embeds,\n            output_attentions=output_attentions,\n            output_hidden_states=output_hidden_states,\n            return_dict=return_dict,\n        )\n        \n        encoder_layer = outputs[0]\n        pooled_output = self.pooler(encoder_layer, attention_mask)\n                      \n        # Multi-sample dropout.\n        num_dps = float(len(self.dropouts))\n        for ii, drop in enumerate(self.dropouts):\n            if ii == 0:\n                logits = (self.classifier(drop(pooled_output)) / num_dps)\n            else:\n                logits += (self.classifier(drop(pooled_output)) / num_dps)\n\n        loss = None\n        if labels is not None:\n            loss_fn = nn.CrossEntropyLoss()\n            logits = logits.view(-1, self.num_labels)\n            loss = loss_fn(logits, labels.view(-1))\n\n        output = (logits,) + outputs[1:]\n\n        return TokenClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states, attentions=outputs.attentions)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:44.101001Z","iopub.execute_input":"2022-07-09T08:58:44.101601Z","iopub.status.idle":"2022-07-09T08:58:44.131792Z","shell.execute_reply.started":"2022-07-09T08:58:44.101564Z","shell.execute_reply":"2022-07-09T08:58:44.131168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    model_config = AutoConfig.from_pretrained(config.model_path / 'backbone_config/config.json')\n    model = DebertaV2ForSequenceClassification(model_config)\n    \n    return CustomModel(model)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:44.133256Z","iopub.execute_input":"2022-07-09T08:58:44.133433Z","iopub.status.idle":"2022-07-09T08:58:44.140292Z","shell.execute_reply.started":"2022-07-09T08:58:44.133410Z","shell.execute_reply":"2022-07-09T08:58:44.139577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model()","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:44.141708Z","iopub.execute_input":"2022-07-09T08:58:44.142196Z","iopub.status.idle":"2022-07-09T08:58:54.880125Z","shell.execute_reply.started":"2022-07-09T08:58:44.142160Z","shell.execute_reply":"2022-07-09T08:58:54.879199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"all_test_data = np.zeros((config.n_folds, len(test_df), len(labels)))\n\nfor fold_num in range(config.n_folds):\n    print(f'Do fold {fold_num}')\n\n    tokenizer = AutoTokenizer.from_pretrained(config.model_path / f'fold_{fold_num}', use_fast=True)\n    tokenizer.model_max_length = config.max_len\n\n    model = get_model()\n\n    state_dict = torch.load(config.model_path / f'fold_{fold_num}/pytorch_model.bin')\n    model.load_state_dict(state_dict)  \n\n    test_dataset = Dataset.from_pandas(test_df[['inputs', 'essay_id']])\n    test_tok_dataset = test_dataset.map(tokenizer_func, batched=True, remove_columns=('inputs', 'essay_id'))\n    \n    data_collator = DataCollatorWithPadding(tokenizer=tokenizer, padding='longest')\n\n    args = TrainingArguments(\n        output_dir=config.output_path,\n        learning_rate=config.lr,\n        lr_scheduler_type='cosine',\n        fp16=True,\n        evaluation_strategy='epoch',\n        per_device_train_batch_size=config.batch_size,\n        per_device_eval_batch_size=config.batch_size * 2,\n        report_to=\"none\",\n        save_strategy='no'\n    )\n    \n    trainer = Trainer(\n        model,\n        args,\n        tokenizer=tokenizer,\n        data_collator=data_collator\n    )\n    \n    outputs = trainer.predict(test_tok_dataset) \n    softmax_outputs = softmax(outputs.predictions, axis=1)\n    \n    all_test_data[fold_num] = softmax_outputs","metadata":{"execution":{"iopub.status.busy":"2022-07-09T08:58:54.881362Z","iopub.execute_input":"2022-07-09T08:58:54.881695Z","iopub.status.idle":"2022-07-09T09:01:18.772920Z","shell.execute_reply.started":"2022-07-09T08:58:54.881654Z","shell.execute_reply":"2022-07-09T09:01:18.772085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Submission","metadata":{}},{"cell_type":"code","source":"preds = np.mean(all_test_data, axis=0)\noutput_df = pd.concat([test_df[['discourse_id']], pd.DataFrame(preds, columns=labels)], axis=1)\noutput_df.to_csv('submission.csv', index=False)\npd.read_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-09T09:01:18.774441Z","iopub.execute_input":"2022-07-09T09:01:18.774707Z","iopub.status.idle":"2022-07-09T09:01:18.795090Z","shell.execute_reply.started":"2022-07-09T09:01:18.774670Z","shell.execute_reply":"2022-07-09T09:01:18.794285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}