{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# The following is necessary if you want to use the fast tokenizer for deberta v2 or v3\n# This must be done before importing transformers\nimport shutil\nfrom pathlib import Path\n\ntransformers_path = Path(\"/opt/conda/lib/python3.7/site-packages/transformers\")\n\ninput_dir = Path(\"../input/deberta-v3-tokenizer-fast\")\n\nconvert_file = input_dir / \"convert_slow_tokenizer.py\"\nconversion_path = transformers_path/convert_file.name\n\nif conversion_path.exists():\n    conversion_path.unlink()\n\nshutil.copy(convert_file, transformers_path)\ndeberta_v2_path = transformers_path / \"models\" / \"deberta_v2\"\n\nfor filename in ['tokenization_deberta_v2.py', 'tokenization_deberta_v2_fast.py']:\n    filepath = deberta_v2_path/filename\n    \n    if filepath.exists():\n        filepath.unlink()\n\n    shutil.copy(input_dir/filename, filepath)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:40.359507Z","iopub.execute_input":"2022-08-07T23:15:40.360253Z","iopub.status.idle":"2022-08-07T23:15:40.422653Z","shell.execute_reply.started":"2022-08-07T23:15:40.360160Z","shell.execute_reply":"2022-08-07T23:15:40.421586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers.convert_slow_tokenizer import SpmConverter\nfrom transformers.models.deberta_v2.tokenization_deberta_v2 import (\n        DebertaV2Tokenizer,\n    )\n\nfrom transformers.models.deberta_v2.tokenization_deberta_v2_fast import DebertaV2TokenizerFast\n\nfrom tokenizers import Regex, normalizers, processors\nclass DebertaV2Converter(SpmConverter):\n    def normalizer(self, proto):\n        list_normalizers = []\n        if self.original_tokenizer.do_lower_case:\n            list_normalizers.append(normalizers.Lowercase())\n\n        # precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap\n        # if precompiled_charsmap:\n        #     list_normalizers.append(normalizers.Precompiled(precompiled_charsmap))\n        list_normalizers.append(normalizers.Replace(Regex(\" {2,}\"), \" \"))\n\n        return normalizers.Sequence(list_normalizers)\n\n    def post_processor(self):\n        return processors.TemplateProcessing(\n            single=\"[CLS]:0 $A:0 [SEP]:0\",\n            pair=\"[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1\",\n            special_tokens=[\n                (\"[CLS]\", self.original_tokenizer.convert_tokens_to_ids(\"[CLS]\")),\n                (\"[SEP]\", self.original_tokenizer.convert_tokens_to_ids(\"[SEP]\")),\n            ],\n        )\n\n\ndef convert_deberta_v2_tokenizer(\n    tokenizer: DebertaV2Tokenizer\n) -> DebertaV2TokenizerFast:\n    tokenizer.vocab_file = tokenizer._tokenizer.vocab_file\n    return DebertaV2TokenizerFast(\n        tokenizer._tokenizer.vocab_file,\n        **tokenizer.init_kwargs,\n        tokenizer_object=DebertaV2Converter(tokenizer).converted()\n    )","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:40.425141Z","iopub.execute_input":"2022-08-07T23:15:40.425908Z","iopub.status.idle":"2022-08-07T23:15:48.487787Z","shell.execute_reply.started":"2022-08-07T23:15:40.425866Z","shell.execute_reply":"2022-08-07T23:15:48.486746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport os\nfrom transformers import AutoConfig,AutoModel\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom transformers import AutoTokenizer, AutoModelForTokenClassification, AutoConfig\nfrom transformers.models.deberta_v2.tokenization_deberta_v2_fast import DebertaV2TokenizerFast\nfrom tqdm import tqdm\nos.environ[\"CUDA_VISIBLE_DEVICES\"]='0'\ndevice='cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:48.489628Z","iopub.execute_input":"2022-08-07T23:15:48.490343Z","iopub.status.idle":"2022-08-07T23:15:48.585089Z","shell.execute_reply.started":"2022-08-07T23:15:48.490288Z","shell.execute_reply":"2022-08-07T23:15:48.584040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Experiment:\n    def __init__(self,DOWNLOADED_MODEL_PATH,\n                 TRAINED_MODEL_PATH,\n                 XGB_PATH,\n                 FOLDS,\n                 hidden_state_dimension,\n                 BATCH_SIZE,\n                 NUM_WORKERS,\n                 MAX_LEN,\n                 WINDOW_SIZE):\n        self.DOWNLOADED_MODEL_PATH=DOWNLOADED_MODEL_PATH\n        self.TRAINED_MODEL_PATH=TRAINED_MODEL_PATH\n        self.XGB_PATH=XGB_PATH\n        self.FOLDS=FOLDS\n        self.hidden_state_dimension=hidden_state_dimension\n        self.BATCH_SIZE=BATCH_SIZE\n        self.NUM_WORKERS=NUM_WORKERS\n        self.MAX_LEN=MAX_LEN\n        self.WINDOW_SIZE=WINDOW_SIZE","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:48.589085Z","iopub.execute_input":"2022-08-07T23:15:48.589435Z","iopub.status.idle":"2022-08-07T23:15:48.598745Z","shell.execute_reply.started":"2022-08-07T23:15:48.589405Z","shell.execute_reply":"2022-08-07T23:15:48.597521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiments=[]\n\n\n\n\n\nexperiments.append(Experiment(DOWNLOADED_MODEL_PATH=\"../input/deberta-v3-large\",\n                              BATCH_SIZE=4,\n                              TRAINED_MODEL_PATH=\"../input/fb-test6-deberta-v3-large-pl-nb\",\n                              XGB_PATH='../input/fbtest6debertav3largeplnbxgb',\n                              FOLDS=np.arange(6),\n                              hidden_state_dimension=1024,\n                              NUM_WORKERS=2,\n                              MAX_LEN=2048,\n                              WINDOW_SIZE=2048))\n\n\n\nexperiments.append(Experiment(DOWNLOADED_MODEL_PATH=\"../input/deberta-v2-xlarge\",\n                              BATCH_SIZE=4,\n                              TRAINED_MODEL_PATH=\"../input/fb-test6-deberta-v2-xlarge-pl\",\n                              XGB_PATH='../input/fbtest6debertav2xlargeplxgb',\n                              FOLDS=np.arange(6),\n                              hidden_state_dimension=1536,\n                              NUM_WORKERS=2,\n                              MAX_LEN=2048,\n                              WINDOW_SIZE=2048))\n\n\nexperiments.append(Experiment(DOWNLOADED_MODEL_PATH=\"../input/deberta-xlarge\",\n                              BATCH_SIZE=4,\n                              TRAINED_MODEL_PATH=\"../input/fb-test7-deberta-x-large-pl-2-\",\n                              XGB_PATH='../input/fbtest7debertaxlargepl2xgb',\n                              FOLDS=np.arange(6),\n                              hidden_state_dimension=1024,\n                              NUM_WORKERS=2,\n                              MAX_LEN=1280,\n                              WINDOW_SIZE=1280))\n\n\nexperiments.append(Experiment(DOWNLOADED_MODEL_PATH=\"../input/pytorch-longformer-large\",\n                              BATCH_SIZE=4,\n                              TRAINED_MODEL_PATH=\"../input/fb-test6-longformer-large-pl-nb\",\n                              XGB_PATH='../input/fbtest6longformerlargeplxgb',\n                              FOLDS=np.arange(6),\n                              hidden_state_dimension=1024,\n                              NUM_WORKERS=2,\n                              MAX_LEN=2048,\n                              WINDOW_SIZE=2048))\n","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:48.602461Z","iopub.execute_input":"2022-08-07T23:15:48.603480Z","iopub.status.idle":"2022-08-07T23:15:48.612461Z","shell.execute_reply.started":"2022-08-07T23:15:48.603449Z","shell.execute_reply":"2022-08-07T23:15:48.611258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Data","metadata":{}},{"cell_type":"code","source":"test=pd.read_csv(\"../input/feedback-prize-effectiveness/test.csv\")\nfull_texts={}\n\nfor essay_id in test['essay_id']:\n    with open(os.path.join(\"../input/feedback-prize-effectiveness/test\",essay_id+'.txt'),'r') as f:\n        text=f.read()\n\n    full_texts[essay_id]=text\n    \ntest['label']=-1    ","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:48.615202Z","iopub.execute_input":"2022-08-07T23:15:48.616810Z","iopub.status.idle":"2022-08-07T23:15:48.650546Z","shell.execute_reply.started":"2022-08-07T23:15:48.616765Z","shell.execute_reply":"2022-08-07T23:15:48.649581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_substring_span(text, substring, min_length=10, fraction=0.999):\n    \"\"\"\n    Returns substring's span from the given text with the certain precision.\n    \"\"\"\n\n    position = text.find(substring)\n    substring_length = len(substring)\n    if position == -1:\n        half_length = int(substring_length * fraction)\n        half_substring = substring[:half_length]\n        half_substring_length = len(half_substring)\n        if half_substring_length < min_length:\n            return [-1, 0]\n        else:\n            return get_substring_span(text=text,\n                                    substring=half_substring,\n                                    min_length=min_length,\n                                    fraction=fraction)\n\n    span = [position, position+substring_length]\n    return span\n\ndiscourse_mapping={'Lead': 0, 'Position': 1, 'Claim': 2, 'Evidence': 3, 'Counterclaim': 4, 'Rebuttal': 5, 'Concluding Statement': 6}\n\nclass FeedbackDataset(Dataset):\n    def __init__(self, tokenizer, df, full_texts, train, aug=False, loss_type=\"BCELoss\", max_len=512):\n        self.tokenizer = tokenizer\n        self.texts = df['discourse_text'].values\n        self.labels = df['label'].values\n        self.discourse_type=df['discourse_type'].values\n        self.essay_ids=df['essay_id'].values\n        self.full_texts=full_texts\n        self.max_len = max_len\n        self.aug = aug\n        #self.nlp_aug=naw.SynonymAug()\n        self.train=train\n\n        self.encodings=[]\n        self.labels=[]\n        self.gather_indices=[]\n        self.discourse_ids=[]\n        self.discourse_type_ids=[]\n        self.input_id_lengths=[]\n        for key in tqdm(df['essay_id'].unique()):\n            discourses=df[df['essay_id']==key]\n            text=full_texts[key]\n            reference_text=text[:]\n\n            for discourse_text,label,id,type in zip(discourses['discourse_text'],discourses['label'],discourses['discourse_id'],discourses['discourse_type']):\n                span=get_substring_span(reference_text, discourse_text.strip())\n                text=text[:span[0]]+f\"({type} start)\"+discourse_text.strip()+f\"({type} end)\"+text[span[1]:]\n                reference_text=reference_text[:span[0]]+f\"({type} start)\"+\"*\"*(span[1]-span[0])+f\"({type} end)\"+reference_text[span[1]:]\n\n\n                #reference_text[:span[0]]+\"*\"*(span[1]-span[0])+text[span[1]:]\n\n            encoding = self.tokenizer(text,\n                                   add_special_tokens=True,\n                                   max_length=self.max_len,\n                                   padding=False,\n                                   return_offsets_mapping=True,\n                                   truncation=True)\n            gather_indices=np.ones(len(encoding['input_ids']))*-1\n            discourse_type_ids=np.zeros(len(encoding['input_ids']))\n            cnt=0\n            sample_labels=[]\n            discourse_ids=[]\n\n\n\n            for discourse_text,label,id,type in zip(discourses['discourse_text'],discourses['label'],discourses['discourse_id'],discourses['discourse_type']):\n                span=get_substring_span(text, discourse_text.strip())\n                n_tokens=0\n                # print(encoding['offset_mapping'])\n                # exit()\n                for i in range(len(gather_indices)):\n                    if encoding['offset_mapping'][i]!=(0,0) and encoding['offset_mapping'][i][0]>=span[0] and encoding['offset_mapping'][i][1]<=span[1]:\n                        gather_indices[i]=cnt\n                        discourse_type_ids[i]=discourse_mapping[type]\n                        n_tokens+=1\n                text=text[:span[0]]+\"*\"*(span[1]-span[0])+text[span[1]:]\n                # if (gather_indices==3).sum()==0:\n                #     print(gather_indices)\n                if (gather_indices==cnt).sum()>0:\n                    sample_labels.append(label)\n                    discourse_ids.append(id)\n                    cnt+=1\n                # else:\n                #     print(cnt)\n                #     print(discourse_text)\n\n            # for cnt in range(int(gather_indices.max())+1):\n            #     if (gather_indices==cnt).sum()==0:\n            #         print(gather_indices)\n            #         print(cnt)\n            #         print(len(sample_labels))\n            #         print(discourses)\n            #         gather_indices=np.ones(len(encoding['input_ids']))*-1\n            #         cnt=0\n            #         sample_labels=[]\n            #         for discourse_text,label in zip(discourses['discourse_text'],discourses['label']):\n            #             span=get_substring_span(text, discourse_text.strip())\n            #             n_tokens=0\n            #             # print(encoding['offset_mapping'])\n            #             # exit()\n            #             for i in range(len(gather_indices)):\n            #                 if encoding['offset_mapping'][i]!=(0,0) and encoding['offset_mapping'][i][0]>=span[0] and encoding['offset_mapping'][i][1]<=span[1]:\n            #                     gather_indices[i]=cnt\n            #                     n_tokens+=1\n            #             # if (gather_indices==3).sum()==0:\n            #             print(gather_indices)\n            #             if (gather_indices==cnt).sum()>0:\n            #                 sample_labels.append(label)\n            #                 cnt+=1\n            #\n            #         exit()\n\n\n            self.encodings.append(encoding)\n            self.labels.append(sample_labels)\n            self.gather_indices.append(gather_indices)\n            self.discourse_ids.append(discourse_ids)\n            self.discourse_type_ids.append(discourse_type_ids)\n            self.input_id_lengths.append(len(encoding['input_ids']))\n        sorted_indices=np.argsort(self.input_id_lengths)\n        self.encodings=[self.encodings[i] for i in sorted_indices]\n        self.labels=[self.labels[i] for i in sorted_indices]\n        self.gather_indices=[self.gather_indices[i] for i in sorted_indices]\n        self.discourse_ids=[self.discourse_ids[i] for i in sorted_indices]\n        self.discourse_type_ids=[self.discourse_type_ids[i] for i in sorted_indices]\n        self.input_id_lengths=[self.input_id_lengths[i] for i in sorted_indices]\n        \n            # print(gather_indices)\n            # print(sample_labels)\n            #\n            # print(key)\n            # print(discourses)\n        #exit()\n\n\n\n\n        # self.anchors = df['anchor'].values\n        # self.targets = df['target'].values\n        # self.contexts = df['context'].values\n        # if loss_type=='BCELoss':\n        #     self.labels = df['score'].values\n        # elif loss_type=='CrossEntropyLoss':\n        #     self.labels = df['score_map'].values\n        # elif loss_type=='OrdinalLoss':\n        #     self.labels=[]\n        #     for label in df['score_map'].values:\n        #         temp=np.zeros(4)\n        #         temp[:label]=1\n        #         self.labels.append(temp)\n            #self.labels = df['score_map'].values\n\n\n        #self.level=level\n\n\n    def __len__(self):\n        return len(self.labels)\n\n\n            # for text in\n\n    def __getitem__(self, idx):\n\n        # text=self.discourse_type[idx].lower()+'[SEP]'+self.texts[idx]\n        #\n        #\n        # encoding = self.tokenizer(text,\n        #                        self.full_texts[self.essay_ids[idx]],\n        #                        add_special_tokens=True,\n        #                        max_length=self.max_len,\n        #                        padding=False,\n        #                        return_offsets_mapping=True,\n        #                        truncation=True)\n        encoding=self.encodings[idx]\n        encoding['wids']=np.array(encoding.word_ids())\n        encoding['wids'][encoding['wids']==None]=-1\n        encoding['wids']=encoding['wids'].astype('int')\n        #encoding.sequence_ids()\n        label = self.labels[idx]\n        sequence_ids=np.array(encoding.sequence_ids())\n        sequence_ids[sequence_ids==None]=-1\n        # print(sequence_ids)\n        # exit()\n\n        data={k:torch.tensor(v, dtype=torch.long) for k,v in encoding.items()}\n        data['labels']=torch.tensor(label, dtype=torch.float)\n        data['sequence_ids']=torch.tensor(sequence_ids.astype(\"int\"))\n        data['gather_indices']=torch.tensor(self.gather_indices[idx])\n        data['discourse_ids']=self.discourse_ids[idx]\n        data['discourse_type_ids']=torch.tensor(self.discourse_type_ids[idx])\n        return data","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:48.653937Z","iopub.execute_input":"2022-08-07T23:15:48.654248Z","iopub.status.idle":"2022-08-07T23:15:48.687604Z","shell.execute_reply.started":"2022-08-07T23:15:48.654221Z","shell.execute_reply":"2022-08-07T23:15:48.686380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomCollate:\n    def __init__(self,tokenizer,train=True,sliding_window=None):\n        self.tokenizer=tokenizer\n        self.train=train\n        self.sliding_window=sliding_window\n\n    def __call__(self,data):\n        \"\"\"\n        need to collate: input_ids, attention_mask, labels\n        input_ids is padded with 1, attention_mask 0, labels -100\n\n        \"\"\"\n\n\n        bs=len(data)\n        # print(data[0])\n        # exit()\n        lengths=[]\n        for i in range(bs):\n            lengths.append(len(data[i]['input_ids']))\n        max_len=max(lengths)\n        if self.sliding_window is not None and max_len > self.sliding_window:\n            max_len= int((np.floor(max_len/self.sliding_window-1e-6)+1)*self.sliding_window)\n\n        input_ids, attention_mask, labels, BIO_labels, discourse_labels=[],[],[],[],[]\n        sequence_ids=[]\n        gather_indices=[]\n        wids=[]\n        discourse_ids=[]\n        discourse_type_ids=[]\n        for i in range(bs):\n            input_ids.append(torch.nn.functional.pad(data[i]['input_ids'],(0,max_len-lengths[i]),value=self.tokenizer.pad_token_id))\n            attention_mask.append(torch.nn.functional.pad(data[i]['attention_mask'],(0,max_len-lengths[i]),value=0))\n            labels.append(data[i]['labels'])\n            sequence_ids.append(torch.nn.functional.pad(data[i]['sequence_ids'],(0,max_len-lengths[i]),value=-1))\n            gather_indices.append(torch.nn.functional.pad(data[i]['gather_indices'],(0,max_len-lengths[i]),value=-1))\n            discourse_type_ids.append(torch.nn.functional.pad(data[i]['discourse_type_ids'],(0,max_len-lengths[i]),value=0))\n            discourse_ids=discourse_ids+data[i]['discourse_ids']\n            #wids.append(torch.nn.functional.pad(data[i]['wids'],(0,max_len-lengths[i]),value=-1))\n        input_ids=torch.stack(input_ids)\n        attention_mask=torch.stack(attention_mask)\n        labels=torch.cat(labels)\n        sequence_ids=torch.stack(sequence_ids)\n        gather_indices=torch.stack(gather_indices)\n        discourse_type_ids=torch.stack(discourse_type_ids)\n        #wids=torch.stack(wids)\n\n        #offsets=[encoding[\"offset_mapping\"] for encoding in data]\n        offsets=[]\n        # print(len(offsets[0]))\n        # exit()\n\n        return {\"input_ids\":input_ids,\"attention_mask\":attention_mask,\n        \"labels\":labels,\"sequence_ids\":sequence_ids,\"wids\":wids,\"offsets\":offsets,\n        \"sample_id\":np.arange(len(input_ids)),\"gather_indices\":gather_indices,\"discourse_ids\":discourse_ids,\n        \"discourse_type_ids\":discourse_type_ids}\n","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:48.691405Z","iopub.execute_input":"2022-08-07T23:15:48.692134Z","iopub.status.idle":"2022-08-07T23:15:48.707397Z","shell.execute_reply.started":"2022-08-07T23:15:48.692101Z","shell.execute_reply":"2022-08-07T23:15:48.706140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Network","metadata":{}},{"cell_type":"code","source":"class ResidualLSTM(nn.Module):\n\n    def __init__(self, d_model, rnn='GRU'):\n        super(ResidualLSTM, self).__init__()\n        self.downsample=nn.Linear(d_model,d_model//2)\n        if rnn=='GRU':\n            self.LSTM=nn.GRU(d_model//2, d_model//2, num_layers=2, bidirectional=False, dropout=0.2)\n        else:\n            self.LSTM=nn.LSTM(d_model//2, d_model//2, num_layers=2, bidirectional=False, dropout=0.2)\n        self.dropout1=nn.Dropout(0.2)\n        self.norm1= nn.LayerNorm(d_model//2)\n        self.linear1=nn.Linear(d_model//2, d_model)\n        self.linear2=nn.Linear(d_model*4, d_model)\n        self.dropout2=nn.Dropout(0.2)\n        self.norm2= nn.LayerNorm(d_model)\n\n    def forward(self, x):\n        x=x.permute(1,0,2)\n        res=x\n        x=self.downsample(x)\n        x, _ = self.LSTM(x)\n        x = self.linear1(x)\n        # x=self.dropout1(x)\n        # x=self.norm1(x)\n        # x=F.relu(self.linear1(x))\n        # x=self.linear2(x)\n        # x=self.dropout2(x)\n        x=res+x\n        x=x.permute(1,0,2)\n        return self.norm2(x)\n\nclass SlidingWindowTransformerModel(nn.Module):\n    def __init__(self,DOWNLOADED_MODEL_PATH, hidden_state_dimension, nclass, rnn='GRU', window_size=512, edge_len=64, no_backbone=False):\n        super(SlidingWindowTransformerModel, self).__init__()\n        config_model = AutoConfig.from_pretrained(DOWNLOADED_MODEL_PATH+'/config.json')\n        self.no_backbone=no_backbone\n        if no_backbone:\n            pass\n        else:\n            self.backbone=AutoModel.from_pretrained(\n                               DOWNLOADED_MODEL_PATH+'/pytorch_model.bin',config=config_model)\n\n        if rnn==\"GRU\" or rnn=='LSTM':\n            self.lstm=ResidualLSTM(hidden_state_dimension,rnn)\n        else:\n            self.lstm=ResNet()\n        self.classification_head=nn.Linear(hidden_state_dimension,nclass)\n        self.window_size=window_size\n        self.edge_len=edge_len\n        self.inner_len=window_size-edge_len*2\n\n        self.discourse_embedding=nn.Embedding(8,256,padding_idx=0)\n        self.downsample=nn.Linear(hidden_state_dimension+256,hidden_state_dimension)\n\n    def forward(self,input_ids,attention_mask,sequence_ids,discourse_type_ids,gather_indices,return_vectors=False,return_transformer_hidden_states=False):\n\n\n\n        # print(L)\n        # exit()\n        #x=self.backbone(input_ids=input_ids,attention_mask=attention_mask,return_dict=False)[0]\n        #x=self.backbone.embeddings(input_ids)#+0.1*self.discourse_embedding(discourse_type_ids)\n        discourse_type_ids=self.discourse_embedding(discourse_type_ids)\n        x=input_ids\n        # x=torch.cat([x,discourse_type_ids],-1)\n        # x=self.downsample(x)\n\n        #x=torch.cat([x,])\n\n        if self.no_backbone==False:\n            B,L=input_ids.shape\n            if L<=self.window_size:\n                x=self.backbone(x,attention_mask=attention_mask,return_dict=False)[0]\n                #pass\n            else:\n                #print(\"####\")\n                #print(input_ids.shape)\n                segments=(L-self.window_size)//self.inner_len\n                if (L-self.window_size)%self.inner_len>self.edge_len:\n                    segments+=1\n                elif segments==0:\n                    segments+=1\n                x_new=self.backbone(x[:,:self.window_size],attention_mask=attention_mask[:,:self.window_size],return_dict=False)[0]\n                # print(x_new.shape)\n                # exit()\n\n                for i in range(1,segments+1):\n                    start=self.window_size-self.edge_len+(i-1)*self.inner_len\n                    end=self.window_size-self.edge_len+(i-1)*self.inner_len+self.window_size\n                    end=min(end,L)\n                    x_next=x[:,start:end]\n                    mask_next=attention_mask[:,start:end]\n                    x_next=self.backbone(x_next,attention_mask=mask_next,return_dict=False)[0]\n                    #L_next=x_next.shape[1]-self.edge_len,\n                    if i==segments:\n                        x_next=x_next[:,self.edge_len:]\n                    else:\n                        x_next=x_next[:,self.edge_len:self.edge_len+self.inner_len]\n                    #print(x_next.shape)\n                    x_new=torch.cat([x_new,x_next],1)\n                x=x_new\n                #print(start,end)\n        #print(x.shape)\n            if return_transformer_hidden_states:\n                transformer_hidden_states=x\n\n            # print(x.shape)\n            # exit()\n\n            # x=torch.cat([x,discourse_type_ids],-1)\n            # x=self.downsample(x)\n\n            #x=self.lstm(x)\n\n            #x=self.classification_head(x).squeeze(-1)\n\n            pooled_outputs=[]\n            if return_vectors:\n                vectors=[]\n            for i in range(len(x)):\n                #n_discourses=gather_indices[i].max()+1\n                # unique_gather_indices=torch.unique_consecutive(gather_indices[i])\n                # unique_gather_indices=unique_gather_indices[unique_gather_indices!=-1]\n                #\n                # #print(unique_gather_indices)\n                #\n                # for j in unique_gather_indices:\n                n_discourses=gather_indices[i].max()+1\n                tmp=[]\n                for j in range(n_discourses):\n\n\n                    vector=x[i][gather_indices[i]==j]\n                    if return_vectors:\n                        vectors.append(self.classification_head(vector))\n                    mean_vector=vector.mean(0)\n                    #max_vector,_=vector.max(0)\n                    # print(max_vector)\n                    # exit()\n                    #pooled=torch.cat([mean_vector,max_vector],-1)\n                    #pooled=mean_vector\n                    tmp.append(mean_vector)\n                    #pooled_outputs.append(pooled)\n                tmp=torch.stack(tmp)\n                tmp=self.lstm(tmp.unsqueeze(0))\n                pooled_outputs.append(tmp.squeeze(0))\n\n\n            #exit()\n            pooled_outputs=torch.cat(pooled_outputs)\n            x=pooled_outputs\n            x=self.classification_head(x).squeeze(-1)\n\n\n        else:\n            transformer_hidden_states=input_ids\n            x=self.lstm(transformer_hidden_states)\n            x=self.classification_head(x)\n\n        if return_vectors:\n            return x,vectors\n        else:\n            return x\n","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:48.709375Z","iopub.execute_input":"2022-08-07T23:15:48.710001Z","iopub.status.idle":"2022-08-07T23:15:48.738607Z","shell.execute_reply.started":"2022-08-07T23:15:48.709957Z","shell.execute_reply":"2022-08-07T23:15:48.737395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def sorted_quantile(array, q):\n    array = np.array(array)\n    n = len(array)\n    index = (n - 1) * q\n    left = np.floor(index).astype(int)\n    fraction = index - left\n    right = left\n    right = right + (fraction > 0).astype(int)\n    i, j = array[left], array[right]\n    return i + (j - i) * fraction\n\nfrom scipy.stats import entropy\n#make features\ndef get_xgb_features(train_df,prob_sequences):\n    features2calculate=[f\"instability_{i}\" for i in range(4)]+\\\n    [f\"begin_{i}\" for i in range(3)]+\\\n    [f\"end_{i}\" for i in range(3)]#+\\\n    #[\"entropy\"]\n\n    calculated_features=[]\n    for i,prob_seq in tqdm(enumerate(prob_sequences)):\n\n        tmp=[]\n        #quants = np.linspace(0,1,n_quan)\n        prob_seq=np.array(prob_seq)\n        instability = []\n        #all_quants=[] \n        tmp.append(np.diff(prob_seq[:,:],0).mean(0))\n        tmp.append([(np.diff(prob_seq[:,[1,2]].sum(1))**2).mean()])\n\n        tmp.append(prob_seq[:5,:].mean(0))\n        tmp.append(prob_seq[-5:,:].mean(0))\n\n        calculated_features.append(np.concatenate(tmp))\n\n\n    train_df[features2calculate]=calculated_features\n    train_df['len']=[len(s) for s in prob_sequences]\n\n    calculated_features=np.array(calculated_features)\n    calculated_features.shape\n\n    p_features=[]\n    n_features=[]\n    neighbor_features=['Ineffective','Adequate','Effective','discourse_type']\n    neighbor_features_values=train_df[neighbor_features].values\n    for i in tqdm(range(len(train_df))):\n        if i>1 and train_df['essay_id'].iloc[i]==train_df['essay_id'].iloc[i-1]:\n            p_features.append(neighbor_features_values[i-1])\n        else:\n            p_features.append(neighbor_features_values[i])\n\n        if i<(len(train_df)-1) and train_df['essay_id'].iloc[i]==train_df['essay_id'].iloc[i+1]:\n            n_features.append(neighbor_features_values[i+1])\n        else:\n            n_features.append(neighbor_features_values[i])\n\n    train_df[[f+\"_previous\" for f in neighbor_features]]=p_features\n    train_df[[f+\"_next\" for f in neighbor_features]]=n_features\n\n    train_df['mean_Ineffective']=train_df.groupby(\"essay_id\")[\"Ineffective\"].transform(\"mean\")\n    train_df['mean_Adequate']=train_df.groupby(\"essay_id\")[\"Adequate\"].transform(\"mean\")\n    train_df['mean_Effective']=train_df.groupby(\"essay_id\")[\"Effective\"].transform(\"mean\")\n\n    train_df['std_Ineffective']=train_df.groupby(\"essay_id\")[\"Ineffective\"].transform(\"std\")\n    train_df['std_Adequate']=train_df.groupby(\"essay_id\")[\"Adequate\"].transform(\"std\")\n    train_df['std_Effective']=train_df.groupby(\"essay_id\")[\"Effective\"].transform(\"std\")\n\n    train_df['discourse_count']=train_df.groupby(\"essay_id\")['discourse_type'].transform(\"count\")\n\n    cnts=train_df.groupby('essay_id')['discourse_type'].apply(lambda x: x.value_counts())\n\n    #new_df=[]\n    discourse_types=['Claim','Evidence','Concluding Statement','Lead','Position','Counterclaim','Rebuttal']\n    value_count_hash={}\n    for t in discourse_types:\n        value_count_hash[t]={}\n    for key in cnts.keys():\n        value_count_hash[key[1]][key[0]]=cnts[key]\n\n    discourse_cnts=[]    \n    for essay_id in train_df['essay_id'].unique():\n        row=[essay_id]\n        for d in discourse_types:\n            try:\n                row.append(value_count_hash[d][essay_id])\n            except:\n                row.append(0)\n        discourse_cnts.append(row)\n\n    discourse_cnts=pd.DataFrame(discourse_cnts,columns=['essay_id']+[f'{d}_count' for d in discourse_types])    \n    #discourse_cnts\n\n    train_df=train_df.merge(discourse_cnts,how='left',on='essay_id')\n    train_df\n\n    #train_df\n\n    return train_df","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:48.743056Z","iopub.execute_input":"2022-08-07T23:15:48.743540Z","iopub.status.idle":"2022-08-07T23:15:48.975029Z","shell.execute_reply.started":"2022-08-07T23:15:48.743509Z","shell.execute_reply":"2022-08-07T23:15:48.973912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neighbor_features=['Ineffective','Adequate','Effective','discourse_type']\ndiscourse_types=['Claim','Evidence','Concluding Statement','Lead','Position','Counterclaim','Rebuttal']\n\nfeatures=[\"Ineffective\",\"Adequate\",\"Effective\",\n          \"instability_0\",\"instability_1\",\"instability_2\",\"instability_3\",\n          \"len\",\"discourse_type\"]\nfeatures+=[f\"begin_{i}\" for i in range(3)]\nfeatures+=[f\"end_{i}\" for i in range(3)]\n\nfeatures=features+[f+\"_previous\" for f in neighbor_features]+[f+\"_next\" for f in neighbor_features]+\\\n['mean_Ineffective','mean_Adequate','mean_Effective']+['std_Ineffective','std_Adequate','std_Effective']+\\\n['discourse_count']+[f'{d}_count' for d in discourse_types]","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:48.976855Z","iopub.execute_input":"2022-08-07T23:15:48.977286Z","iopub.status.idle":"2022-08-07T23:15:48.985658Z","shell.execute_reply.started":"2022-08-07T23:15:48.977240Z","shell.execute_reply":"2022-08-07T23:15:48.984434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from cuml import ForestInference\nimport xgboost as xgb\nimport pickle\nlabel_mapping={'Ineffective': 0, 'Adequate': 1, 'Effective': 2}\n#xgb_preds = []\n\nsubs=[]\nfor exp in experiments:\n    test_params = {'batch_size': exp.BATCH_SIZE,\n                'shuffle': False,\n                'num_workers': exp.NUM_WORKERS,\n                'pin_memory':True\n                }\n\n    if \"v2\" in exp.DOWNLOADED_MODEL_PATH or \"v3\" in exp.DOWNLOADED_MODEL_PATH:\n        tokenizer = AutoTokenizer.from_pretrained(exp.DOWNLOADED_MODEL_PATH)\n        tokenizer = convert_deberta_v2_tokenizer(tokenizer)\n    else:\n        tokenizer = AutoTokenizer.from_pretrained(exp.DOWNLOADED_MODEL_PATH)\n    test_dataset = FeedbackDataset(tokenizer, test, full_texts, False, 0, ' ', exp.MAX_LEN)\n    test_loader = DataLoader(test_dataset, **test_params, collate_fn=CustomCollate(tokenizer))\n    model = SlidingWindowTransformerModel(exp.DOWNLOADED_MODEL_PATH,\n                                      hidden_state_dimension=exp.hidden_state_dimension,\n                                      window_size=exp.WINDOW_SIZE,\n                                      nclass=3)\n    model.to(device);\n    preds=[]\n    discourse_ids=[]\n    for fold in exp.FOLDS:\n        model.load_state_dict(torch.load(os.path.join(exp.TRAINED_MODEL_PATH,f\"fold{fold}.pt\")))\n        model.eval()\n        tmp=[]\n        tmp_vectors=[]\n        for batch in tqdm(test_loader):\n            ids = batch['input_ids'].to(device, dtype = torch.long)\n            mask = batch['attention_mask'].to(device, dtype = torch.long)\n            sequence_ids = batch['sequence_ids'].to(device, dtype = torch.long)\n            sample_id=batch['sample_id']\n            gather_indices = batch['gather_indices'].to(device, dtype = torch.long)\n            discourse_type_ids = batch['discourse_type_ids'].to(device, dtype = torch.long)\n            if fold==0:\n                discourse_ids=discourse_ids+batch['discourse_ids']\n            max_sample_id=sample_id.max()\n            with torch.no_grad():\n                output,vectors = model(ids,mask,sequence_ids,discourse_type_ids,gather_indices,return_vectors=True)\n                vectors=[torch.nn.functional.softmax(v,-1).cpu().numpy() for v in vectors]\n                tmp_vectors+=vectors\n                output=torch.nn.functional.softmax(output,-1)\n\n                \n\n\n            tmp.append(output.cpu())\n        tmp=torch.cat(tmp)\n        preds.append(tmp)\n        \n        if fold==0:\n            prob_sequences=[tmp_vectors]\n        else:\n    #         for i in range(len(prob_sequences)):\n    #             prob_sequences[i]=prob_sequences[i]+tmp_vectors[i]\n            prob_sequences.append(tmp_vectors)            \n        \n        \n    preds=torch.stack(preds).numpy()\n    \n    \n    xgb_preds=[]\n    for fold in exp.FOLDS:\n        sub=pd.DataFrame(columns=['discourse_id']+list(label_mapping.keys()))\n        sub['discourse_id']=discourse_ids\n        sub[list(label_mapping.keys())]=preds[fold]\n        sub=sub.merge(test[['discourse_id','discourse_type','essay_id']],how='left',on='discourse_id')\n\n        sub=get_xgb_features(sub,prob_sequences[fold])\n\n        for f in features:\n            if f not in ['discourse_type_previous','discourse_type_next','discourse_type']:\n                sub[f]= sub[f].astype('float')\n            else:    \n                sub[f]= sub[f].astype('category')\n\n        d_test = xgb.DMatrix(sub[features],enable_categorical=True)\n        for xgb_fold in exp.FOLDS:\n            xgb_model_loaded = pickle.load(open(f\"{exp.XGB_PATH}/xgb_{xgb_fold}.p\", \"rb\"))\n\n            xgb_preds.append(xgb_model_loaded.predict(d_test))\n\n    xgb_preds=np.stack(xgb_preds)\n    xgb_preds.shape\n    xgb_preds=xgb_preds.mean(0)        \n    \n    submission=pd.read_csv(\"../input/feedback-prize-effectiveness/sample_submission.csv\")\n\n    discourse_ids=list(submission['discourse_id'])\n    \n    submission[\"Ineffective\"]=1e-9\n    submission[\"Adequate\"]=1e-9\n    submission[\"Effective\"]=1e-9\n    \n    for i in range(len(sub)):\n        index=discourse_ids.index(sub['discourse_id'].iloc[i])\n        submission[\"Ineffective\"].iloc[index]=xgb_preds[i,0]\n        submission[\"Adequate\"].iloc[index]=xgb_preds[i,1]\n        submission[\"Effective\"].iloc[index]=xgb_preds[i,2]\n\n    submission.to_csv(\"submission.csv\",index=False)\n    subs.append(submission)\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:15:48.987829Z","iopub.execute_input":"2022-08-07T23:15:48.988797Z","iopub.status.idle":"2022-08-07T23:21:43.336100Z","shell.execute_reply.started":"2022-08-07T23:15:48.988739Z","shell.execute_reply":"2022-08-07T23:21:43.334371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subs[0]","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:21:43.338846Z","iopub.execute_input":"2022-08-07T23:21:43.339343Z","iopub.status.idle":"2022-08-07T23:21:43.364061Z","shell.execute_reply.started":"2022-08-07T23:21:43.339295Z","shell.execute_reply":"2022-08-07T23:21:43.362819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#weights=[0.33918623, 0.43399894, 0.22681483]\n\n#weights=[0.33988562, 0.48792195, 0.17219243]\n\n#weights=[0.29195689, 0.36194071, 0.16637285, 0.17972955]\n\nweights=[0.27095668, 0.36492328, 0.19755267, 0.16656738]\n\n\nassert len(subs)==len(weights)\n\nweights=np.array(weights)\nweights=weights/weights.sum()\nsubmission=subs[0].copy()\nsubmission[\"Ineffective\"]=submission[\"Ineffective\"].values*weights[0]\nsubmission[\"Adequate\"]=submission[\"Adequate\"].values*weights[0]\nsubmission[\"Effective\"]=submission[\"Effective\"].values*weights[0]\n\n\nfor sub,weight in zip(subs[1:],weights[1:]):\n    submission[\"Ineffective\"]=submission[\"Ineffective\"].values+sub[\"Ineffective\"].values*weight\n    submission[\"Adequate\"]=submission[\"Adequate\"].values+sub[\"Adequate\"].values*weight\n    submission[\"Effective\"]=submission[\"Effective\"].values+sub[\"Effective\"].values*weight\n\nsubmission.to_csv(\"submission.csv\",index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:25:03.718597Z","iopub.execute_input":"2022-08-07T23:25:03.719215Z","iopub.status.idle":"2022-08-07T23:25:03.749806Z","shell.execute_reply.started":"2022-08-07T23:25:03.719157Z","shell.execute_reply":"2022-08-07T23:25:03.748522Z"},"trusted":true},"execution_count":null,"outputs":[]}]}