{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"*Hi Guys,<br> This is the train notebook. Few notebooks that contributed a lot to my work are:-<br> https://www.kaggle.com/code/dlaststark/fpe-no-fancy-stuff and https://www.kaggle.com/code/lextoumbourou/feedback-prize-the-complete-overview.<br> For any suggestions please reach out to me.*","metadata":{}},{"cell_type":"markdown","source":"Things that are there in this notebook: <br>\n* Max_length = 512\n* Model = Deberta Base\n* Use of 2021 Dataset\n* Topics name were added\n<br><br>\nThings to try: <br><br>\n* Higher max_length\n* DeBERTa Large models\n* Better use of 2021 dataset for pseudo labelling\n* Ensembling ","metadata":{}},{"cell_type":"markdown","source":"## Importing Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport re\nimport string\nfrom types import SimpleNamespace\nimport logging\nfrom pathlib import Path\nimport math\nfrom datetime import datetime\nfrom sklearn.model_selection import StratifiedKFold,GroupKFold,StratifiedGroupKFold\nfrom sklearn.metrics import log_loss\nimport torch\nfrom torch.utils.data import DataLoader,Dataset\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer, AutoConfig\nfrom transformers import TrainingArguments, Trainer , DataCollatorWithPadding\nfrom datasets import Dataset, load_metric\nimport random\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom scipy.special import softmax\nimport wandb\ntqdm.pandas()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run_type = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '')\nif run_type == 'Interactive':\n    print('Wandb in offline mode.')\n    os.environ['WANDB_MODE'] = 'offline'\nprint('Authenticating with wandb.')\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nwandb_creds = user_secrets.get_secret(\"wandb\")\n\n!wandb login {wandb_creds}    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Input","metadata":{}},{"cell_type":"code","source":"INPUT_DIR =\"../input/feedback-prize-effectiveness/\"\nOUTPUT_DIR=\"./\"\ntrain=pd.read_csv(INPUT_DIR+\"train.csv\")\ntest=pd.read_csv(INPUT_DIR+\"test.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing","metadata":{}},{"cell_type":"code","source":"from text_unidecode import unidecode\nfrom typing import Dict,List,Tuple\nimport codecs\ndef replace_utf8(error: UnicodeError) -> Tuple[bytes,int]:\n    return error.object[error.start : error.end].encode(\"utf-8\"),error.end\ndef replace_cp1252(error : UnicodeError) -> Tuple[str,int]:\n    return error.object[error.start : error.end].decode(\"cp1252\"),error.end\ncodecs.register_error(\"replace_utf8\",replace_utf8)\ncodecs.register_error(\"replace_cp1252\",replace_cp1252)\ndef resolve_encodings_and_normalize(text: str) ->str:\n    text=(\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\",errors=\"replace_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_utf8\")\n        .decode(\"utf-8\", errors=\"replace_cp1252\")\n    )\n    text = unidecode(text)\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_essay_train(essay_id):\n    path = os.path.join(INPUT_DIR+\"train/\", f'{essay_id}.txt')\n    essay_text = open(path, 'r').read()\n    return essay_text\ntrain['essay_text']=train['essay_id'].apply(get_essay_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"misspell_mapping = {\n    'studentdesigned': 'student designed',\n    'teacherdesigned': 'teacher designed',\n    'genericname': 'generic name',\n    'winnertakeall': 'winner take all',\n    'studentname': 'student name',\n    'driveless': 'driverless',\n    'teachername': 'teacher name',\n    'propername': 'proper name',\n    'bestlaid': 'best laid',\n    'genericschool': 'generic school',\n    'schoolname': 'school name',\n    'winnertakesall': 'winner take all',\n    'elctoral': 'electoral',\n    'eletoral': 'electoral',\n    'genericcity': 'generic city',\n    'elctors': 'electoral',\n    'venuse': 'venue',\n    'blimplike': 'blimp like',\n    'selfdriving': 'self driving',\n    'electorals': 'electoral',\n    'nearrecord': 'near record',\n    'egyptianstyle': 'egyptian style',\n    'oddnumbered': 'odd numbered',\n    'carintensive': 'car intensive',\n    'elecoral': 'electoral',\n    'oction': 'auction',\n    'electroal': 'electoral',\n    'evennumbered': 'even numbered',\n    'mesalandforms': 'mesa landforms',\n    'electoralvote': 'electoral vote',\n    'relativename': 'relative name',\n    '22euro': 'twenty two euro',\n    'ellectoral': 'electoral',\n    'thirtyplus': 'thirty plus',\n    'collegewon': 'college won',\n    'hisher': 'higher',\n    'teacherbased': 'teacher based',\n    'computeranimated': 'computer animated',\n    'canadidate': 'candidate',\n    'studentbased': 'student based',\n    'gorethanks': 'gore thanks',\n    'clouddraped': 'cloud draped',\n    'edgarsnyder': 'edgar snyder',\n    'emotionrecognition': 'emotion recognition',\n    'landfrom': 'land form',\n    'fivedays': 'five days',\n    'electoal': 'electoral',\n    'lanform': 'land form',\n    'electral': 'electoral',\n    'presidentbut': 'president but',\n    'teacherassigned': 'teacher assigned',\n    'beacuas': 'because',\n    'positionestimating': 'position estimating',\n    'selfeducation': 'self education',\n    'diverless': 'driverless',\n    'computerdriven': 'computer driven',\n    'outofcontrol': 'out of control',\n    'faultthe': 'fault the',\n    'unfairoutdated': 'unfair outdated',\n    'aviods': 'avoid',\n    'momdad': 'mom dad',\n    'statesbig': 'states big',\n    'presidentswing': 'president swing',\n    'inconclusion': 'in conclusion',\n    'handsonlearning': 'hands on learning',\n    'electroral': 'electoral',\n    'carowner': 'car owner',\n    'elecotral': 'electoral',\n    'studentassigned': 'student assigned',\n    'collegefive': 'college five',\n    'presidant': 'president',\n    'unfairoutdatedand': 'unfair outdated and',\n    'nixonjimmy': 'nixon jimmy',\n    'canadates': 'candidate',\n    'tabletennis': 'table tennis',\n    'himher': 'him her',\n    'studentsummerpacketdesigners': 'student summer packet designers',\n    'studentdesign': 'student designed',\n    'limting': 'limiting',\n    'electrol': 'electoral',\n    'campaignto': 'campaign to',\n    'presendent': 'president',\n    'thezebra': 'the zebra',\n    'landformation': 'land formation',\n    'eyetoeye': 'eye to eye',\n    'selfreliance': 'self reliance',\n    'studentdriven': 'student driven',\n    'winnertake': 'winner take',\n    'alliens': 'aliens',\n    '2000but': '2000 but',\n    'electionto': 'election to',\n    'candidatesas': 'candidates as',\n    'electers': 'electoral',\n    'winnertakes': 'winner takes',\n    'isfeet': 'is feet',\n    'incar': 'incur',\n    'covid19': 'something',\n    'aflcio': '',\n    'outdatedand': 'outdated and',\n    'httpswww': '',\n    '51998': '',\n    'iswing': '',\n    'ascertainments': '',\n    'athome': '',\n    'risorius': '',\n    'votes538': '',\n    '41971': '',\n    'palpabraeus': '',\n    'figurelandform': 'figure landform',\n    'possibleit': 'possible it',\n    'takeall': 'take all',\n    'inschool': 'in school',\n    'fouces': 'focus',\n    'presidentand': 'president and',\n    'elecotrs': 'electoral',\n    'formationwhich': 'formation which',\n    'electorswho': 'electoral who',\n    'presidnt': 'president',\n    'eletors': 'electoral',\n    'sinceraly': 'sincerely',\n    'emotionshappiness': 'emotions happiness',\n    'carterbob': 'carter bob',\n    'donãât': 'do not',\n    'eyesnose': 'eyes nose',\n    'smartroad': 'smart road',\n    'systemvoters': 'system voters',\n    'emtions': 'emotions',\n    'statedemocrats': 'state democrats',\n    'lowcar': 'low car',\n    'elcetoral': 'electoral',\n    'expressivefor': 'expressive for',\n    'animails': 'animals',\n    'oppertonuty': 'opportunity',\n    'tempetures': 'temperature',\n    'recevies': 'receives',\n    'twoseat': 'two seat',\n    'consistution': 'constitution',\n    'horsesyoung': 'horses young',\n    'semidriverless': 'semi driverless',\n    'presisdent': 'president',\n    'exspression': 'expression',\n    'valcanoes': 'volcano',\n    'actiry': '',\n    'lifejust': 'life just',\n    'selfreliant': 'self reliant',\n    'comcaraccidentcauseofaccidentcellphonecellphonestatistics': 'car accident cause of accident cellphone statistics',\n    'vaubangermany': 'germany',\n    'fourtyfour': 'fourty four',\n    'atomspheric': 'atmospheric',\n    'mid1990': '',\n    'activitis': 'activities',\n    'paragrpah': 'paragraph',\n    'electora': 'electoral',\n    'elcetion': 'election',\n    'stressfree': 'stress free',\n    'seegoing': 'see going',\n    'coferencing': 'conferencing',\n    'ctrdot': '',\n    'segoing': '',\n    'teacherdesign': 'teacher design',\n    'kidsteens': 'kids teens',\n    'elcetors': 'electoral',\n    'poulltion': 'pollution',\n    'surportive': 'supportive',\n    'presisent': 'president',\n    'technollogy': 'technology',\n    'precidency': 'president',\n    'voteswhile': 'votes while',\n    'headformed': 'head formed',\n    'swingstates': 'swing states',\n    'candates': 'candidate',\n    'locationname': 'location name',\n    'venuss': 'venues',\n    'astronmers': 'astronomers',\n    'democtratic': 'democratic',\n    'canadent': 'candidate',\n    'cyndonia': '',\n    'computure': 'computer',\n    'nasas': 'nasa',\n    'onehalf': 'one half',\n    'preident': 'president',\n    'ressons': 'reasons',\n    'presidentvice': 'president vice',\n    'nonswing': 'non swing',\n    'thirtyeight': 'thirty eight',\n    'processnot': 'process not',\n    'facetoface': 'face to face',\n    'teendriversource': 'teen driver source',\n    'sadnessand': 'sadness and',\n    'abloish': 'abolish',\n    'driveing': 'driving',\n    'navagating': 'navigating',\n    'electorsthe': 'electoral',\n    'vothing': 'voting',\n    'callage': 'college',\n    'senseit': 'sense it',\n    'mercedesbenz': 'mercedes benz',\n    'electorall': 'electoral'\n}\n    \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decontraction(phrase):\n    phrase = re.sub(r\"won\\'t\", \"will not\", phrase)\n    phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n    phrase = re.sub(r\"he's\", \"he is\", phrase)\n    phrase = re.sub(r\"there's\", \"there is\", phrase)\n    phrase = re.sub(r\"We're\", \"We are\", phrase)\n    phrase = re.sub(r\"That's\", \"That is\", phrase)\n    phrase = re.sub(r\"won't\", \"will not\", phrase)\n    phrase = re.sub(r\"they're\", \"they are\", phrase)\n    phrase = re.sub(r\"Can't\", \"Cannot\", phrase)\n    phrase = re.sub(r\"wasn't\", \"was not\", phrase)\n    phrase = re.sub(r\"don\\x89Ûªt\", \"do not\", phrase)\n    phrase = re.sub(r\"donãât\", \"do not\", phrase)\n    phrase = re.sub(r\"aren't\", \"are not\", phrase)\n    phrase = re.sub(r\"isn't\", \"is not\", phrase)\n    phrase = re.sub(r\"What's\", \"What is\", phrase)\n    phrase = re.sub(r\"haven't\", \"have not\", phrase)\n    phrase = re.sub(r\"hasn't\", \"has not\", phrase)\n    phrase = re.sub(r\"There's\", \"There is\", phrase)\n    phrase = re.sub(r\"He's\", \"He is\", phrase)\n    phrase = re.sub(r\"It's\", \"It is\", phrase)\n    phrase = re.sub(r\"You're\", \"You are\", phrase)\n    phrase = re.sub(r\"I'M\", \"I am\", phrase)\n    phrase = re.sub(r\"shouldn't\", \"should not\", phrase)\n    phrase = re.sub(r\"wouldn't\", \"would not\", phrase)\n    phrase = re.sub(r\"i'm\", \"I am\", phrase)\n    phrase = re.sub(r\"I\\x89Ûªm\", \"I am\", phrase)\n    phrase = re.sub(r\"I'm\", \"I am\", phrase)\n    phrase = re.sub(r\"Isn't\", \"is not\", phrase)\n    phrase = re.sub(r\"Here's\", \"Here is\", phrase)\n    phrase = re.sub(r\"you've\", \"you have\", phrase)\n    phrase = re.sub(r\"you\\x89Ûªve\", \"you have\", phrase)\n    phrase = re.sub(r\"we're\", \"we are\", phrase)\n    phrase = re.sub(r\"what's\", \"what is\", phrase)\n    phrase = re.sub(r\"couldn't\", \"could not\", phrase)\n    phrase = re.sub(r\"we've\", \"we have\", phrase)\n    phrase = re.sub(r\"it\\x89Ûªs\", \"it is\", phrase)\n    phrase = re.sub(r\"doesn\\x89Ûªt\", \"does not\", phrase)\n    phrase = re.sub(r\"It\\x89Ûªs\", \"It is\", phrase)\n    phrase = re.sub(r\"Here\\x89Ûªs\", \"Here is\", phrase)\n    phrase = re.sub(r\"who's\", \"who is\", phrase)\n    phrase = re.sub(r\"I\\x89Ûªve\", \"I have\", phrase)\n    phrase = re.sub(r\"y'all\", \"you all\", phrase)\n    phrase = re.sub(r\"can\\x89Ûªt\", \"cannot\", phrase)\n    phrase = re.sub(r\"would've\", \"would have\", phrase)\n    phrase = re.sub(r\"it'll\", \"it will\", phrase)\n    phrase = re.sub(r\"we'll\", \"we will\", phrase)\n    phrase = re.sub(r\"wouldn\\x89Ûªt\", \"would not\", phrase)\n    phrase = re.sub(r\"We've\", \"We have\", phrase)\n    phrase = re.sub(r\"he'll\", \"he will\", phrase)\n    phrase = re.sub(r\"Y'all\", \"You all\", phrase)\n    phrase = re.sub(r\"Weren't\", \"Were not\", phrase)\n    phrase = re.sub(r\"Didn't\", \"Did not\", phrase)\n    phrase = re.sub(r\"they'll\", \"they will\", phrase)\n    phrase = re.sub(r\"they'd\", \"they would\", phrase)\n    phrase = re.sub(r\"DON'T\", \"DO NOT\", phrase)\n    phrase = re.sub(r\"That\\x89Ûªs\", \"That is\", phrase)\n    phrase = re.sub(r\"they've\", \"they have\", phrase)\n    phrase = re.sub(r\"i'd\", \"I would\", phrase)\n    phrase = re.sub(r\"should've\", \"should have\", phrase)\n    phrase = re.sub(r\"You\\x89Ûªre\", \"You are\", phrase)\n    phrase = re.sub(r\"where's\", \"where is\", phrase)\n    phrase = re.sub(r\"Don\\x89Ûªt\", \"Do not\", phrase)\n    phrase = re.sub(r\"we'd\", \"we would\", phrase)\n    phrase = re.sub(r\"i'll\", \"I will\", phrase)\n    phrase = re.sub(r\"weren't\", \"were not\", phrase)\n    phrase = re.sub(r\"They're\", \"They are\", phrase)\n    phrase = re.sub(r\"Can\\x89Ûªt\", \"Cannot\", phrase)\n    phrase = re.sub(r\"you\\x89Ûªll\", \"you will\", phrase)\n    phrase = re.sub(r\"I\\x89Ûªd\", \"I would\", phrase)\n    phrase = re.sub(r\"let's\", \"let us\", phrase)\n    phrase = re.sub(r\"it's\", \"it is\", phrase)\n    phrase = re.sub(r\"can't\", \"cannot\", phrase)\n    phrase = re.sub(r\"don't\", \"do not\", phrase)\n    phrase = re.sub(r\"you're\", \"you are\", phrase)\n    phrase = re.sub(r\"i've\", \"I have\", phrase)\n    phrase = re.sub(r\"that's\", \"that is\", phrase)\n    phrase = re.sub(r\"i'll\", \"I will\", phrase)\n    phrase = re.sub(r\"doesn't\", \"does not\",phrase)\n    phrase = re.sub(r\"i'd\", \"I would\", phrase)\n    phrase = re.sub(r\"didn't\", \"did not\", phrase)\n    phrase = re.sub(r\"ain't\", \"am not\", phrase)\n    phrase = re.sub(r\"you'll\", \"you will\", phrase)\n    phrase = re.sub(r\"I've\", \"I have\", phrase)\n    phrase = re.sub(r\"Don't\", \"do not\", phrase)\n    phrase = re.sub(r\"I'll\", \"I will\", phrase)\n    phrase = re.sub(r\"I'd\", \"I would\", phrase)\n    phrase = re.sub(r\"Let's\", \"Let us\", phrase)\n    phrase = re.sub(r\"you'd\", \"You would\", phrase)\n    phrase = re.sub(r\"It's\", \"It is\", phrase)\n    phrase = re.sub(r\"Ain't\", \"am not\", phrase)\n    phrase = re.sub(r\"Haven't\", \"Have not\", phrase)\n    phrase = re.sub(r\"Could've\", \"Could have\", phrase)\n    phrase = re.sub(r\"youve\", \"you have\", phrase)  \n    phrase = re.sub(r\"donå«t\", \"do not\", phrase)\n    return phrase","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_punctuations(text):\n    for punctuation in list(string.punctuation):\n        text = text.replace(punctuation, '')\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_number(text):\n    text = re.sub(r'(\\d+)([a-zA-Z])', '\\g<1> \\g<2>', text)\n    text = re.sub(r'(\\d+) (th|st|nd|rd) ', '\\g<1>\\g<2> ', text)\n    text = re.sub(r'(\\d+),(\\d+)', '\\g<1>\\g<2>', text)\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_misspell(text):\n    for bad_word in misspell_mapping:\n        if bad_word in text:\n            text = text.replace(bad_word, misspell_mapping[bad_word])\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(text):\n    text = decontraction(text)\n    text = remove_punctuations(text)\n    text = clean_number(text)\n    text = clean_misspell(text)\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Topic Names ","metadata":{}},{"cell_type":"code","source":"topic_pred_df = pd.read_csv('../input/feedback-topics-identification-with-bertopic/topic_model_feedback.csv')\ntopic_pred_df = topic_pred_df.drop(columns={'prob'})\ntopic_pred_df = topic_pred_df.rename(columns={'id': 'essay_id'})\n\ntopic_meta_df = pd.read_csv('../input/feedback-topics-identification-with-bertopic/topic_model_metadata.csv')\ntopic_meta_df = topic_meta_df.rename(columns={'Topic': 'topic', 'Name': 'topic_name'}).drop(columns=['Count'])\ntopic_meta_df.topic_name = topic_meta_df.topic_name.apply(lambda n: ' '.join(n.split('_')[1:]))\n\ntopic_pred_df = topic_pred_df.merge(topic_meta_df, on='topic', how='left')\ntrain = train.merge(topic_pred_df, on='essay_id', how='left')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_old_feedback=pd.read_csv(\"../input/feedback-prize-2021/train.csv\")\ntrain_2021_preds=pd.read_csv(\"../input/feedback-pseudo-labelling-full-2021-dataset/train_2021_preds.csv\")\ntrain_2021_preds=train_2021_preds[train_2021_preds.in_2022 == False]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_essay2_train(essay_id):\n    path = os.path.join(\"../input/feedback-prize-2021/train/\", f'{essay_id}.txt')\n    essay_text = open(path, 'r').read()\n    return essay_text\ntrain_2021_preds['essay_text']=train_2021_preds['essay_id'].apply(get_essay2_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_2021_preds=train_2021_preds.drop(['in_2022','essay_fn','Adequate','Effective','Ineffective'],axis=1)\ntrain_2021_preds=train_2021_preds[['discourse_id','essay_id','discourse_text','discourse_type','discourse_effectiveness','essay_text','topic','topic_name']]\ntrain_df=pd.concat([train,train_2021_preds])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_text']=train_df['discourse_text'].str.lower()\ntrain_df['discourse_type']=train_df['discourse_type'].str.lower()\ntrain_df['essay_text']=train_df['essay_text'].str.lower()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_text']=train_df['discourse_text'].apply(lambda x : resolve_encodings_and_normalize(x))\ntrain_df['essay_text']=train_df['essay_text'].apply(lambda x:resolve_encodings_and_normalize(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_text']=train_df['discourse_text'].progress_apply(clean_text)\ntrain_df['essay_text']=train_df['essay_text'].progress_apply(clean_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED']=str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"config = SimpleNamespace()\n\nconfig.seed = 42\nconfig.model_name = 'microsoft/deberta-v3-base'\nconfig.output_path = Path('./')\nconfig.input_path = Path('../input/feedback-prize-effectiveness')\n\nconfig.n_folds = 5\nconfig.lr = 1e-5\nconfig.weight_decay = 0.01\nconfig.epochs = 3\nconfig.batch_size = 16\nconfig.gradient_accumulation_steps = 1\nconfig.warm_up_ratio = 0.1\nconfig.max_len = 512\nconfig.hidden_dropout_prob = 0.1\nconfig.label_smoothing_factor = 0.1\nconfig.eval_per_epoch = 2\n\nlogging.disable(logging.WARNING)\n\nseed_everything(config.seed)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.config = config.__dict__\nwandb.init(project=\"feedback-prize-effectiveness\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disc_types = [\n    \"Claim\",\n    \"Concluding Statement\",\n    \"Counterclaim\",\n    \"Evidence\",\n    \"Lead\",\n    \"Position\",\n    \"Rebuttal\",\n]\ncls_tokens_map = {label: f\"[CLS_{label.upper()}]\" for label in disc_types}\nend_tokens_map = {label: f\"[END_{label.upper()}]\" for label in disc_types}\n\nlabel2id = {\n    \"Adequate\": 0,\n    \"Effective\": 1,\n    \"Ineffective\": 2,\n}\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedKFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\ntrain_df['fold']=-1\nfor fold_num,(train_idxs,test_idxs) in enumerate(cv.split(train_df.index,train_df.discourse_effectiveness,train_df.essay_id)):\n    train_df.iloc[test_idxs,len(train_df.columns) - 1]=fold_num","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(config.model_name, use_fast=True)\ntokenizer.model_max_length = config.max_len","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer.add_special_tokens(\n    {\"additional_special_tokens\": list(cls_tokens_map.values())+list(end_tokens_map.values())}\n)\ncls_id_map = {\n    label: tokenizer.encode(tkn)[1]\n    for label, tkn in cls_tokens_map.items()\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['label']=train_df['discourse_effectiveness'].map({v: i for i, v in enumerate([\"Ineffective\", \"Adequate\", \"Effective\"]\n)}\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEP = tokenizer.sep_token\ntrain_df['text']=train_df['discourse_type'] + ' ' +train_df['topic_name']+ SEP +train_df['discourse_text'] + SEP + train_df['essay_text']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenizer_func(x):\n    return tokenizer(x[\"text\"], truncation=True, max_length=config.max_len)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('fold')['discourse_effectiveness'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels= ['Adequate', 'Effective', 'Ineffective']\nlabels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ntrain_df['discourse_effectiveness'] = encoder.fit_transform(train_df['discourse_effectiveness'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch import nn\nfrom transformers import AutoConfig, AutoModelForSequenceClassification\nfrom transformers.models.deberta_v2.modeling_deberta_v2 import ContextPooler\nfrom transformers.models.deberta_v2.modeling_deberta_v2 import StableDropout\nfrom transformers.modeling_outputs import TokenClassifierOutput\n\ndef get_dropouts(num, start_prob, increment):\n    return [StableDropout(start_prob + (increment * i)) for i in range(num)]  \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MeanPooling(nn.Module):\n    def __init__(self):\n        super(MeanPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n        sum_mask = input_mask_expanded.sum(1)\n        sum_mask = torch.clamp(sum_mask, min=1e-9)\n        mean_embeddings = sum_embeddings / sum_mask\n\n        return mean_embeddings\n\n    \nclass MeanMaxPooling(nn.Module):\n    def __init__(self):\n        super(MeanMaxPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        mean_pooling_embeddings = torch.mean(last_hidden_state, 1)\n        _, max_pooling_embeddings = torch.max(last_hidden_state, 1)\n        mean_max_embeddings = torch.cat((mean_pooling_embeddings, max_pooling_embeddings), 1)\n        return mean_max_embeddings\n\n    \nclass LSTMPooling(nn.Module):\n    def __init__(self, num_layers, hidden_size, hiddendim_lstm):\n        super(LSTMPooling, self).__init__()\n        self.num_hidden_layers = num_layers\n        self.hidden_size = hidden_size\n        self.hiddendim_lstm = hiddendim_lstm\n        self.lstm = nn.LSTM(self.hidden_size, self.hiddendim_lstm, batch_first=True)\n        self.dropout = nn.Dropout(0.1)\n    \n    def forward(self, all_hidden_states):\n        ## forward\n        hidden_states = torch.stack([all_hidden_states[layer_i][:, 0].squeeze()\n                                     for layer_i in range(1, self.num_hidden_layers+1)], dim=-1)\n        hidden_states = hidden_states.view(-1, self.num_hidden_layers, self.hidden_size)\n        out, _ = self.lstm(hidden_states, None)\n        out = self.dropout(out[:, -1, :])\n        return out\n    \nclass WeightedLayerPooling(nn.Module):\n    def __init__(self, num_hidden_layers, layer_start: int = 4, layer_weights = None):\n        super(WeightedLayerPooling, self).__init__()\n        self.layer_start = layer_start\n        self.num_hidden_layers = num_hidden_layers\n        self.layer_weights = layer_weights if layer_weights is not None \\\n            else nn.Parameter(\n                torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float)\n            )\n\n    def forward(self, all_hidden_states):\n        all_layer_embedding = torch.stack(list(all_hidden_states), dim=0)\n        all_layer_embedding = all_layer_embedding[self.layer_start:, :, :, :]\n        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n        return weighted_average","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Collate:\n    def __init__(self, tokenizer, isTrain=True):\n        self.tokenizer = tokenizer\n        self.isTrain = isTrain\n        # self.args = args\n\n    def __call__(self, batch):\n        output = dict()\n        output[\"input_ids\"] = [sample[\"input_ids\"] for sample in batch]\n        output[\"attention_mask\"] = [sample[\"attention_mask\"] for sample in batch]\n        if self.isTrain:\n            output[\"target\"] = [sample[\"target\"] for sample in batch]\n\n        # calculate max token length of this batch\n        batch_max = max([len(ids) for ids in output[\"input_ids\"]])\n\n        # add padding\n        if self.tokenizer.padding_side == \"right\":\n            output[\"input_ids\"] = [s + (batch_max - len(s)) * [self.tokenizer.pad_token_id] for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [s + (batch_max - len(s)) * [0] for s in output[\"attention_mask\"]]\n        else:\n            output[\"input_ids\"] = [(batch_max - len(s)) * [self.tokenizer.pad_token_id] + s for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [(batch_max - len(s)) * [0] + s for s in output[\"attention_mask\"]]\n\n        # convert to tensors\n        output[\"input_ids\"] = torch.tensor(output[\"input_ids\"], dtype=torch.long)\n        output[\"attention_mask\"] = torch.tensor(output[\"attention_mask\"], dtype=torch.long)\n        if self.isTrain:\n            output[\"target\"] = torch.tensor(output[\"target\"], dtype=torch.long)\n\n        return output\n\n# collate_fn = DataCollatorWithPadding(tokenizer=CFG.tokenizer)\ncollate_fn = Collate(tokenizer)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class CustomModel(nn.Module):\n    def __init__(self, backbone):\n        super(CustomModel, self).__init__()\n        \n        self.model = backbone\n        self.config = self.model.config\n        self.num_labels = self.config.num_labels\n\n        # self.pooler = ContextPooler(self.config)\n        self.weighted_pooler = WeightedLayerPooling(num_hidden_layers=self.config.num_hidden_layers, layer_start=4)\n        self.pooler = MeanPooling()\n        \n        self.classifier = nn.Linear(self.config.hidden_size, self.num_labels)\n        self.bilstm = nn.LSTM(self.config.hidden_size, self.config.hidden_size//2, num_layers=2, \n                             dropout=self.config.hidden_dropout_prob, batch_first=True,\n                             bidirectional=False)\n    \n        self.dropouts = get_dropouts(num=5, start_prob=config.hidden_dropout_prob - 0.02, increment=0.01)\n    \n    def forward(\n        self,\n        input_ids=None,\n        attention_mask=None,\n        token_type_ids=None,\n        position_ids=None,\n        inputs_embeds=None,\n        labels=None,\n        output_attentions=None,\n        output_hidden_states=None,\n        return_dict=None\n    ):\n        \n        outputs = self.model.deberta(\n            input_ids,\n            token_type_ids=token_type_ids,\n            attention_mask=attention_mask,\n            position_ids=position_ids,\n            inputs_embeds=inputs_embeds,\n            output_attentions=output_attentions,\n            output_hidden_states=output_hidden_states,\n            return_dict=return_dict,\n        )\n        \n        \n        encoder_layer = outputs[0]\n        \n        pooled_output = self.pooler(encoder_layer, attention_mask)\n                     \n        # Multi-sample dropout.\n        num_dps = float(len(self.dropouts))\n        for ii, drop in enumerate(self.dropouts):\n            if ii == 0:\n                logits = (self.classifier(drop(pooled_output)) / num_dps)\n            else:\n                logits += (self.classifier(drop(pooled_output)) / num_dps)\n\n        \n        loss = None\n        if labels is not None:\n            loss_fn = nn.CrossEntropyLoss()\n            logits = logits.view(-1, self.num_labels)\n            loss = loss_fn(logits, labels.view(-1))\n\n        output = (logits,) + outputs[1:]\n\n        return TokenClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states, attentions=outputs.attentions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_backbone_config():\n    model_config = AutoConfig.from_pretrained(config.model_name, num_labels=3)\n    model_config.hidden_dropout_prob = config.hidden_dropout_prob\n    return model_config","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    model_config = get_backbone_config()\n\n    model = AutoModelForSequenceClassification.from_pretrained(\n        config.model_name,\n        config=model_config,\n    )\n    return CustomModel(model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"backbone_config = get_backbone_config()\nbackbone_config.save_pretrained('./backbone_config')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = load_metric('accuracy')\n\ndef compute_metrics(eval_pred):\n    predictions, labels = eval_pred\n    predictions = np.argmax(predictions, axis=1)\n    return metric.compute(predictions=predictions, references=labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"def do_fold(fold_num):\n    train_data  = train_df.query(f'fold != {fold_num}').reset_index(drop=True)\n\n    val_data  = train_df.query(f'fold == {fold_num}').reset_index(drop=True)\n    \n\n    \n\n    train_dataset = Dataset.from_pandas(train_data[['text', 'essay_id', 'discourse_effectiveness']]).rename_column('discourse_effectiveness', 'label').class_encode_column(\"label\")\n    val_dataset = Dataset.from_pandas(val_data[['text', 'essay_id', 'discourse_effectiveness']]).rename_column('discourse_effectiveness', 'label').class_encode_column(\"label\")\n\n    train_tok_dataset = train_dataset.map(tokenizer_func, batched=True, remove_columns=('text', 'essay_id'))\n    val_tok_dataset = val_dataset.map(tokenizer_func, batched=True, remove_columns=('text', 'essay_id'))\n\n    data_collator = DataCollatorWithPadding(tokenizer=tokenizer, padding=True)\n\n    num_steps = len(train_data) / config.batch_size / config.gradient_accumulation_steps\n    eval_steps = 3500\n    print(f'Num steps: {num_steps}, eval steps: {eval_steps}')\n\n    args = TrainingArguments(\n        output_dir=config.output_path,\n        learning_rate=config.lr,\n        warmup_ratio=config.warm_up_ratio,\n        lr_scheduler_type='cosine_with_restarts',\n        fp16=True,\n        per_device_train_batch_size=config.batch_size,\n        per_device_eval_batch_size=config.batch_size * 2,\n        num_train_epochs=config.epochs,\n        weight_decay=config.weight_decay,\n        report_to=\"wandb\",\n\n        evaluation_strategy='steps',\n        eval_steps=eval_steps, \n        save_strategy='steps',\n        save_steps=eval_steps,\n        \n        load_best_model_at_end=True,\n        gradient_accumulation_steps=config.gradient_accumulation_steps,\n        label_smoothing_factor=config.label_smoothing_factor,\n        save_total_limit=1  # Prevents running out of disk space.\n    )\n\n    model = get_model()\n\n    trainer = Trainer(\n        model,\n        args,\n        train_dataset=train_tok_dataset,\n        eval_dataset=val_tok_dataset,\n        tokenizer=tokenizer,\n        data_collator=data_collator,\n        compute_metrics=compute_metrics\n    )\n\n    trainer.train()\n    \n    trainer.save_model(config.output_path / f'fold_{fold_num}')\n    \n    outputs = trainer.predict(val_tok_dataset)\n\n    val_data[labels] = softmax(outputs.predictions, axis=1)\n    \n    !rm -rf {config.output_path / 'checkpoint'}*\n    \n    return val_data\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fold 0","metadata":{}},{"cell_type":"code","source":"val_preds_df = pd.DataFrame()\n\nval_data = do_fold(0)\n\nval_preds_df = pd.concat([val_preds_df, val_data])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Rest of the Folds","metadata":{}},{"cell_type":"code","source":"for fold in range(1,5):\n    val_data = do_fold(fold)\n    val_preds_df = pd.concat([val_preds_df, val_data])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = log_loss(val_preds_df['discourse_effectiveness'], val_preds_df[labels])\ncv","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.log({\"cv\": cv})","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}