{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\niskaggle = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-03T05:32:44.099218Z","iopub.execute_input":"2022-08-03T05:32:44.099714Z","iopub.status.idle":"2022-08-03T05:32:44.106520Z","shell.execute_reply.started":"2022-08-03T05:32:44.099668Z","shell.execute_reply":"2022-08-03T05:32:44.104776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:44.110325Z","iopub.execute_input":"2022-08-03T05:32:44.111478Z","iopub.status.idle":"2022-08-03T05:32:44.119850Z","shell.execute_reply.started":"2022-08-03T05:32:44.111417Z","shell.execute_reply":"2022-08-03T05:32:44.118612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if iskaggle:\n    path = Path('../input/nlp-getting-started')\n    ! pip install -q datasets # it's a bash command ","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:44.121482Z","iopub.execute_input":"2022-08-03T05:32:44.122168Z","iopub.status.idle":"2022-08-03T05:32:55.697232Z","shell.execute_reply.started":"2022-08-03T05:32:44.122126Z","shell.execute_reply":"2022-08-03T05:32:55.696019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls {path}","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:55.699014Z","iopub.execute_input":"2022-08-03T05:32:55.699452Z","iopub.status.idle":"2022-08-03T05:32:56.812518Z","shell.execute_reply.started":"2022-08-03T05:32:55.699412Z","shell.execute_reply":"2022-08-03T05:32:56.810808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:56.817385Z","iopub.execute_input":"2022-08-03T05:32:56.818560Z","iopub.status.idle":"2022-08-03T05:32:56.825337Z","shell.execute_reply.started":"2022-08-03T05:32:56.818499Z","shell.execute_reply":"2022-08-03T05:32:56.824114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:56.827181Z","iopub.execute_input":"2022-08-03T05:32:56.828137Z","iopub.status.idle":"2022-08-03T05:32:56.867445Z","shell.execute_reply.started":"2022-08-03T05:32:56.828091Z","shell.execute_reply":"2022-08-03T05:32:56.866239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:56.869230Z","iopub.execute_input":"2022-08-03T05:32:56.870014Z","iopub.status.idle":"2022-08-03T05:32:56.893680Z","shell.execute_reply.started":"2022-08-03T05:32:56.869968Z","shell.execute_reply":"2022-08-03T05:32:56.892667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe(include = 'object')","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:56.895516Z","iopub.execute_input":"2022-08-03T05:32:56.896516Z","iopub.status.idle":"2022-08-03T05:32:56.929292Z","shell.execute_reply.started":"2022-08-03T05:32:56.896465Z","shell.execute_reply":"2022-08-03T05:32:56.928003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:56.930800Z","iopub.execute_input":"2022-08-03T05:32:56.931687Z","iopub.status.idle":"2022-08-03T05:32:56.937180Z","shell.execute_reply.started":"2022-08-03T05:32:56.931627Z","shell.execute_reply":"2022-08-03T05:32:56.935987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['input'] = 'TEXT1: ' + df.keyword + '; TEXT2: ' + df.location + '; TEXT3: ' + df.text \n","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:56.938671Z","iopub.execute_input":"2022-08-03T05:32:56.939181Z","iopub.status.idle":"2022-08-03T05:32:56.965038Z","shell.execute_reply.started":"2022-08-03T05:32:56.939140Z","shell.execute_reply":"2022-08-03T05:32:56.963455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.dropna()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:56.966914Z","iopub.execute_input":"2022-08-03T05:32:56.967374Z","iopub.status.idle":"2022-08-03T05:32:56.982533Z","shell.execute_reply.started":"2022-08-03T05:32:56.967334Z","shell.execute_reply":"2022-08-03T05:32:56.981401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.reset_index(drop=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:56.984194Z","iopub.execute_input":"2022-08-03T05:32:56.984729Z","iopub.status.idle":"2022-08-03T05:32:56.992458Z","shell.execute_reply.started":"2022-08-03T05:32:56.984684Z","shell.execute_reply":"2022-08-03T05:32:56.991020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:56.999295Z","iopub.execute_input":"2022-08-03T05:32:57.000014Z","iopub.status.idle":"2022-08-03T05:32:57.019367Z","shell.execute_reply.started":"2022-08-03T05:32:56.999968Z","shell.execute_reply":"2022-08-03T05:32:57.018233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import Dataset, DatasetDict\nds = Dataset.from_pandas(df)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:57.020984Z","iopub.execute_input":"2022-08-03T05:32:57.022022Z","iopub.status.idle":"2022-08-03T05:32:57.043768Z","shell.execute_reply.started":"2022-08-03T05:32:57.021978Z","shell.execute_reply":"2022-08-03T05:32:57.042486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:57.046096Z","iopub.execute_input":"2022-08-03T05:32:57.047012Z","iopub.status.idle":"2022-08-03T05:32:57.054399Z","shell.execute_reply.started":"2022-08-03T05:32:57.046953Z","shell.execute_reply":"2022-08-03T05:32:57.052987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_nm = 'microsoft/deberta-v3-small'","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:57.055992Z","iopub.execute_input":"2022-08-03T05:32:57.057073Z","iopub.status.idle":"2022-08-03T05:32:57.064949Z","shell.execute_reply.started":"2022-08-03T05:32:57.057002Z","shell.execute_reply":"2022-08-03T05:32:57.063895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForSequenceClassification, AutoTokenizer\ntokz = AutoTokenizer.from_pretrained(model_nm)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:32:57.066556Z","iopub.execute_input":"2022-08-03T05:32:57.067210Z","iopub.status.idle":"2022-08-03T05:33:07.788200Z","shell.execute_reply.started":"2022-08-03T05:32:57.067158Z","shell.execute_reply":"2022-08-03T05:33:07.786940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tok_func(x): return tokz(x[\"input\"])","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:07.791184Z","iopub.execute_input":"2022-08-03T05:33:07.792508Z","iopub.status.idle":"2022-08-03T05:33:07.798323Z","shell.execute_reply.started":"2022-08-03T05:33:07.792464Z","shell.execute_reply":"2022-08-03T05:33:07.796800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tok_ds = ds.map(tok_func, batched=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:07.800985Z","iopub.execute_input":"2022-08-03T05:33:07.802274Z","iopub.status.idle":"2022-08-03T05:33:08.456671Z","shell.execute_reply.started":"2022-08-03T05:33:07.802229Z","shell.execute_reply":"2022-08-03T05:33:08.455327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tok_ds = tok_ds.rename_columns({'target':'labels'})","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.458667Z","iopub.execute_input":"2022-08-03T05:33:08.459090Z","iopub.status.idle":"2022-08-03T05:33:08.468612Z","shell.execute_reply.started":"2022-08-03T05:33:08.459052Z","shell.execute_reply":"2022-08-03T05:33:08.467091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = tok_ds[0]\nrow['input']","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.470308Z","iopub.execute_input":"2022-08-03T05:33:08.471012Z","iopub.status.idle":"2022-08-03T05:33:08.479391Z","shell.execute_reply.started":"2022-08-03T05:33:08.470959Z","shell.execute_reply":"2022-08-03T05:33:08.478090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df = pd.read_csv(path/'test.csv')\neval_df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.481292Z","iopub.execute_input":"2022-08-03T05:33:08.482210Z","iopub.status.idle":"2022-08-03T05:33:08.517545Z","shell.execute_reply.started":"2022-08-03T05:33:08.482150Z","shell.execute_reply":"2022-08-03T05:33:08.516537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df['input'] = 'TEXT1: ' + df.keyword + '; TEXT2: ' + df.location + '; TEXT3: ' + df.text \n","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.519191Z","iopub.execute_input":"2022-08-03T05:33:08.519708Z","iopub.status.idle":"2022-08-03T05:33:08.532668Z","shell.execute_reply.started":"2022-08-03T05:33:08.519656Z","shell.execute_reply":"2022-08-03T05:33:08.531489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df = eval_df.dropna()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.534389Z","iopub.execute_input":"2022-08-03T05:33:08.534990Z","iopub.status.idle":"2022-08-03T05:33:08.546774Z","shell.execute_reply.started":"2022-08-03T05:33:08.534945Z","shell.execute_reply":"2022-08-03T05:33:08.545325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df = eval_df.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.548284Z","iopub.execute_input":"2022-08-03T05:33:08.549274Z","iopub.status.idle":"2022-08-03T05:33:08.555552Z","shell.execute_reply.started":"2022-08-03T05:33:08.549228Z","shell.execute_reply":"2022-08-03T05:33:08.554262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_ds = Dataset.from_pandas(eval_df).map(tok_func, batched=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.557424Z","iopub.execute_input":"2022-08-03T05:33:08.557885Z","iopub.status.idle":"2022-08-03T05:33:08.839995Z","shell.execute_reply.started":"2022-08-03T05:33:08.557843Z","shell.execute_reply":"2022-08-03T05:33:08.839000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import TrainingArguments, Trainer","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.841341Z","iopub.execute_input":"2022-08-03T05:33:08.842287Z","iopub.status.idle":"2022-08-03T05:33:08.847499Z","shell.execute_reply.started":"2022-08-03T05:33:08.842227Z","shell.execute_reply":"2022-08-03T05:33:08.846220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bs = 128\nepochs = 4\nlr = 8e-5","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.848886Z","iopub.execute_input":"2022-08-03T05:33:08.849336Z","iopub.status.idle":"2022-08-03T05:33:08.859797Z","shell.execute_reply.started":"2022-08-03T05:33:08.849296Z","shell.execute_reply":"2022-08-03T05:33:08.858781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args = TrainingArguments('outputs', learning_rate=lr, warmup_ratio=0.1, lr_scheduler_type='cosine', \n    evaluation_strategy=\"epoch\", per_device_train_batch_size=bs, per_device_eval_batch_size=bs*2, \n    num_train_epochs=epochs, weight_decay=0.01, report_to='none') ","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.861374Z","iopub.execute_input":"2022-08-03T05:33:08.862021Z","iopub.status.idle":"2022-08-03T05:33:08.874740Z","shell.execute_reply.started":"2022-08-03T05:33:08.861976Z","shell.execute_reply":"2022-08-03T05:33:08.873441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def corr(x,y): return np.corrcoef(x,y)[0][1]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.876539Z","iopub.execute_input":"2022-08-03T05:33:08.877116Z","iopub.status.idle":"2022-08-03T05:33:08.887694Z","shell.execute_reply.started":"2022-08-03T05:33:08.877070Z","shell.execute_reply":"2022-08-03T05:33:08.886593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def corr_d(eval_pred): return {'pearson': corr(*eval_pred)}","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.889207Z","iopub.execute_input":"2022-08-03T05:33:08.890078Z","iopub.status.idle":"2022-08-03T05:33:08.901180Z","shell.execute_reply.started":"2022-08-03T05:33:08.890023Z","shell.execute_reply":"2022-08-03T05:33:08.899878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dds = tok_ds.train_test_split(0.25, seed=42)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.903144Z","iopub.execute_input":"2022-08-03T05:33:08.904076Z","iopub.status.idle":"2022-08-03T05:33:08.922609Z","shell.execute_reply.started":"2022-08-03T05:33:08.904021Z","shell.execute_reply":"2022-08-03T05:33:08.921508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = AutoModelForSequenceClassification.from_pretrained(model_nm, num_labels=1)\ntrainer = Trainer(model, args, train_dataset=dds['train'], eval_dataset=dds['test'],\n                  tokenizer=tokz, compute_metrics=corr_d)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:08.924554Z","iopub.execute_input":"2022-08-03T05:33:08.925396Z","iopub.status.idle":"2022-08-03T05:33:12.434261Z","shell.execute_reply.started":"2022-08-03T05:33:08.925353Z","shell.execute_reply":"2022-08-03T05:33:12.432967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:12.435825Z","iopub.execute_input":"2022-08-03T05:33:12.436250Z","iopub.status.idle":"2022-08-03T05:33:28.456593Z","shell.execute_reply.started":"2022-08-03T05:33:12.436212Z","shell.execute_reply":"2022-08-03T05:33:28.454355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = trainer.predict(eval_ds).predictions.astype(float)\npreds","metadata":{"execution":{"iopub.status.busy":"2022-08-03T05:33:28.458320Z","iopub.status.idle":"2022-08-03T05:33:28.458871Z","shell.execute_reply.started":"2022-08-03T05:33:28.458638Z","shell.execute_reply":"2022-08-03T05:33:28.458662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}