{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-11T04:57:51.483297Z","iopub.execute_input":"2022-07-11T04:57:51.484529Z","iopub.status.idle":"2022-07-11T04:57:51.519207Z","shell.execute_reply.started":"2022-07-11T04:57:51.484413Z","shell.execute_reply":"2022-07-11T04:57:51.518266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport os\n\niskaggle = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '')\nif iskaggle:\n    !pip install -Uqq fastai\nelse:\n    import zipfile,kaggle\n    path = Path('us-patent-phrase-to-phrase-matching')\n    kaggle.api.competition_download_cli(str(path))\n    zipfile.ZipFile(f'{path}.zip').extractall(path)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:57:51.523942Z","iopub.execute_input":"2022-07-11T04:57:51.526405Z","iopub.status.idle":"2022-07-11T04:58:04.909293Z","shell.execute_reply.started":"2022-07-11T04:57:51.526369Z","shell.execute_reply":"2022-07-11T04:58:04.908116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.imports import *","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:04.911703Z","iopub.execute_input":"2022-07-11T04:58:04.912097Z","iopub.status.idle":"2022-07-11T04:58:05.045616Z","shell.execute_reply.started":"2022-07-11T04:58:04.912048Z","shell.execute_reply":"2022-07-11T04:58:05.044496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if iskaggle: path = Path('../input/us-patent-phrase-to-phrase-matching')\npath.ls()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:05.053046Z","iopub.execute_input":"2022-07-11T04:58:05.055948Z","iopub.status.idle":"2022-07-11T04:58:05.073058Z","shell.execute_reply.started":"2022-07-11T04:58:05.055901Z","shell.execute_reply":"2022-07-11T04:58:05.071804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')\ndf","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:05.077867Z","iopub.execute_input":"2022-07-11T04:58:05.080446Z","iopub.status.idle":"2022-07-11T04:58:05.183211Z","shell.execute_reply.started":"2022-07-11T04:58:05.080402Z","shell.execute_reply":"2022-07-11T04:58:05.182335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df = pd.read_csv(path/'test.csv')\nlen(eval_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:05.184892Z","iopub.execute_input":"2022-07-11T04:58:05.185525Z","iopub.status.idle":"2022-07-11T04:58:05.197411Z","shell.execute_reply.started":"2022-07-11T04:58:05.185450Z","shell.execute_reply":"2022-07-11T04:58:05.196366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:05.200056Z","iopub.execute_input":"2022-07-11T04:58:05.200906Z","iopub.status.idle":"2022-07-11T04:58:05.213386Z","shell.execute_reply.started":"2022-07-11T04:58:05.200850Z","shell.execute_reply":"2022-07-11T04:58:05.212314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:05.214677Z","iopub.execute_input":"2022-07-11T04:58:05.215099Z","iopub.status.idle":"2022-07-11T04:58:05.245439Z","shell.execute_reply.started":"2022-07-11T04:58:05.215062Z","shell.execute_reply":"2022-07-11T04:58:05.243601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.anchor.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:05.247169Z","iopub.execute_input":"2022-07-11T04:58:05.247896Z","iopub.status.idle":"2022-07-11T04:58:05.261931Z","shell.execute_reply.started":"2022-07-11T04:58:05.247859Z","shell.execute_reply":"2022-07-11T04:58:05.260768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.context.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:05.266398Z","iopub.execute_input":"2022-07-11T04:58:05.267073Z","iopub.status.idle":"2022-07-11T04:58:05.281673Z","shell.execute_reply.started":"2022-07-11T04:58:05.267043Z","shell.execute_reply":"2022-07-11T04:58:05.280607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['section'] = df.context.str[0]\ndf.section.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:05.282997Z","iopub.execute_input":"2022-07-11T04:58:05.283326Z","iopub.status.idle":"2022-07-11T04:58:05.325125Z","shell.execute_reply.started":"2022-07-11T04:58:05.283293Z","shell.execute_reply":"2022-07-11T04:58:05.324204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.score.hist();","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:05.326541Z","iopub.execute_input":"2022-07-11T04:58:05.327106Z","iopub.status.idle":"2022-07-11T04:58:05.587413Z","shell.execute_reply.started":"2022-07-11T04:58:05.327069Z","shell.execute_reply":"2022-07-11T04:58:05.585292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df.score==1]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:05.589835Z","iopub.execute_input":"2022-07-11T04:58:05.590392Z","iopub.status.idle":"2022-07-11T04:58:05.638152Z","shell.execute_reply.started":"2022-07-11T04:58:05.590351Z","shell.execute_reply":"2022-07-11T04:58:05.637220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader\nimport warnings,transformers,logging,torch\nfrom transformers import TrainingArguments,Trainer\nfrom transformers import AutoModelForSequenceClassification,AutoTokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:05.639661Z","iopub.execute_input":"2022-07-11T04:58:05.640233Z","iopub.status.idle":"2022-07-11T04:58:12.535213Z","shell.execute_reply.started":"2022-07-11T04:58:05.640196Z","shell.execute_reply":"2022-07-11T04:58:12.534240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if iskaggle:\n    !pip install -q datasets\nimport datasets\nfrom datasets import load_dataset, Dataset, DatasetDict","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:12.536653Z","iopub.execute_input":"2022-07-11T04:58:12.537355Z","iopub.status.idle":"2022-07-11T04:58:21.657498Z","shell.execute_reply.started":"2022-07-11T04:58:12.537296Z","shell.execute_reply":"2022-07-11T04:58:21.656316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.simplefilter('ignore')\nlogging.disable(logging.WARNING)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:21.659570Z","iopub.execute_input":"2022-07-11T04:58:21.659954Z","iopub.status.idle":"2022-07-11T04:58:21.666619Z","shell.execute_reply.started":"2022-07-11T04:58:21.659916Z","shell.execute_reply":"2022-07-11T04:58:21.665716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_nm = 'microsoft/deberta-v3-small'","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:21.667951Z","iopub.execute_input":"2022-07-11T04:58:21.668177Z","iopub.status.idle":"2022-07-11T04:58:21.681141Z","shell.execute_reply.started":"2022-07-11T04:58:21.668146Z","shell.execute_reply":"2022-07-11T04:58:21.680295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokz = AutoTokenizer.from_pretrained(model_nm)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:21.682638Z","iopub.execute_input":"2022-07-11T04:58:21.682883Z","iopub.status.idle":"2022-07-11T04:58:26.145073Z","shell.execute_reply.started":"2022-07-11T04:58:21.682850Z","shell.execute_reply":"2022-07-11T04:58:26.144127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sep = tokz.sep_token\nsep","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:26.146349Z","iopub.execute_input":"2022-07-11T04:58:26.146598Z","iopub.status.idle":"2022-07-11T04:58:26.154601Z","shell.execute_reply.started":"2022-07-11T04:58:26.146555Z","shell.execute_reply":"2022-07-11T04:58:26.153711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['inputs'] = df.context + sep + df.anchor + sep + df.target","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:26.155955Z","iopub.execute_input":"2022-07-11T04:58:26.156464Z","iopub.status.idle":"2022-07-11T04:58:26.203309Z","shell.execute_reply.started":"2022-07-11T04:58:26.156427Z","shell.execute_reply":"2022-07-11T04:58:26.202312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = Dataset.from_pandas(df).rename_column('score', 'label')\neval_ds = Dataset.from_pandas(eval_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:26.204797Z","iopub.execute_input":"2022-07-11T04:58:26.205050Z","iopub.status.idle":"2022-07-11T04:58:26.248637Z","shell.execute_reply.started":"2022-07-11T04:58:26.205014Z","shell.execute_reply":"2022-07-11T04:58:26.247794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tok_func(x): return tokz(x[\"inputs\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:26.250009Z","iopub.execute_input":"2022-07-11T04:58:26.250237Z","iopub.status.idle":"2022-07-11T04:58:26.256587Z","shell.execute_reply.started":"2022-07-11T04:58:26.250204Z","shell.execute_reply":"2022-07-11T04:58:26.254470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tok_func(ds[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:26.257895Z","iopub.execute_input":"2022-07-11T04:58:26.258883Z","iopub.status.idle":"2022-07-11T04:58:26.270258Z","shell.execute_reply.started":"2022-07-11T04:58:26.258847Z","shell.execute_reply":"2022-07-11T04:58:26.269395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokz.all_special_tokens","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:26.271431Z","iopub.execute_input":"2022-07-11T04:58:26.272009Z","iopub.status.idle":"2022-07-11T04:58:26.281695Z","shell.execute_reply.started":"2022-07-11T04:58:26.271968Z","shell.execute_reply":"2022-07-11T04:58:26.280849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inps = \"anchor\",\"target\",\"context\"\ntok_ds = ds.map(tok_func, batched=True, remove_columns=inps+('inputs','id','section'))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:26.283323Z","iopub.execute_input":"2022-07-11T04:58:26.283967Z","iopub.status.idle":"2022-07-11T04:58:32.228247Z","shell.execute_reply.started":"2022-07-11T04:58:26.283929Z","shell.execute_reply":"2022-07-11T04:58:32.227369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tok_ds[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:32.229542Z","iopub.execute_input":"2022-07-11T04:58:32.230440Z","iopub.status.idle":"2022-07-11T04:58:32.238980Z","shell.execute_reply.started":"2022-07-11T04:58:32.230401Z","shell.execute_reply":"2022-07-11T04:58:32.238043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anchors = df.anchor.unique()\nnp.random.seed(42)\nnp.random.shuffle(anchors)\n\nval_prop = 0.25\nval_sz = int(len(anchors) * val_prop)\nval_anchors = anchors[:val_sz]\n\nis_val = np.isin(df.anchor, val_anchors)\nidxs = np.arange(len(df))\nval_idxs = idxs[is_val]\ntrn_idxs = idxs[~is_val]\nlen(val_idxs), len(trn_idxs)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:32.240366Z","iopub.execute_input":"2022-07-11T04:58:32.240722Z","iopub.status.idle":"2022-07-11T04:58:32.471093Z","shell.execute_reply.started":"2022-07-11T04:58:32.240667Z","shell.execute_reply":"2022-07-11T04:58:32.470234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dds = DatasetDict({\"train\":tok_ds.select(trn_idxs),\n             \"test\": tok_ds.select(val_idxs)})","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:32.476565Z","iopub.execute_input":"2022-07-11T04:58:32.476768Z","iopub.status.idle":"2022-07-11T04:58:32.506197Z","shell.execute_reply.started":"2022-07-11T04:58:32.476743Z","shell.execute_reply":"2022-07-11T04:58:32.505323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.iloc[trn_idxs].score.mean(), df.iloc[val_idxs].score.mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:32.507686Z","iopub.execute_input":"2022-07-11T04:58:32.507939Z","iopub.status.idle":"2022-07-11T04:58:32.525860Z","shell.execute_reply.started":"2022-07-11T04:58:32.507904Z","shell.execute_reply":"2022-07-11T04:58:32.524508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1>MODEL PART</h1>","metadata":{}},{"cell_type":"code","source":"def corr(eval_pred): return {'pearson': np.corrcoef(*eval_pred)[0][1]}","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:32.528925Z","iopub.execute_input":"2022-07-11T04:58:32.529138Z","iopub.status.idle":"2022-07-11T04:58:32.534874Z","shell.execute_reply.started":"2022-07-11T04:58:32.529112Z","shell.execute_reply":"2022-07-11T04:58:32.533700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr, bs = 8e-5, 128\nwd, epochs = 0.01, 20","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:32.536634Z","iopub.execute_input":"2022-07-11T04:58:32.537075Z","iopub.status.idle":"2022-07-11T04:58:32.544085Z","shell.execute_reply.started":"2022-07-11T04:58:32.536975Z","shell.execute_reply":"2022-07-11T04:58:32.542964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_trainer(dds):\n    args = TrainingArguments('outputs', learning_rate=lr, warmup_ratio=0.1, lr_scheduler_type='cosine', fp16=True,\n        evaluation_strategy=\"epoch\", per_device_train_batch_size=bs, per_device_eval_batch_size=bs*2,\n        num_train_epochs=epochs, weight_decay=wd, report_to='none')\n    model = AutoModelForSequenceClassification.from_pretrained(model_nm, num_labels=1)\n    return Trainer(model, args, train_dataset=dds['train'], eval_dataset=dds['test'],\n                   tokenizer=tokz, compute_metrics=corr)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:32.545779Z","iopub.execute_input":"2022-07-11T04:58:32.546114Z","iopub.status.idle":"2022-07-11T04:58:32.555248Z","shell.execute_reply.started":"2022-07-11T04:58:32.546072Z","shell.execute_reply":"2022-07-11T04:58:32.553620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args = TrainingArguments('outputs', learning_rate=lr, warmup_ratio=0.1, lr_scheduler_type='cosine', fp16=True,\n    evaluation_strategy=\"epoch\", per_device_train_batch_size=bs, per_device_eval_batch_size=bs*2,\n    num_train_epochs=epochs, weight_decay=wd, report_to='none', logging_steps=100)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:32.556812Z","iopub.execute_input":"2022-07-11T04:58:32.557135Z","iopub.status.idle":"2022-07-11T04:58:32.632287Z","shell.execute_reply.started":"2022-07-11T04:58:32.557094Z","shell.execute_reply":"2022-07-11T04:58:32.631386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = AutoModelForSequenceClassification.from_pretrained(model_nm, num_labels=1)\ntrainer = Trainer(model, args, train_dataset=dds['train'], eval_dataset=dds['test'],\n               tokenizer=tokz, compute_metrics=corr)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:32.633785Z","iopub.execute_input":"2022-07-11T04:58:32.634023Z","iopub.status.idle":"2022-07-11T04:58:47.355916Z","shell.execute_reply.started":"2022-07-11T04:58:32.633990Z","shell.execute_reply":"2022-07-11T04:58:47.354985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:47.358147Z","iopub.execute_input":"2022-07-11T04:58:47.358744Z","iopub.status.idle":"2022-07-11T04:58:47.365753Z","shell.execute_reply.started":"2022-07-11T04:58:47.358704Z","shell.execute_reply":"2022-07-11T04:58:47.364879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = trainer.train();","metadata":{"execution":{"iopub.status.busy":"2022-07-11T04:58:47.366972Z","iopub.execute_input":"2022-07-11T04:58:47.367498Z","iopub.status.idle":"2022-07-11T05:19:29.393033Z","shell.execute_reply.started":"2022-07-11T04:58:47.367460Z","shell.execute_reply":"2022-07-11T05:19:29.392032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.compute_loss","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:23:02.715297Z","iopub.execute_input":"2022-07-11T05:23:02.715756Z","iopub.status.idle":"2022-07-11T05:23:02.725610Z","shell.execute_reply.started":"2022-07-11T05:23:02.715706Z","shell.execute_reply":"2022-07-11T05:23:02.724328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}