{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-18T21:15:53.886274Z","iopub.execute_input":"2022-07-18T21:15:53.886782Z","iopub.status.idle":"2022-07-18T21:15:54.165175Z","shell.execute_reply.started":"2022-07-18T21:15:53.886656Z","shell.execute_reply":"2022-07-18T21:15:54.164087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nfrom fastai.imports import *\nimport os\n\niskaggle = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '')\n\npath = Path('us-patent-phrase-to-phrase-matching')","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:15:54.166947Z","iopub.execute_input":"2022-07-18T21:15:54.167792Z","iopub.status.idle":"2022-07-18T21:15:54.362485Z","shell.execute_reply.started":"2022-07-18T21:15:54.167751Z","shell.execute_reply":"2022-07-18T21:15:54.361000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if iskaggle: path = Path('../input/us-patent-phrase-to-phrase-matching')\npath.ls()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:15:54.364254Z","iopub.execute_input":"2022-07-18T21:15:54.364879Z","iopub.status.idle":"2022-07-18T21:15:54.377734Z","shell.execute_reply.started":"2022-07-18T21:15:54.364834Z","shell.execute_reply":"2022-07-18T21:15:54.376702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_cpc_texts():\n    contexts = []\n    pattern = '[A-Z]\\d+'\n    for file_name in os.listdir('../input/cpc-data/CPCSchemeXML202105'):\n        result = re.findall(pattern, file_name)\n        if result:\n            contexts.append(result)\n    contexts = sorted(set(sum(contexts, [])))\n    results = {}\n    for cpc in ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'Y']:\n        with open(f'../input/cpc-data/CPCTitleList202202/cpc-section-{cpc}_20220201.txt') as f:\n            s = f.read()\n        pattern = f'{cpc}\\t\\t.+'\n        result = re.findall(pattern, s)\n        cpc_result = result[0].lstrip(pattern)\n        for context in [c for c in contexts if c[0] == cpc]:\n            pattern = f'{context}\\t\\t.+'\n            result = re.findall(pattern, s)\n            results[context] = cpc_result + \". \" + result[0].lstrip(pattern)\n    return results\n\ncpc_texts = get_cpc_texts()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:15:54.380507Z","iopub.execute_input":"2022-07-18T21:15:54.381084Z","iopub.status.idle":"2022-07-18T21:15:55.139197Z","shell.execute_reply.started":"2022-07-18T21:15:54.381048Z","shell.execute_reply":"2022-07-18T21:15:55.138158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')\ndf_eval = pd.read_csv(path/'test.csv')\neval_df = pd.read_csv(path/'test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:15:55.142918Z","iopub.execute_input":"2022-07-18T21:15:55.143205Z","iopub.status.idle":"2022-07-18T21:15:55.243062Z","shell.execute_reply.started":"2022-07-18T21:15:55.143180Z","shell.execute_reply":"2022-07-18T21:15:55.241966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['section'] = df.context.str[0]\ndf_eval[\"section\"] = df_eval.context.str[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:15:55.244258Z","iopub.execute_input":"2022-07-18T21:15:55.244588Z","iopub.status.idle":"2022-07-18T21:15:55.288444Z","shell.execute_reply.started":"2022-07-18T21:15:55.244554Z","shell.execute_reply":"2022-07-18T21:15:55.287453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['context_text'] = df['context'].map(cpc_texts).apply(lambda x:x.lower())\ndf_eval['context_text'] = df_eval['context'].map(cpc_texts).apply(lambda x:x.lower())\n\ndf = df.join(df.groupby('anchor').target.agg(list).rename('ref'), on='anchor')\ndf['ref'] = df.apply(lambda x:[i for i in x['ref']], axis=1)\ndf['ref'] = df.ref.apply(lambda x:', '.join(sorted(list(set(x)), key=x.index)))\n\ndf_eval = df_eval.join(df_eval.groupby('anchor').target.agg(list).rename('ref'), on='anchor')\ndf_eval['ref'] = df_eval.apply(lambda x:[i for i in x['ref']], axis=1)\ndf_eval['ref'] = df_eval.ref.apply(lambda x:', '.join(sorted(list(set(x)), key=x.index)))","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:15:55.293496Z","iopub.execute_input":"2022-07-18T21:15:55.295084Z","iopub.status.idle":"2022-07-18T21:15:57.685283Z","shell.execute_reply.started":"2022-07-18T21:15:55.295046Z","shell.execute_reply":"2022-07-18T21:15:57.684321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:15:57.686769Z","iopub.execute_input":"2022-07-18T21:15:57.687123Z","iopub.status.idle":"2022-07-18T21:15:57.707484Z","shell.execute_reply.started":"2022-07-18T21:15:57.687085Z","shell.execute_reply":"2022-07-18T21:15:57.706698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_eval.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:15:57.708718Z","iopub.execute_input":"2022-07-18T21:15:57.709248Z","iopub.status.idle":"2022-07-18T21:15:57.722734Z","shell.execute_reply.started":"2022-07-18T21:15:57.709212Z","shell.execute_reply":"2022-07-18T21:15:57.721637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader\nimport warnings, transformers, logging, torch\nfrom transformers import TrainingArguments, Trainer\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:15:57.724602Z","iopub.execute_input":"2022-07-18T21:15:57.725366Z","iopub.status.idle":"2022-07-18T21:16:05.179959Z","shell.execute_reply.started":"2022-07-18T21:15:57.725332Z","shell.execute_reply":"2022-07-18T21:16:05.179024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model_nm = 'microsoft/deberta-v3-base'\n#tokz = AutoTokenizer.from_pretrained(\"/kaggle/input/debertabasepatent1000\")\ntokz = AutoTokenizer.from_pretrained(\"microsoft/deberta-v3-base\")","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:16:05.181363Z","iopub.execute_input":"2022-07-18T21:16:05.181992Z","iopub.status.idle":"2022-07-18T21:16:12.679856Z","shell.execute_reply.started":"2022-07-18T21:16:05.181956Z","shell.execute_reply":"2022-07-18T21:16:12.678792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sectoks = list(df.sectok.unique()) #\n# tokz.add_special_tokens({'additional_special_tokens': sectoks}) #","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:16:12.681185Z","iopub.execute_input":"2022-07-18T21:16:12.681768Z","iopub.status.idle":"2022-07-18T21:16:12.687681Z","shell.execute_reply.started":"2022-07-18T21:16:12.681730Z","shell.execute_reply":"2022-07-18T21:16:12.686644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if iskaggle:\n    !pip install -q datasets\nimport datasets\nfrom datasets import load_dataset, Dataset, DatasetDict","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:16:12.689022Z","iopub.execute_input":"2022-07-18T21:16:12.689432Z","iopub.status.idle":"2022-07-18T21:16:23.235115Z","shell.execute_reply.started":"2022-07-18T21:16:12.689370Z","shell.execute_reply":"2022-07-18T21:16:23.233993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.simplefilter('ignore')\nlogging.disable(logging.WARNING)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:16:23.237776Z","iopub.execute_input":"2022-07-18T21:16:23.238642Z","iopub.status.idle":"2022-07-18T21:16:23.244788Z","shell.execute_reply.started":"2022-07-18T21:16:23.238592Z","shell.execute_reply":"2022-07-18T21:16:23.243810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df[\"section\"] = df[\"section\"].str.lower()\n#df_eval[\"section\"] = df_eval[\"section\"].str.lower()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:16:23.250155Z","iopub.execute_input":"2022-07-18T21:16:23.250917Z","iopub.status.idle":"2022-07-18T21:16:23.255838Z","shell.execute_reply.started":"2022-07-18T21:16:23.250888Z","shell.execute_reply":"2022-07-18T21:16:23.254964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:16:23.261044Z","iopub.execute_input":"2022-07-18T21:16:23.261638Z","iopub.status.idle":"2022-07-18T21:16:23.278068Z","shell.execute_reply.started":"2022-07-18T21:16:23.261598Z","shell.execute_reply":"2022-07-18T21:16:23.277043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_eval.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:16:23.279518Z","iopub.execute_input":"2022-07-18T21:16:23.280510Z","iopub.status.idle":"2022-07-18T21:16:23.294682Z","shell.execute_reply.started":"2022-07-18T21:16:23.280473Z","shell.execute_reply":"2022-07-18T21:16:23.293584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sep = tokz.sep_token\n# df_eval['inputs'] = df_eval.anchor + sep + df_eval.target + sep + df_eval.context_text\n# df[\"inputs\"] = df.anchor + sep + df.target + sep + df.context_text\ndf_eval['inputs'] = df_eval.anchor + sep + df_eval.target + sep + df_eval.context_text + sep + df_eval.ref\ndf[\"inputs\"] = df.anchor + sep + df.target + sep + df.context_text + sep + df.ref","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:16:23.296323Z","iopub.execute_input":"2022-07-18T21:16:23.296998Z","iopub.status.idle":"2022-07-18T21:16:23.369046Z","shell.execute_reply.started":"2022-07-18T21:16:23.296955Z","shell.execute_reply":"2022-07-18T21:16:23.368165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = Dataset.from_pandas(df).rename_column('score', 'label')\neval_ds = Dataset.from_pandas(df_eval)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:16:23.370318Z","iopub.execute_input":"2022-07-18T21:16:23.370566Z","iopub.status.idle":"2022-07-18T21:16:23.509372Z","shell.execute_reply.started":"2022-07-18T21:16:23.370533Z","shell.execute_reply":"2022-07-18T21:16:23.508536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tok_func(x): return tokz(x[\"inputs\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:16:23.511566Z","iopub.execute_input":"2022-07-18T21:16:23.512182Z","iopub.status.idle":"2022-07-18T21:16:23.518622Z","shell.execute_reply.started":"2022-07-18T21:16:23.512128Z","shell.execute_reply":"2022-07-18T21:16:23.517794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inps = \"anchor\",\"target\",\"context\"\ntok_ds = ds.map(tok_func, batched=True, remove_columns=inps+('inputs','id','section'))\ntok_ds_eval = eval_ds.map(tok_func, batched=True, remove_columns=inps+('inputs', \"id\", \"section\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:16:23.520305Z","iopub.execute_input":"2022-07-18T21:16:23.520878Z","iopub.status.idle":"2022-07-18T21:17:25.394547Z","shell.execute_reply.started":"2022-07-18T21:16:23.520843Z","shell.execute_reply":"2022-07-18T21:17:25.393703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anchors = df.anchor.unique()\nprint(len(anchors))\nnp.random.seed(42)\nnp.random.shuffle(anchors)\nanchors[:5]","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:17:25.396035Z","iopub.execute_input":"2022-07-18T21:17:25.396516Z","iopub.status.idle":"2022-07-18T21:17:25.409403Z","shell.execute_reply.started":"2022-07-18T21:17:25.396478Z","shell.execute_reply":"2022-07-18T21:17:25.408303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_prop = 0.25\nval_sz = int(len(anchors)*val_prop)\nval_anchors = anchors[:val_sz]\n\nis_val = np.isin(df.anchor, val_anchors)\nidxs = np.arange(len(df))\nval_idxs = idxs[ is_val]\ntrn_idxs = idxs[~is_val]\nlen(val_idxs),len(trn_idxs)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:17:25.410876Z","iopub.execute_input":"2022-07-18T21:17:25.411429Z","iopub.status.idle":"2022-07-18T21:17:25.637719Z","shell.execute_reply.started":"2022-07-18T21:17:25.411386Z","shell.execute_reply":"2022-07-18T21:17:25.636728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dds = DatasetDict({\"train\":tok_ds.select(trn_idxs),\n             \"test\": tok_ds.select(val_idxs)})\ndds_eval = DatasetDict({\"eval\":tok_ds_eval})","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:17:25.639391Z","iopub.execute_input":"2022-07-18T21:17:25.639676Z","iopub.status.idle":"2022-07-18T21:17:25.661466Z","shell.execute_reply.started":"2022-07-18T21:17:25.639628Z","shell.execute_reply":"2022-07-18T21:17:25.660700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.iloc[trn_idxs].score.mean(), df.iloc[val_idxs].score.mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:17:25.662635Z","iopub.execute_input":"2022-07-18T21:17:25.663080Z","iopub.status.idle":"2022-07-18T21:17:25.704514Z","shell.execute_reply.started":"2022-07-18T21:17:25.663041Z","shell.execute_reply":"2022-07-18T21:17:25.703617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def corr(eval_pred): return {'pearson': np.corrcoef(*eval_pred)[0][1]}","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:17:25.705903Z","iopub.execute_input":"2022-07-18T21:17:25.706141Z","iopub.status.idle":"2022-07-18T21:17:25.712649Z","shell.execute_reply.started":"2022-07-18T21:17:25.706107Z","shell.execute_reply":"2022-07-18T21:17:25.710522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr, bs = 8e-6, 16\nwd, epochs = 0.01, 10","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:17:25.714321Z","iopub.execute_input":"2022-07-18T21:17:25.714773Z","iopub.status.idle":"2022-07-18T21:17:25.721932Z","shell.execute_reply.started":"2022-07-18T21:17:25.714735Z","shell.execute_reply":"2022-07-18T21:17:25.719641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args = TrainingArguments('outputs', learning_rate=lr, warmup_ratio=0.1, lr_scheduler_type='cosine', fp16=True,\n   evaluation_strategy=\"epoch\", per_device_train_batch_size=bs, per_device_eval_batch_size=bs*2,\n   num_train_epochs=epochs, weight_decay=wd, report_to='none', logging_steps=100, \n   save_strategy=\"no\")","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:17:25.723637Z","iopub.execute_input":"2022-07-18T21:17:25.723967Z","iopub.status.idle":"2022-07-18T21:17:25.795638Z","shell.execute_reply.started":"2022-07-18T21:17:25.723925Z","shell.execute_reply":"2022-07-18T21:17:25.794812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model = AutoModelForSequenceClassification.from_pretrained(\"/kaggle/input/debertabasepatent1000\", num_labels=1)\nmodel = AutoModelForSequenceClassification.from_pretrained(\"microsoft/deberta-v3-base\", num_labels=1)\ntrainer = Trainer(model, args, train_dataset=dds['train'], eval_dataset=dds['test'],\n              tokenizer=tokz, compute_metrics=corr)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:17:25.796996Z","iopub.execute_input":"2022-07-18T21:17:25.797224Z","iopub.status.idle":"2022-07-18T21:18:07.964595Z","shell.execute_reply.started":"2022-07-18T21:17:25.797191Z","shell.execute_reply":"2022-07-18T21:18:07.963726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = trainer.train();","metadata":{"execution":{"iopub.status.busy":"2022-07-18T21:18:07.966110Z","iopub.execute_input":"2022-07-18T21:18:07.966454Z","iopub.status.idle":"2022-07-19T02:04:15.645066Z","shell.execute_reply.started":"2022-07-18T21:18:07.966416Z","shell.execute_reply":"2022-07-19T02:04:15.644009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}