{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-11T05:25:44.740953Z","iopub.execute_input":"2022-07-11T05:25:44.74181Z","iopub.status.idle":"2022-07-11T05:25:44.766034Z","shell.execute_reply.started":"2022-07-11T05:25:44.741706Z","shell.execute_reply":"2022-07-11T05:25:44.76504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport os\n\niskaggle = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '')\nif iskaggle:\n    !pip install -Uqq fastai\nelse:\n    import zipfile,kaggle\n    path = Path('us-patent-phrase-to-phrase-matching')\n    kaggle.api.competition_download_cli(str(path))\n    zipfile.ZipFile(f'{path}.zip').extractall(path)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:25:44.767941Z","iopub.execute_input":"2022-07-11T05:25:44.768574Z","iopub.status.idle":"2022-07-11T05:25:59.53395Z","shell.execute_reply.started":"2022-07-11T05:25:44.768539Z","shell.execute_reply":"2022-07-11T05:25:59.532691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.imports import *","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:25:59.536104Z","iopub.execute_input":"2022-07-11T05:25:59.536519Z","iopub.status.idle":"2022-07-11T05:25:59.668665Z","shell.execute_reply.started":"2022-07-11T05:25:59.536478Z","shell.execute_reply":"2022-07-11T05:25:59.667591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if iskaggle: path = Path('../input/us-patent-phrase-to-phrase-matching')\npath.ls()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:25:59.669882Z","iopub.execute_input":"2022-07-11T05:25:59.670591Z","iopub.status.idle":"2022-07-11T05:25:59.684405Z","shell.execute_reply.started":"2022-07-11T05:25:59.670553Z","shell.execute_reply":"2022-07-11T05:25:59.683498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')\ndf","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:25:59.690153Z","iopub.execute_input":"2022-07-11T05:25:59.692657Z","iopub.status.idle":"2022-07-11T05:25:59.807133Z","shell.execute_reply.started":"2022-07-11T05:25:59.692622Z","shell.execute_reply":"2022-07-11T05:25:59.806249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df = pd.read_csv(path/'test.csv')\nlen(eval_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:25:59.808649Z","iopub.execute_input":"2022-07-11T05:25:59.809004Z","iopub.status.idle":"2022-07-11T05:25:59.822709Z","shell.execute_reply.started":"2022-07-11T05:25:59.808968Z","shell.execute_reply":"2022-07-11T05:25:59.821734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:25:59.824018Z","iopub.execute_input":"2022-07-11T05:25:59.824453Z","iopub.status.idle":"2022-07-11T05:25:59.835234Z","shell.execute_reply.started":"2022-07-11T05:25:59.824399Z","shell.execute_reply":"2022-07-11T05:25:59.833988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:25:59.837134Z","iopub.execute_input":"2022-07-11T05:25:59.837963Z","iopub.status.idle":"2022-07-11T05:25:59.868594Z","shell.execute_reply.started":"2022-07-11T05:25:59.837928Z","shell.execute_reply":"2022-07-11T05:25:59.867615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.anchor.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:25:59.870015Z","iopub.execute_input":"2022-07-11T05:25:59.870435Z","iopub.status.idle":"2022-07-11T05:25:59.884878Z","shell.execute_reply.started":"2022-07-11T05:25:59.870399Z","shell.execute_reply":"2022-07-11T05:25:59.883216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.context.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:25:59.886502Z","iopub.execute_input":"2022-07-11T05:25:59.887069Z","iopub.status.idle":"2022-07-11T05:25:59.899588Z","shell.execute_reply.started":"2022-07-11T05:25:59.887036Z","shell.execute_reply":"2022-07-11T05:25:59.898686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['section'] = df.context.str[0]\ndf.section.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:25:59.904198Z","iopub.execute_input":"2022-07-11T05:25:59.905873Z","iopub.status.idle":"2022-07-11T05:25:59.9484Z","shell.execute_reply.started":"2022-07-11T05:25:59.905848Z","shell.execute_reply":"2022-07-11T05:25:59.947576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.score.hist();","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:25:59.951388Z","iopub.execute_input":"2022-07-11T05:25:59.951955Z","iopub.status.idle":"2022-07-11T05:26:00.169247Z","shell.execute_reply.started":"2022-07-11T05:25:59.951918Z","shell.execute_reply":"2022-07-11T05:26:00.168326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df.score==1]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:00.170757Z","iopub.execute_input":"2022-07-11T05:26:00.171329Z","iopub.status.idle":"2022-07-11T05:26:00.198226Z","shell.execute_reply.started":"2022-07-11T05:26:00.17129Z","shell.execute_reply":"2022-07-11T05:26:00.197411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changedict = {\"A\": \"Human Necessities\",\n             \"B\": \"Performing Operations; Transporting\",\n             \"C\": \"Chemistry; Metallurgy\", \n             \"D\": \"Textiles; Paper\",\n             \"E\": \"Fixed Constructions\",\n             \"F\": \"Mechanical Engineering; Lighting; Heating; Weapons; Blasting\",\n             \"G\": \"Physics\",\n             \"H\": \"Electricity\"}\ndf = df.replace({\"section\": changedict})\ndf['number'] = df.context.str[1:]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:00.200397Z","iopub.execute_input":"2022-07-11T05:26:00.200975Z","iopub.status.idle":"2022-07-11T05:26:00.245248Z","shell.execute_reply.started":"2022-07-11T05:26:00.200938Z","shell.execute_reply":"2022-07-11T05:26:00.244369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader\nimport warnings,transformers,logging,torch\nfrom transformers import TrainingArguments,Trainer\nfrom transformers import AutoModelForSequenceClassification,AutoTokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:00.246717Z","iopub.execute_input":"2022-07-11T05:26:00.247062Z","iopub.status.idle":"2022-07-11T05:26:07.474804Z","shell.execute_reply.started":"2022-07-11T05:26:00.247027Z","shell.execute_reply":"2022-07-11T05:26:07.47385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if iskaggle:\n    !pip install -q datasets\nimport datasets\nfrom datasets import load_dataset, Dataset, DatasetDict","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:07.476333Z","iopub.execute_input":"2022-07-11T05:26:07.477257Z","iopub.status.idle":"2022-07-11T05:26:17.306101Z","shell.execute_reply.started":"2022-07-11T05:26:07.477219Z","shell.execute_reply":"2022-07-11T05:26:17.304968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.simplefilter('ignore')\nlogging.disable(logging.WARNING)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:17.307878Z","iopub.execute_input":"2022-07-11T05:26:17.308579Z","iopub.status.idle":"2022-07-11T05:26:17.315421Z","shell.execute_reply.started":"2022-07-11T05:26:17.30854Z","shell.execute_reply":"2022-07-11T05:26:17.314421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_nm = 'microsoft/deberta-v3-small'","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:17.317178Z","iopub.execute_input":"2022-07-11T05:26:17.317829Z","iopub.status.idle":"2022-07-11T05:26:17.329683Z","shell.execute_reply.started":"2022-07-11T05:26:17.317792Z","shell.execute_reply":"2022-07-11T05:26:17.328681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokz = AutoTokenizer.from_pretrained(model_nm)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:17.332038Z","iopub.execute_input":"2022-07-11T05:26:17.333403Z","iopub.status.idle":"2022-07-11T05:26:29.214117Z","shell.execute_reply.started":"2022-07-11T05:26:17.333356Z","shell.execute_reply":"2022-07-11T05:26:29.213217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sep = tokz.sep_token\n\ndf['inputs'] = df.section + sep + df.number + sep + df.anchor + sep + df.target","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:29.215422Z","iopub.execute_input":"2022-07-11T05:26:29.215681Z","iopub.status.idle":"2022-07-11T05:26:29.253899Z","shell.execute_reply.started":"2022-07-11T05:26:29.215645Z","shell.execute_reply":"2022-07-11T05:26:29.25307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = Dataset.from_pandas(df).rename_column('score', 'label')\neval_ds = Dataset.from_pandas(eval_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:29.255318Z","iopub.execute_input":"2022-07-11T05:26:29.255579Z","iopub.status.idle":"2022-07-11T05:26:29.301348Z","shell.execute_reply.started":"2022-07-11T05:26:29.255545Z","shell.execute_reply":"2022-07-11T05:26:29.300456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tok_func(x): return tokz(x[\"inputs\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:29.302902Z","iopub.execute_input":"2022-07-11T05:26:29.303427Z","iopub.status.idle":"2022-07-11T05:26:29.308386Z","shell.execute_reply.started":"2022-07-11T05:26:29.30339Z","shell.execute_reply":"2022-07-11T05:26:29.307481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tok_func(ds[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:29.309996Z","iopub.execute_input":"2022-07-11T05:26:29.310313Z","iopub.status.idle":"2022-07-11T05:26:29.321597Z","shell.execute_reply.started":"2022-07-11T05:26:29.310279Z","shell.execute_reply":"2022-07-11T05:26:29.320477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokz.all_special_tokens","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:29.323337Z","iopub.execute_input":"2022-07-11T05:26:29.323775Z","iopub.status.idle":"2022-07-11T05:26:29.331747Z","shell.execute_reply.started":"2022-07-11T05:26:29.323738Z","shell.execute_reply":"2022-07-11T05:26:29.330766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inps = \"anchor\",\"target\",\"context\"\ntok_ds = ds.map(tok_func, batched=True, remove_columns=inps+('inputs','id','section'))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:29.333156Z","iopub.execute_input":"2022-07-11T05:26:29.333821Z","iopub.status.idle":"2022-07-11T05:26:36.76303Z","shell.execute_reply.started":"2022-07-11T05:26:29.333785Z","shell.execute_reply":"2022-07-11T05:26:36.762157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tok_ds[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:36.764295Z","iopub.execute_input":"2022-07-11T05:26:36.765122Z","iopub.status.idle":"2022-07-11T05:26:36.77395Z","shell.execute_reply.started":"2022-07-11T05:26:36.765081Z","shell.execute_reply":"2022-07-11T05:26:36.772902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anchors = df.anchor.unique()\nprint(len(anchors))\nnp.random.seed(42)\nnp.random.shuffle(anchors)\nanchors[:5]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:36.775545Z","iopub.execute_input":"2022-07-11T05:26:36.776005Z","iopub.status.idle":"2022-07-11T05:26:36.790325Z","shell.execute_reply.started":"2022-07-11T05:26:36.775963Z","shell.execute_reply":"2022-07-11T05:26:36.789363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_prop = 0.25\nval_sz = int(len(anchors)*val_prop)\nval_anchors = anchors[:val_sz]\n\nis_val = np.isin(df.anchor, val_anchors)\nidxs = np.arange(len(df))\nval_idxs = idxs[ is_val]\ntrn_idxs = idxs[~is_val]\nlen(val_idxs),len(trn_idxs)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:36.791811Z","iopub.execute_input":"2022-07-11T05:26:36.792067Z","iopub.status.idle":"2022-07-11T05:26:37.019493Z","shell.execute_reply.started":"2022-07-11T05:26:36.792031Z","shell.execute_reply":"2022-07-11T05:26:37.018679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dds = DatasetDict({\"train\":tok_ds.select(trn_idxs),\n             \"test\": tok_ds.select(val_idxs)})","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:37.025816Z","iopub.execute_input":"2022-07-11T05:26:37.026014Z","iopub.status.idle":"2022-07-11T05:26:37.049174Z","shell.execute_reply.started":"2022-07-11T05:26:37.025988Z","shell.execute_reply":"2022-07-11T05:26:37.048285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.iloc[trn_idxs].score.mean(), df.iloc[val_idxs].score.mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:37.050554Z","iopub.execute_input":"2022-07-11T05:26:37.050783Z","iopub.status.idle":"2022-07-11T05:26:37.073121Z","shell.execute_reply.started":"2022-07-11T05:26:37.05075Z","shell.execute_reply":"2022-07-11T05:26:37.072209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def corr(eval_pred): return {'pearson': np.corrcoef(*eval_pred)[0][1]}","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:37.074534Z","iopub.execute_input":"2022-07-11T05:26:37.074797Z","iopub.status.idle":"2022-07-11T05:26:37.08094Z","shell.execute_reply.started":"2022-07-11T05:26:37.074761Z","shell.execute_reply":"2022-07-11T05:26:37.079758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr, bs = 8e-5, 128\nwd, epochs = 0.01, 15","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:37.082533Z","iopub.execute_input":"2022-07-11T05:26:37.082846Z","iopub.status.idle":"2022-07-11T05:26:37.091019Z","shell.execute_reply.started":"2022-07-11T05:26:37.082798Z","shell.execute_reply":"2022-07-11T05:26:37.089876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args = TrainingArguments('outputs', learning_rate=lr, warmup_ratio=0.1, lr_scheduler_type='cosine', fp16=True,\n    evaluation_strategy=\"epoch\", per_device_train_batch_size=bs, per_device_eval_batch_size=bs*2,\n    num_train_epochs=epochs, weight_decay=wd, report_to='none', logging_steps=100)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:37.092672Z","iopub.execute_input":"2022-07-11T05:26:37.093093Z","iopub.status.idle":"2022-07-11T05:26:37.154679Z","shell.execute_reply.started":"2022-07-11T05:26:37.092914Z","shell.execute_reply":"2022-07-11T05:26:37.153801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = AutoModelForSequenceClassification.from_pretrained(model_nm, num_labels=1)\ntrainer = Trainer(model, args, train_dataset=dds['train'], eval_dataset=dds['test'],\n               tokenizer=tokz, compute_metrics=corr)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:26:37.156257Z","iopub.execute_input":"2022-07-11T05:26:37.156732Z","iopub.status.idle":"2022-07-11T05:27:05.546968Z","shell.execute_reply.started":"2022-07-11T05:26:37.156695Z","shell.execute_reply":"2022-07-11T05:27:05.545992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = trainer.train();","metadata":{"execution":{"iopub.status.busy":"2022-07-11T05:27:05.548586Z","iopub.execute_input":"2022-07-11T05:27:05.548842Z","iopub.status.idle":"2022-07-11T05:43:54.468232Z","shell.execute_reply.started":"2022-07-11T05:27:05.548805Z","shell.execute_reply":"2022-07-11T05:43:54.467212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}