{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook original autor is from https://www.kaggle.com/yuanzhezhou just changed couple of things on the optimizer and training loop, but getting very low score 0.75, any hints will be apreciated. thanks.","metadata":{}},{"cell_type":"markdown","source":"# Pairwise training\nIn this notebook, we demonstrate how to use pairwise model to predict in this competition. Please note that the inference time is much longer than pointwise method or using cosine similarity. \n\n1. I used a bert-small model pretrained with pairwise-mlm.\n2. **Training with pairwise examples with negative samples randomly sampled.**\n3. Inference and predict for all the pairs for test dataset.\n\n* [Pretrain](https://www.kaggle.com/code/yuanzhezhou/ai4code-pairwise-bertsmall-pretrain/notebook)\n* [Training](https://www.kaggle.com/yuanzhezhou/ai4code-pairwise-bertsmall-training)\n* [Inference](https://www.kaggle.com/yuanzhezhou/ai4code-pairwise-bertsmall-inference)","metadata":{"papermill":{"duration":0.023717,"end_time":"2022-05-22T16:43:13.442675","exception":false,"start_time":"2022-05-22T16:43:13.418958","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import sparse\nfrom tqdm import tqdm\nimport os\n\nos.environ[\"WANDB_DISABLED\"] = \"true\"\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\npd.options.display.width = 180\npd.options.display.max_colwidth = 120\n\n# BERT_PATH = \"../input/huggingface-bert-variants/distilbert-base-uncased/distilbert-base-uncased\"\nBERT_PATH = \"distilbert-base-uncased\"\n\ndata_dir = Path('../input/AI4Code')","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:13.489649Z","iopub.status.busy":"2022-05-22T16:43:13.488935Z","iopub.status.idle":"2022-05-22T16:43:13.563122Z","shell.execute_reply":"2022-05-22T16:43:13.562416Z"},"papermill":{"duration":0.100106,"end_time":"2022-05-22T16:43:13.565147","exception":false,"start_time":"2022-05-22T16:43:13.465041","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAIN = 200\n\n\ndef read_notebook(path):\n    return (\n        pd.read_json(\n            path,\n            dtype={'cell_type': 'category', 'source': 'str'})\n        .assign(id=path.stem)\n        .rename_axis('cell_id')\n    )\n\n\npaths_train = list((data_dir / 'train').glob('*.json'))#[:NUM_TRAIN]\nnotebooks_train = [\n    read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')\n]\ndf = (\n    pd.concat(notebooks_train)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)\n\n# df","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:13.612833Z","iopub.status.busy":"2022-05-22T16:43:13.612178Z","iopub.status.idle":"2022-05-22T16:43:18.361949Z","shell.execute_reply":"2022-05-22T16:43:18.361218Z"},"papermill":{"duration":4.77604,"end_time":"2022-05-22T16:43:18.363886","exception":false,"start_time":"2022-05-22T16:43:13.587846","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get an example notebook\nnb_id = df.index.unique('id')[6]\nprint('Notebook:', nb_id)\n\nprint(\"The disordered notebook:\")\nnb = df.loc[nb_id, :]\n# display(nb)\n# print()","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:18.424052Z","iopub.status.busy":"2022-05-22T16:43:18.423806Z","iopub.status.idle":"2022-05-22T16:43:18.438847Z","shell.execute_reply":"2022-05-22T16:43:18.437923Z"},"papermill":{"duration":0.049053,"end_time":"2022-05-22T16:43:18.442943","exception":false,"start_time":"2022-05-22T16:43:18.393890","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders = pd.read_csv(\n    data_dir / 'train_orders.csv',\n    index_col='id',\n    squeeze=True,\n).str.split()  # Split the string representation of cell_ids into a list\n\n# df_orders","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:18.507080Z","iopub.status.busy":"2022-05-22T16:43:18.506548Z","iopub.status.idle":"2022-05-22T16:43:20.863860Z","shell.execute_reply":"2022-05-22T16:43:20.863194Z"},"papermill":{"duration":2.39277,"end_time":"2022-05-22T16:43:20.865898","exception":false,"start_time":"2022-05-22T16:43:18.473128","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_orders.loc[\"002ba502bdac45\"])","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:20.928431Z","iopub.status.busy":"2022-05-22T16:43:20.928203Z","iopub.status.idle":"2022-05-22T16:43:20.935823Z","shell.execute_reply":"2022-05-22T16:43:20.935000Z"},"papermill":{"duration":0.042145,"end_time":"2022-05-22T16:43:20.938460","exception":false,"start_time":"2022-05-22T16:43:20.896315","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_order = df_orders.loc[nb_id]\n\nprint(\"The ordered notebook:\")\nnb.loc[cell_order, :]","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:21.004762Z","iopub.status.busy":"2022-05-22T16:43:21.004554Z","iopub.status.idle":"2022-05-22T16:43:21.020436Z","shell.execute_reply":"2022-05-22T16:43:21.019619Z"},"papermill":{"duration":0.051122,"end_time":"2022-05-22T16:43:21.022235","exception":false,"start_time":"2022-05-22T16:43:20.971113","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_ranks(base, derived):\n    return [base.index(d) for d in derived]\n\ncell_ranks = get_ranks(cell_order, list(nb.index))\nnb.insert(0, 'rank', cell_ranks)\n\n# nb","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:21.090255Z","iopub.status.busy":"2022-05-22T16:43:21.090032Z","iopub.status.idle":"2022-05-22T16:43:21.104980Z","shell.execute_reply":"2022-05-22T16:43:21.104155Z"},"papermill":{"duration":0.050619,"end_time":"2022-05-22T16:43:21.106751","exception":false,"start_time":"2022-05-22T16:43:21.056132","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders_ = df_orders.to_frame().join(\n    df.reset_index('cell_id').groupby('id')['cell_id'].apply(list),\n    how='right',\n)\n\nranks = {}\nfor id_, cell_order, cell_id in df_orders_.itertuples():\n    ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}\n\ndf_ranks = (\n    pd.DataFrame\n    .from_dict(ranks, orient='index')\n    .rename_axis('id')\n    .apply(pd.Series.explode)\n    .set_index('cell_id', append=True)\n)\n\n# df_ranks","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:21.176642Z","iopub.status.busy":"2022-05-22T16:43:21.176248Z","iopub.status.idle":"2022-05-22T16:43:21.269731Z","shell.execute_reply":"2022-05-22T16:43:21.268764Z"},"papermill":{"duration":0.131243,"end_time":"2022-05-22T16:43:21.272137","exception":false,"start_time":"2022-05-22T16:43:21.140894","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ancestors = pd.read_csv(data_dir / 'train_ancestors.csv', index_col='id')\n# df_ancestors","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:21.344849Z","iopub.status.busy":"2022-05-22T16:43:21.344539Z","iopub.status.idle":"2022-05-22T16:43:21.541734Z","shell.execute_reply":"2022-05-22T16:43:21.541035Z"},"papermill":{"duration":0.234618,"end_time":"2022-05-22T16:43:21.543715","exception":false,"start_time":"2022-05-22T16:43:21.309097","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.reset_index().merge(df_ranks, on=[\"id\", \"cell_id\"]).merge(df_ancestors, on=[\"id\"])\n# df","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:21.611297Z","iopub.status.busy":"2022-05-22T16:43:21.611074Z","iopub.status.idle":"2022-05-22T16:43:21.675022Z","shell.execute_reply":"2022-05-22T16:43:21.674257Z"},"papermill":{"duration":0.099547,"end_time":"2022-05-22T16:43:21.676797","exception":false,"start_time":"2022-05-22T16:43:21.577250","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"pct_rank\"] = df[\"rank\"] / df.groupby(\"id\")[\"cell_id\"].transform(\"count\")\n\ndf[\"pct_rank\"].hist(bins=10)","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:21.745847Z","iopub.status.busy":"2022-05-22T16:43:21.745391Z","iopub.status.idle":"2022-05-22T16:43:21.979061Z","shell.execute_reply":"2022-05-22T16:43:21.978396Z"},"papermill":{"duration":0.270151,"end_time":"2022-05-22T16:43:21.980878","exception":false,"start_time":"2022-05-22T16:43:21.710727","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_cellid_source = dict(zip(df['cell_id'].values, df['source'].values))","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:22.050944Z","iopub.status.busy":"2022-05-22T16:43:22.050749Z","iopub.status.idle":"2022-05-22T16:43:22.056285Z","shell.execute_reply":"2022-05-22T16:43:22.055546Z"},"papermill":{"duration":0.042415,"end_time":"2022-05-22T16:43:22.058131","exception":false,"start_time":"2022-05-22T16:43:22.015716","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"class Punctuation:\n    html = re.compile(r'&apos;|&quot;')\n    punctuation = re.compile(r'[^\\w\\s·]|_')\n    spaces = re.compile(r'\\s+')\n    ela_geminada = re.compile(r'l · l')\n\n    def strip(self, s):\n        '''\n        Remove all punctuation characters.\n        '''\n        s = self.html.sub(' ', s)\n        s = self.punctuation.sub(' ', s)\n        s = self.spaces.sub(' ', s).strip()\n        s = self.ela_geminada.sub('l·l', s)\n        return s\n    \npunc = Punctuation()\nwith open(input_path, 'r', encoding='utf-8') as inpf, open(output_path, 'w', encoding='utf-8') as outf:\n    for line in inpf:\n        line = punc.strip(line)\n        print(line, file=outf)    ","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\n# import fasttext\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom nltk.stem import WordNetLemmatizer\nfrom pathlib import Path\nimport nltk\nnltk.download('wordnet')\n\nstemmer = WordNetLemmatizer()\n\ndef preprocess_text(document):\n        # Remove all the special characters\n        document = re.sub(r'\\W', ' ', str(document))\n\n        # remove all single characters\n        document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n\n        # Remove single characters from the start\n        document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n\n        # Substituting multiple spaces with single space\n        document = re.sub(r'\\s+', ' ', document, flags=re.I)\n\n        # Removing prefixed 'b'\n        document = re.sub(r'^b\\s+', '', document)\n\n        # Converting to Lowercase\n        document = document.lower()\n        #return document\n\n        # Lemmatization\n        tokens = document.split()\n        tokens = [stemmer.lemmatize(word) for word in tokens]\n        tokens = [word for word in tokens if len(word) > 3]\n\n        preprocessed_text = ' '.join(tokens)\n        return preprocessed_text\n\n    \ndef preprocess_df(df):\n    \"\"\"\n    This function is for processing sorce of notebook\n    returns preprocessed dataframe\n    \"\"\"\n    return [preprocess_text(message) for message in df.source]\n\ndf.source = df.source.apply(preprocess_text)","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:22.128083Z","iopub.status.busy":"2022-05-22T16:43:22.127892Z","iopub.status.idle":"2022-05-22T16:43:27.747383Z","shell.execute_reply":"2022-05-22T16:43:27.746634Z"},"papermill":{"duration":5.657896,"end_time":"2022-05-22T16:43:27.750179","exception":false,"start_time":"2022-05-22T16:43:22.092283","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport sys, os\ntry:\n  from transformers import DistilBertModel, DistilBertTokenizer\nexcept:\n  !pip install transformers\n  from transformers import DistilBertModel, DistilBertTokenizer\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch\n\nfrom transformers import BertConfig, BertForMaskedLM, DataCollatorForLanguageModeling\nfrom transformers import AutoModelWithLMHead, AutoTokenizer, AutoModel\n\n# if not os.path.exists('text.txt'):\n#   with open('text.txt','w') as f:\n#     for id, item in tqdm(df.groupby('id')):\n#       df_markdown =  item[item['cell_type']=='markdown']\n#       for source, rank in df_markdown[['source', 'rank']].values:\n#         cell_source = df_markdown[df_markdown['rank']==(rank+1)]\n#         if len(cell_source):\n#           setence = source + ' [SEP] ' + cell_source.source.values[0]\n#           f.write(setence+'\\n')\n      \n\n# # Train a tokenizer\n# import tokenizers\n# from transformers import BertTokenizer, LineByLineTextDataset\n\n# tokenizer = AutoTokenizer.from_pretrained('prajjwal1/bert-small')","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:27.867905Z","iopub.status.busy":"2022-05-22T16:43:27.867511Z","iopub.status.idle":"2022-05-22T16:43:34.333919Z","shell.execute_reply":"2022-05-22T16:43:34.333179Z"},"papermill":{"duration":6.527429,"end_time":"2022-05-22T16:43:34.336034","exception":false,"start_time":"2022-05-22T16:43:27.808605","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = AutoModelWithLMHead.from_pretrained('prajjwal1/bert-small')\n\n\n# data_collator = DataCollatorForLanguageModeling(\n#     tokenizer=tokenizer, mlm=True, mlm_probability=0.15\n# )\n\n# from transformers import Trainer, TrainingArguments\n\n# dataset= LineByLineTextDataset(\n#     tokenizer = tokenizer,\n#     file_path = './text.txt',\n#     block_size = 128  # maximum sequence length\n# )\n\n# print('No. of lines: ', len(dataset)) # No of lines in your datset\n\n# training_args = TrainingArguments(\n#     output_dir='./',\n#     overwrite_output_dir=True,\n#     num_train_epochs=10,\n#     per_device_train_batch_size=64,\n#     save_steps=10000,\n# )\n# trainer = Trainer(\n#     model=model,\n#     args=training_args,\n#     data_collator=data_collator,\n#     train_dataset=dataset,\n# )\n# trainer.train()\n# trainer.save_model('./')","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:34.407265Z","iopub.status.busy":"2022-05-22T16:43:34.406556Z","iopub.status.idle":"2022-05-22T16:43:34.411828Z","shell.execute_reply":"2022-05-22T16:43:34.411170Z"},"papermill":{"duration":0.042389,"end_time":"2022-05-22T16:43:34.413426","exception":false,"start_time":"2022-05-22T16:43:34.371037","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_triplet(df, mode='train'):\n  triplets = []\n  ids = df.id.unique()\n  random_drop = np.random.random(size=10000)>0.9\n  count = 0\n\n  for id, df_tmp in tqdm(df.groupby('id')):\n    df_tmp_markdown = df_tmp[df_tmp['cell_type']=='markdown']\n\n    df_tmp_code = df_tmp[df_tmp['cell_type']=='code']\n    df_tmp_code_rank = df_tmp_code['rank'].values\n    df_tmp_code_cell_id = df_tmp_code['cell_id'].values\n\n    for cell_id, rank in df_tmp_markdown[['cell_id', 'rank']].values:\n      labels = np.array([(r==(rank+1)) for r in df_tmp_code_rank]).astype('int')\n\n      for cid, label in zip(df_tmp_code_cell_id, labels):\n        count += 1\n        if label==1:\n          triplets.append( [cell_id, cid, label] )\n          # triplets.append( [cid, cell_id, label] )\n        elif mode == 'test':\n          triplets.append( [cell_id, cid, label] )\n          # triplets.append( [cid, cell_id, label] )\n        elif random_drop[count%10000]:\n          triplets.append( [cell_id, cid, label] )\n          # triplets.append( [cid, cell_id, label] )\n    \n  return triplets\n\ntriplets = generate_triplet(df)","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:34.492993Z","iopub.status.busy":"2022-05-22T16:43:34.492447Z","iopub.status.idle":"2022-05-22T16:43:34.908453Z","shell.execute_reply":"2022-05-22T16:43:34.907695Z"},"papermill":{"duration":0.462805,"end_time":"2022-05-22T16:43:34.911178","exception":false,"start_time":"2022-05-22T16:43:34.448373","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from bisect import bisect\n\n\ndef count_inversions(a):\n    inversions = 0\n    sorted_so_far = []\n    for i, u in enumerate(a):\n        j = bisect(sorted_so_far, u)\n        inversions += i - j\n        sorted_so_far.insert(j, u)\n    return inversions\n\n\ndef kendall_tau(ground_truth, predictions):\n    total_inversions = 0\n    total_2max = 0  # twice the maximum possible inversions across all instances\n    for gt, pred in zip(ground_truth, predictions):\n        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n        total_inversions += count_inversions(ranks)\n        n = len(gt)\n        total_2max += n * (n - 1)\n    return 1 - 4 * total_inversions / total_2max","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:34.987368Z","iopub.status.busy":"2022-05-22T16:43:34.986970Z","iopub.status.idle":"2022-05-22T16:43:34.993602Z","shell.execute_reply":"2022-05-22T16:43:34.992857Z"},"papermill":{"duration":0.046036,"end_time":"2022-05-22T16:43:34.995292","exception":false,"start_time":"2022-05-22T16:43:34.949256","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport sys, os\ntry:\n  from transformers import DistilBertModel, DistilBertTokenizer\nexcept:\n  !pip install transformers\n  !pip install sentencepiece\n  from transformers import DistilBertModel, DistilBertTokenizer\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch\nfrom transformers import AutoModelWithLMHead, AutoTokenizer, AutoModel\n\n\nimport torch\nfrom transformers import RobertaTokenizer, RobertaConfig, RobertaModel\nfrom transformers import AlbertTokenizer, AlbertModel\n\n\nMAX_LEN = 128\n    \nclass MarkdownModel(nn.Module):\n    def __init__(self):\n        super(MarkdownModel, self).__init__()\n        self.distill_bert = AutoModel.from_pretrained(\"./checkpoint-20000\")\n#         self.distill_bert = AutoModel.from_pretrained(\"../input/mymodelbertsmallpretrained/checkpoint-120000\")\n        self.top = nn.Linear(512, 1)\n\n        self.dropout = nn.Dropout(0.2)\n        \n    def forward(self, ids, mask):\n        x = self.distill_bert(ids, mask)[0]\n        x = self.dropout(x)\n        x = self.top(x[:, 0, :])\n        x = torch.sigmoid(x) \n        return x","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:35.070198Z","iopub.status.busy":"2022-05-22T16:43:35.070000Z","iopub.status.idle":"2022-05-22T16:43:35.095946Z","shell.execute_reply":"2022-05-22T16:43:35.095277Z"},"papermill":{"duration":0.065085,"end_time":"2022-05-22T16:43:35.097660","exception":false,"start_time":"2022-05-22T16:43:35.032575","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset\n\n\n\nclass MarkdownDataset(Dataset):\n    \n    def __init__(self, df, max_len, mode='train'):\n        super().__init__()\n        self.df = df\n        self.max_len = max_len\n        self.tokenizer = AutoTokenizer.from_pretrained(\"prajjwal1/bert-small\", do_lower_case=True)\n        self.mode=mode\n\n    def __getitem__(self, index):\n        row = self.df[index]\n\n        label = row[-1]\n\n        txt = dict_cellid_source[row[0]] + '[SEP]' + dict_cellid_source[row[1]]\n\n        inputs = self.tokenizer.encode_plus(\n            txt,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_len,\n            padding=\"max_length\",\n            return_token_type_ids=True,\n            truncation=True\n        )\n        ids = torch.LongTensor(inputs['input_ids'])\n        mask = torch.LongTensor(inputs['attention_mask'])\n\n        return ids, mask, torch.FloatTensor([label])\n\n\n\n\n    def __len__(self):\n        return len(self.df)\n\n\ntrain_ds = MarkdownDataset(triplets, max_len=MAX_LEN)\n\n# train_ds[1]","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:35.172411Z","iopub.status.busy":"2022-05-22T16:43:35.172210Z","iopub.status.idle":"2022-05-22T16:43:47.344652Z","shell.execute_reply":"2022-05-22T16:43:47.343937Z"},"papermill":{"duration":12.21199,"end_time":"2022-05-22T16:43:47.346622","exception":false,"start_time":"2022-05-22T16:43:35.134632","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjust_lr(optimizer, epoch):\n    if epoch < 1:\n        lr = 3e-3\n#     elif epoch < 2:\n#         lr = 5e-5\n#     elif epoch < 5:\n#         lr = 5e-5\n    else:\n        lr = 5e-5\n\n    for p in optimizer.param_groups:\n        p['lr'] = lr\n    return lr\n    \ndef get_optimizer(net):\n    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, \n                                        net.parameters()), \n                                 lr=3e-4, betas=(0.9, 0.999),\n                                 eps=1e-8 ) #1e-08)\n    return optimizer","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BS = 1504\nNW = 8\n\ntrain_loader = DataLoader(train_ds, \n                          batch_size=BS, \n                          shuffle=True, \n                          num_workers=NW,\n                          pin_memory=False, \n                          drop_last=True)\n\n","metadata":{"execution":{"iopub.execute_input":"2022-05-22T16:43:47.425062Z","iopub.status.busy":"2022-05-22T16:43:47.424856Z","iopub.status.idle":"2022-05-22T16:43:47.434300Z","shell.execute_reply":"2022-05-22T16:43:47.432691Z"},"papermill":{"duration":0.050968,"end_time":"2022-05-22T16:43:47.436612","exception":false,"start_time":"2022-05-22T16:43:47.385644","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data(data):\n    return tuple(d.cuda() for d in data[:-1]), data[-1].cuda()\n\ndef validate(model, val_loader, mode='train'):\n    model.eval()\n    \n    tbar = tqdm(val_loader, file=sys.stdout)\n    \n    preds = []\n    labels = []\n\n    with torch.no_grad():\n        for idx, data in enumerate(tbar):\n            inputs, target = read_data(data)\n\n            pred = model(inputs[0], inputs[1])\n\n            preds.append(pred.detach().cpu().numpy().ravel())\n            if mode=='test':\n              labels.append(target.detach().cpu().numpy().ravel())\n    if mode=='test':\n      return np.concatenate(preds)\n    else:\n      return np.concatenate(labels), np.concatenate(preds)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counter = 0","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModel, AutoTokenizer, AdamW, get_linear_schedule_with_warmup","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, train_loader, epochs, Type='markdown'):\n    np.random.seed(0)\n\n    # Creating optimizer and lr schedulers\n    param_optimizer = list(model.named_parameters())\n    \n    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n    \n    optimizer_grouped_parameters = [\n        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n    ]\n    \n    num_train_optimization_steps = int(epochs * len(train_loader) / 1 )\n    \n    optimizer = AdamW(optimizer_grouped_parameters, \n                      lr=1e-5,\n                      correct_bias=False)  # To reproduce BertAdam specific behavior set correct_bias=False\n    \n    scheduler = get_linear_schedule_with_warmup(optimizer, \n                                                num_warmup_steps=0.05 * num_train_optimization_steps,\n                                                num_training_steps=num_train_optimization_steps)  # PyTorch scheduler\n\n    criterion = torch.nn.L1Loss()\n    scaler = torch.cuda.amp.GradScaler()    \n       \n    for e in range(epochs):   \n        model.train()\n        tbar = tqdm(train_loader, file=sys.stdout)\n        loss_list = []\n        preds = []\n        labels = []\n               \n        for idx, data in enumerate(tbar):\n            inputs, target = read_data(data)\n            \n            with torch.cuda.amp.autocast():\n                pred = model(inputs[0], inputs[1]) #pred = model(*inputs)\n                loss = criterion(pred, target)\n                \n            scaler.scale(loss).backward()\n            \n            if idx % 1 == 0 or idx == len(tbar) - 1:\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n                scheduler.step()\n                \n            loss_list.append(loss.detach().cpu().item())\n            preds.append(pred.detach().cpu().numpy().ravel())\n            labels.append(target.detach().cpu().numpy().ravel())\n            \n            avg_loss = np.round(np.mean(loss_list), 4)\n            \n            tbar.set_description(f\"Epoch {e + 1} Loss: {avg_loss} lr: {scheduler.get_last_lr()}\")\n                   \n        output_model_file = f\"./model_output/model_{e}.bin\"\n        model_to_save = model.module if hasattr(model, 'module') else model\n        torch.save(model_to_save.state_dict(), output_model_file)\n\n    return model\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MarkdownModel()\nmodel = model.cuda()\nmodel.load_state_dict(torch.load('./model_output/model_3_0.0753.bin'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = train(model, train_loader, epochs=14, Type='markdown')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### On the cell below it is the output. it took 14 hours to train.","metadata":{}},{"cell_type":"code","source":"# Epoch 1 Loss: 0.0728 lr: [9.774436090225566e-06, 9.774436090225566e-06]: 100%|█| 80\n# Epoch 2 Loss: 0.0722 lr: [9.022556390977446e-06, 9.022556390977446e-06]: 100%|█| 80\n# Epoch 3 Loss: 0.0716 lr: [8.270676691729324e-06, 8.270676691729324e-06]: 100%|█| 80\n# Epoch 4 Loss: 0.0711 lr: [7.5187969924812045e-06, 7.5187969924812045e-06]: 100%|█| \n# Epoch 5 Loss: 0.0707 lr: [6.766917293233084e-06, 6.766917293233084e-06]: 100%|█| 80\n# Epoch 6 Loss: 0.0702 lr: [6.015037593984963e-06, 6.015037593984963e-06]: 100%|█| 80\n# Epoch 7 Loss: 0.0699 lr: [5.263157894736843e-06, 5.263157894736843e-06]: 100%|█| 80\n# Epoch 8 Loss: 0.0695 lr: [4.511278195488723e-06, 4.511278195488723e-06]: 100%|█| 80\n# Epoch 9 Loss: 0.0692 lr: [3.7593984962406023e-06, 3.7593984962406023e-06]: 100%|█| \n# Epoch 10 Loss: 0.0688 lr: [3.0075187969924816e-06, 3.0075187969924816e-06]: 100%|█|\n# Epoch 11 Loss: 0.0686 lr: [2.2556390977443614e-06, 2.2556390977443614e-06]: 100%|█|\n# Epoch 12 Loss: 0.0682 lr: [2.203888031542089e-06, 2.203888031542089e-06]:   7%| | 5","metadata":{"execution":{"iopub.status.busy":"2022-07-16T13:43:58.130910Z","iopub.execute_input":"2022-07-16T13:43:58.131419Z","iopub.status.idle":"2022-07-16T13:43:58.151923Z","shell.execute_reply.started":"2022-07-16T13:43:58.131303Z","shell.execute_reply":"2022-07-16T13:43:58.151069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.079003,"end_time":"2022-05-22T16:44:26.258997","exception":false,"start_time":"2022-05-22T16:44:26.179994","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}