{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Machine Learning Assignment Report\n* **Student name:** Đặng Trung Cương\n* **Student ID:** 19021229\n* **Class:** Machine Learning - INT3405E 20\n* **Instructor:** Trần Quốc Long","metadata":{}},{"cell_type":"markdown","source":"# Problem Description\n\n**Quora** is a question and answer website where people go to find information about literally anything (it's like StackOverflow is mainly for engineer and Quora is for anything not just tech). Up on the website are tons of good questions (sincere) and bad questions (insincere) and our mission is to classify which question belongs to which category (0 for sincere and 1 for insincere - binary classification)\n\nThe **input** (datasets) that Quora provided contains:\n* train.csv: train dataset contains questions and all the questions are labeled 0 or 1 for us.\n* test.csv: validation dataset\n* sample_submission.csv: for kaggle submission.\n* embedding.zip: contains some embedding libs such as GloVe,...\n\nOutput: submission.csv, in which we need to classify and label (0 or 1) for each question within 'sample_submission.csv' ","metadata":{}},{"cell_type":"markdown","source":"Since the submission requires Internet connection turned off, we need to download some packages/dependencies and import them into the dataset.","metadata":{}},{"cell_type":"markdown","source":"**Install packages/dependencies for pytorch-xla (TPU)**","metadata":{}},{"cell_type":"code","source":"# Turn on TPU before executing these lines\n# !cp ../input/pytorch-xla-setup-script/torch-nightly-cp37-cp37m-linux_x86_64.whl ./torch-nightly-cp37-cp37m-linux_x86_64.whl\n# !cp ../input/pytorch-xla-setup-script/torch_xla-nightly-cp37-cp37m-linux_x86_64.whl ./torch_xla-nightly-cp37-cp37m-linux_x86_64.whl\n# !cp ../input/pytorch-xla-setup-script/torchvision-nightly-cp37-cp37m-linux_x86_64.whl ./torchvision-nightly-cp37-cp37m-linux_x86_64.whl\n\n# # This deb files are the dependencies, copying them to the working dir.\n# !cp ../input/pytorch-xla-setup-script/libgfortran4_7.5.0-3ubuntu1_18.04_amd64.deb ./libgfortran4_7.5.0-3ubuntu1_18.04_amd64.deb\n# !cp ../input/pytorch-xla-setup-script/libomp5_5.0.1-1_amd64.deb ./libomp5_5.0.1-1_amd64.deb\n# !cp ../input/pytorch-xla-setup-script/libopenblas-base_0.2.20ds-4_amd64.deb ./libopenblas-base_0.2.20ds-4_amd64.deb\n# !cp ../input/pytorch-xla-setup-script/libopenblas-dev_0.2.20ds-4_amd64.deb ./libopenblas-dev_0.2.20ds-4_amd64.deb\n\n# #installing pytorch-xla by running this script\n# !python ../input/pytorch-xla-setup-script/pytorch-xla-env-setup.py --version nightly\n\n# #Now, istalling depedencies\n# !dpkg -i ./libgfortran4_7.5.0-3ubuntu1_18.04_amd64.deb\n# !dpkg -i ./libomp5_5.0.1-1_amd64.deb\n# !dpkg -i ./libopenblas-base_0.2.20ds-4_amd64.deb\n# !dpkg -i ./libopenblas-dev_0.2.20ds-4_amd64.deb\n\n# # Removing wheel and deb files, as we don't need them now.\n# !rm torch-nightly-cp37-cp37m-linux_x86_64.whl \n# !rm torch_xla-nightly-cp37-cp37m-linux_x86_64.whl \n# !rm torchvision-nightly-cp37-cp37m-linux_x86_64.whl\n# !rm libgfortran4_7.5.0-3ubuntu1_18.04_amd64.deb \n# !rm libomp5_5.0.1-1_amd64.deb \n# !rm libopenblas-base_0.2.20ds-4_amd64.deb \n# !rm libopenblas-dev_0.2.20ds-4_amd64.deb","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:42.887223Z","iopub.execute_input":"2022-01-09T12:59:42.887854Z","iopub.status.idle":"2022-01-09T12:59:42.892966Z","shell.execute_reply.started":"2022-01-09T12:59:42.887811Z","shell.execute_reply":"2022-01-09T12:59:42.892209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Import normal packages**","metadata":{}},{"cell_type":"code","source":"# Please switch on the TPU before running these lines.\nimport pandas as pd\nimport numpy as np\nimport os\nimport gc\n\nimport random\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\n# set a seed value\ntorch.manual_seed(555)\n\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import roc_auc_score, accuracy_score, f1_score\n\nimport transformers\nfrom transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification\nfrom transformers import AdamW\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n\nprint(torch.__version__)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:42.894843Z","iopub.execute_input":"2022-01-09T12:59:42.895352Z","iopub.status.idle":"2022-01-09T12:59:42.907777Z","shell.execute_reply.started":"2022-01-09T12:59:42.895316Z","shell.execute_reply":"2022-01-09T12:59:42.907003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Have to install torch-1.8-1 since Kaggle environments (by default) only provide torch-1.7.1+\n# which is not compatible with torch-xla\n# !pip install /kaggle/input/torch181whl/torch-1.8.1+cu111-cp37-cp37m-linux_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:42.909976Z","iopub.execute_input":"2022-01-09T12:59:42.910514Z","iopub.status.idle":"2022-01-09T12:59:42.91907Z","shell.execute_reply.started":"2022-01-09T12:59:42.910478Z","shell.execute_reply":"2022-01-09T12:59:42.918369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Imports required to use TPUs with Pytorch.\n# https://pytorch.org/xla/release/1.5/index.html\n\n# import torch_xla\n# import torch_xla.core.xla_model as xm","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:42.921941Z","iopub.execute_input":"2022-01-09T12:59:42.922597Z","iopub.status.idle":"2022-01-09T12:59:42.92811Z","shell.execute_reply.started":"2022-01-09T12:59:42.92256Z","shell.execute_reply":"2022-01-09T12:59:42.927408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get files from Kaggle directory","metadata":{}},{"cell_type":"code","source":"# If Kaggle runs out of RAM then we may need this.\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:42.930933Z","iopub.execute_input":"2022-01-09T12:59:42.931307Z","iopub.status.idle":"2022-01-09T12:59:42.947034Z","shell.execute_reply.started":"2022-01-09T12:59:42.931259Z","shell.execute_reply":"2022-01-09T12:59:42.946345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check dir contents\nos.listdir('../input/quora-insincere-questions-classification')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:42.950486Z","iopub.execute_input":"2022-01-09T12:59:42.95238Z","iopub.status.idle":"2022-01-09T12:59:42.965762Z","shell.execute_reply.started":"2022-01-09T12:59:42.952337Z","shell.execute_reply":"2022-01-09T12:59:42.965054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the training data.\n\npath = '../input/quora-insincere-questions-classification/train.csv'\ndf_train = pd.read_csv(path)\n\nprint(df_train.shape)\n\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:42.967016Z","iopub.execute_input":"2022-01-09T12:59:42.967272Z","iopub.status.idle":"2022-01-09T12:59:45.712845Z","shell.execute_reply.started":"2022-01-09T12:59:42.967239Z","shell.execute_reply":"2022-01-09T12:59:45.712128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the test data.\n\npath = '../input/quora-insincere-questions-classification/test.csv'\ndf_test = pd.read_csv(path)\n\nprint(df_test.shape)\n\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:45.714899Z","iopub.execute_input":"2022-01-09T12:59:45.715494Z","iopub.status.idle":"2022-01-09T12:59:46.379189Z","shell.execute_reply.started":"2022-01-09T12:59:45.715452Z","shell.execute_reply":"2022-01-09T12:59:46.377896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Investigate the datasets (train.csv && test.csv)","metadata":{}},{"cell_type":"code","source":"# Check info of the intact file\ndf_train.info() ","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:46.380552Z","iopub.execute_input":"2022-01-09T12:59:46.381Z","iopub.status.idle":"2022-01-09T12:59:46.652987Z","shell.execute_reply.started":"2022-01-09T12:59:46.380957Z","shell.execute_reply":"2022-01-09T12:59:46.652241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modify to reduce memory usage but keep the file's integrity\ndf_train = reduce_mem_usage(df_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:46.65436Z","iopub.execute_input":"2022-01-09T12:59:46.654802Z","iopub.status.idle":"2022-01-09T12:59:46.667916Z","shell.execute_reply.started":"2022-01-09T12:59:46.654763Z","shell.execute_reply":"2022-01-09T12:59:46.666847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:46.669388Z","iopub.execute_input":"2022-01-09T12:59:46.669685Z","iopub.status.idle":"2022-01-09T12:59:46.935858Z","shell.execute_reply.started":"2022-01-09T12:59:46.669645Z","shell.execute_reply":"2022-01-09T12:59:46.93507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = reduce_mem_usage(df_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:46.937384Z","iopub.execute_input":"2022-01-09T12:59:46.937682Z","iopub.status.idle":"2022-01-09T12:59:46.945957Z","shell.execute_reply.started":"2022-01-09T12:59:46.937645Z","shell.execute_reply":"2022-01-09T12:59:46.945075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Investigate on \"target\" column**","metadata":{}},{"cell_type":"code","source":"# Print len(df_train) and len(df_test)\nprint(\"length of df_train: \", len(df_train))\nprint(\"length of df_test: \", len(df_test))\n\n# sinc_q: sincere question // insinc_q: insincere\nsinc_q = df_train[df_train.target == 0]\ninsinc_q = df_train[df_train.target == 1]\n# Print number of sincere questions\nprint(\"Number of sincere question: \", len(sinc_q))\n# Print number of insincere questions\nprint(\"Number of sincere question: \", len(insinc_q))\n# Ratio of sinc_q and insinc_q with total number of questions\nprint(\"Percentage of sincere questions: \", len(sinc_q)/len(df_train)*100, \"%\")\nprint(\"Percentage of insincere questions: \", len(insinc_q)/len(df_train)*100, \"%\")","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:46.947104Z","iopub.execute_input":"2022-01-09T12:59:46.947491Z","iopub.status.idle":"2022-01-09T12:59:47.051857Z","shell.execute_reply.started":"2022-01-09T12:59:46.947455Z","shell.execute_reply":"2022-01-09T12:59:47.051141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Datasets overview**\n\n**Train dataset(train.csv):**\n\nNumber of questions: 1306122\n\nColumns: 3 (qid, question_text(text_column), target(label_column)<0 = sincere; 1 = insincere>)\n\n**Test dataset(test.csv):**\n\nNumber of questions: 375806\n\nColumns: 2 (qid, question_text)\n\n**\"target\" column of train dataset:**\n\nDataset is overloaded with sincere questions and lack of insincere questions (1225312 of sincere versus 80810 of insincere, hence the ratio sincere:insincre is appoximately 15:1). This imbalance may leads to some problems:\n* Accuracy should not be used as the main metric to judge model's ability because if we predict all the questions are sincere (target=0) then model's accuracy will be ~0.94.\n=> Need to use other evaluate metrics such as f1_score, roc, auc...\n* Since there are too many sincere questions, the trained model will be good at detecting sincere questions. Meanwhile, its performance in insincere questions detection will be decreased.\n\nOther than using other metrics, we can try using pre-trained model (BERT...) or use Boosting methods (LightGBM, XGBoost,...)","metadata":{}},{"cell_type":"markdown","source":"# Text Preprocessing\n","metadata":{}},{"cell_type":"markdown","source":"**Remove punctuations**","metadata":{}},{"cell_type":"code","source":"\ndef clean_text(x):\n    puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\n","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:47.055111Z","iopub.execute_input":"2022-01-09T12:59:47.055377Z","iopub.status.idle":"2022-01-09T12:59:47.065673Z","shell.execute_reply.started":"2022-01-09T12:59:47.055347Z","shell.execute_reply":"2022-01-09T12:59:47.064967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Remove numbers**","metadata":{}},{"cell_type":"code","source":"import re\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:47.066874Z","iopub.execute_input":"2022-01-09T12:59:47.067302Z","iopub.status.idle":"2022-01-09T12:59:47.078184Z","shell.execute_reply.started":"2022-01-09T12:59:47.067255Z","shell.execute_reply":"2022-01-09T12:59:47.077538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Fix mispelled/stop words**","metadata":{}},{"cell_type":"code","source":"def _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n \n \nmispell_dict = {'colour':'color','centre':'center','didnt':'did not','doesnt':'does not',\n                'isnt':'is not','shouldnt':'should not','favourite':'favorite','travelling':'traveling',\n                'counselling':'counseling','theatre':'theater','cancelled':'canceled','labour':'labor',\n                'organisation':'organization','wwii':'world war 2','citicise':'criticize','instagram': 'social medium',\n                'whatsapp': 'social medium','snapchat': 'social medium',\"ain't\": \"is not\", \n                \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \n                \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \n                \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\n                \"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \n                \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \n                \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \n                \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",\"i'll've\": \"i will have\",\n                \"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \n                \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\n                \"it's\": \"it is\",\"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \n                \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \n                \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\",\n                \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \n                \"oughtn't\": \"ought not\",\"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \n                \"sha'n't\": \"shall not\",\"shan't've\": \"shall not have\",\"she'd\": \"she would\", \n                \"she'd've\": \"she would have\",\"she'll\": \"she will\",\"she'll've\": \"she will have\", \n                \"she's\": \"she is\",\"should've\": \"should have\",\"shouldn't\": \"should not\",\"shouldn't've\": \"should not have\", \n                \"so've\": \"so have\",\"so's\": \"so as\",\"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \n                \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \n                \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\",\n                \"they'll\": \"they will\", \"they'll've\": \"they will have\", \n                \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \n                \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \n                \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \n                \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \n                \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\",\n                \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \n                \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \n                \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \n                \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \n                \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \n                \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \n                \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\n                \"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \n                \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\",\n                \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', \n                'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', \n                'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', \n                'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', \n                'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', \n                'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', \n                'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do',\n                'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', \n                'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \n                \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', \n                'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', \n                'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \n                \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', \n                'demonitization': 'demonetization', 'demonetisation': 'demonetization'\n                }\n \nmispellings, mispellings_re = _get_mispell(mispell_dict)\n \ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n \n    return mispellings_re.sub(replace, text)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:47.079766Z","iopub.execute_input":"2022-01-09T12:59:47.080323Z","iopub.status.idle":"2022-01-09T12:59:47.10339Z","shell.execute_reply.started":"2022-01-09T12:59:47.080272Z","shell.execute_reply":"2022-01-09T12:59:47.102557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Process**","metadata":{}},{"cell_type":"code","source":"# lower\ndf_train['prep_question_text'] = df_train['question_text'].apply(lambda x : x.lower())\ndf_test['prep_question_text'] = df_test['question_text'].apply(lambda x : x.lower())\n \n# clean the text\ndf_train[\"prep_question_text\"] = df_train[\"prep_question_text\"].apply(lambda x : clean_text(x))\ndf_test[\"prep_question_text\"] = df_test[\"prep_question_text\"].apply(lambda x : clean_text(x))\n \n# clean numbers\ndf_train[\"prep_question_text\"] = df_train[\"prep_question_text\"].apply(lambda x: clean_numbers(x))\ndf_test[\"prep_question_text\"] = df_test[\"prep_question_text\"].apply(lambda x : clean_numbers(x))\n \n# clean spellings\ndf_train['prep_question_text'] = df_train['prep_question_text'].apply(lambda x: replace_typical_misspell(x))\ndf_test['prep_question_text'] = df_test['prep_question_text'].apply(lambda x: replace_typical_misspell(x))\n \n","metadata":{"execution":{"iopub.status.busy":"2022-01-09T12:59:47.105725Z","iopub.execute_input":"2022-01-09T12:59:47.10731Z","iopub.status.idle":"2022-01-09T13:02:17.339374Z","shell.execute_reply.started":"2022-01-09T12:59:47.107265Z","shell.execute_reply":"2022-01-09T13:02:17.338546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# processed text\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:17.340589Z","iopub.execute_input":"2022-01-09T13:02:17.340839Z","iopub.status.idle":"2022-01-09T13:02:17.352629Z","shell.execute_reply.started":"2022-01-09T13:02:17.340805Z","shell.execute_reply":"2022-01-09T13:02:17.351891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.to_csv('prep.csv', columns=['qid', 'target', 'prep_question_text'], index=False)\n!cp 'prep.csv' './prep.csv'\n\ndf_prep = pd.read_csv('./prep.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:17.353979Z","iopub.execute_input":"2022-01-09T13:02:17.354589Z","iopub.status.idle":"2022-01-09T13:02:27.204108Z","shell.execute_reply.started":"2022-01-09T13:02:17.354549Z","shell.execute_reply":"2022-01-09T13:02:27.203228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_prep.groupby(\"target\").size()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.205837Z","iopub.execute_input":"2022-01-09T13:02:27.206158Z","iopub.status.idle":"2022-01-09T13:02:27.23597Z","shell.execute_reply.started":"2022-01-09T13:02:27.206114Z","shell.execute_reply":"2022-01-09T13:02:27.235212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Training (XGBoost and XLM-RoBERTa)","metadata":{}},{"cell_type":"markdown","source":"**XGBoost**","metadata":{}},{"cell_type":"markdown","source":"**XGBoost** is an optimized distributed gradient boosting library designed to be highly efficient, flexible and portable. It implements machine learning algorithms under the Gradient Boosting framework. XGBoost provides a parallel tree boosting (also known as GBDT, GBM) that solve many data science problems in a fast and accurate way. The same code runs on major distributed environment (Hadoop, SGE, MPI) and can solve problems beyond billions of examples.","metadata":{}},{"cell_type":"code","source":"# Install, looks like XGBoost in pre-installed in Kaggle kernel.\n\n# !pip install xgboost","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.237482Z","iopub.execute_input":"2022-01-09T13:02:27.237737Z","iopub.status.idle":"2022-01-09T13:02:27.241284Z","shell.execute_reply.started":"2022-01-09T13:02:27.237702Z","shell.execute_reply":"2022-01-09T13:02:27.240487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from numpy import loadtxt\n# from xgboost import XGBClassifier\n# from sklearn.model_selection import train_test_split\n# from sklearn.metrics import accuracy_score, f1_score, classification_report","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.242874Z","iopub.execute_input":"2022-01-09T13:02:27.24343Z","iopub.status.idle":"2022-01-09T13:02:27.25091Z","shell.execute_reply.started":"2022-01-09T13:02:27.243363Z","shell.execute_reply":"2022-01-09T13:02:27.250208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Vectorizer data\n# from sklearn.feature_extraction.text import TfidfVectorizer\n# vectorizer = TfidfVectorizer(stop_words=\"english\",\n#                              ngram_range=(1, 3))\n# X = vectorizer.fit_transform(df_train['prep_question_text'])\n# x = vectorizer.transform(df_test['prep_question_text'])","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.252145Z","iopub.execute_input":"2022-01-09T13:02:27.252662Z","iopub.status.idle":"2022-01-09T13:02:27.259873Z","shell.execute_reply.started":"2022-01-09T13:02:27.252625Z","shell.execute_reply":"2022-01-09T13:02:27.259157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Split train and test data\n# from sklearn.model_selection import train_test_split\n# X_train, X_test, y_train, y_test = train_test_split(X, df_train['target'], test_size=0.3, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.261499Z","iopub.execute_input":"2022-01-09T13:02:27.262292Z","iopub.status.idle":"2022-01-09T13:02:27.268769Z","shell.execute_reply.started":"2022-01-09T13:02:27.262254Z","shell.execute_reply":"2022-01-09T13:02:27.267952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Function for f1_score calculation\n# from sklearn.metrics import f1_score, accuracy_score, classification_report\n# def get_score(model, name):\n#   y_train_pred, y_pred = model.predict(X_train), model.predict(X_test)\n#   print(classification_report(y_test, y_pred), '\\n')\n\n#   print('{} model with F1 score = {}'.format(name, f1_score(y_test, y_pred)))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.271242Z","iopub.execute_input":"2022-01-09T13:02:27.27148Z","iopub.status.idle":"2022-01-09T13:02:27.277916Z","shell.execute_reply.started":"2022-01-09T13:02:27.271449Z","shell.execute_reply":"2022-01-09T13:02:27.277236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Fit\n# import xgboost as xgb\n# xgb = xgb.XGBClassifier()\n# xgb.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.279419Z","iopub.execute_input":"2022-01-09T13:02:27.27997Z","iopub.status.idle":"2022-01-09T13:02:27.28887Z","shell.execute_reply.started":"2022-01-09T13:02:27.279933Z","shell.execute_reply":"2022-01-09T13:02:27.288133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = XGBClassifier()\n# model.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.29006Z","iopub.execute_input":"2022-01-09T13:02:27.29089Z","iopub.status.idle":"2022-01-09T13:02:27.297562Z","shell.execute_reply.started":"2022-01-09T13:02:27.290854Z","shell.execute_reply":"2022-01-09T13:02:27.296811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get_score(xgb, 'XGBClassifier')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.298709Z","iopub.execute_input":"2022-01-09T13:02:27.299603Z","iopub.status.idle":"2022-01-09T13:02:27.306779Z","shell.execute_reply.started":"2022-01-09T13:02:27.299565Z","shell.execute_reply":"2022-01-09T13:02:27.306151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**XLM-RoBERTa**","metadata":{}},{"cell_type":"markdown","source":"**XLM-RoBERTa** is a multilingual version of **RoBERTa**. It is pre-trained on 2.5TB of filtered CommonCrawl data containing 100 languages.\n\n**RoBERTa** is a transformers model pretrained on a large corpus in a self-supervised fashion. This means it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of publicly available data) with an automatic process to generate inputs and labels from those texts.\n\nMore precisely, it was pretrained with the Masked language modeling (MLM) objective. Taking a sentence, the model randomly masks 15% of the words in the input then run the entire masked sentence through the model and has to predict the masked words. This is different from traditional recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the sentence.","metadata":{}},{"cell_type":"code","source":"# # Undersampling to 1:10 => reduce imbalancing\n# from sklearn.utils import resample\n\n# df_train = pd.concat([resample(sinc_q, replace = True, n_samples = len(insinc_q)*10), insinc_q])\n# df_train","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.307833Z","iopub.execute_input":"2022-01-09T13:02:27.310299Z","iopub.status.idle":"2022-01-09T13:02:27.31572Z","shell.execute_reply.started":"2022-01-09T13:02:27.310261Z","shell.execute_reply":"2022-01-09T13:02:27.314992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Create training folds**","metadata":{}},{"cell_type":"code","source":"# from sklearn.model_selection import KFold, StratifiedKFold\n\n# # shuffle\n# df = shuffle(df_train)\n\n# # initialize kfold\n# kf = StratifiedKFold(n_splits=3, shuffle=True, random_state=1024)\n\n# # for stratification\n# y = df['target']\n\n# # Note:\n# # Each fold is a tuple ([train_index_values], [val_index_values])\n\n# # Put the folds into a list. This is a list of tuples.\n# fold_list = list(kf.split(df, y))\n\n# train_df_list = []\n# val_df_list = []\n\n# for i, fold in enumerate(fold_list):\n\n#     # map the train and val index values to dataframe rows\n#     df_train = df[df.index.isin(fold[0])]\n#     df_val = df[df.index.isin(fold[1])]\n    \n#     train_df_list.append(df_train)\n#     val_df_list.append(df_val)\n    \n    \n\n# print(len(train_df_list))\n# print(len(val_df_list))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.316753Z","iopub.execute_input":"2022-01-09T13:02:27.319072Z","iopub.status.idle":"2022-01-09T13:02:27.325795Z","shell.execute_reply.started":"2022-01-09T13:02:27.319042Z","shell.execute_reply":"2022-01-09T13:02:27.32502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Display one train fold\n\n# df_train = train_df_list[0]\n\n# df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.327284Z","iopub.execute_input":"2022-01-09T13:02:27.327814Z","iopub.status.idle":"2022-01-09T13:02:27.334187Z","shell.execute_reply.started":"2022-01-09T13:02:27.327766Z","shell.execute_reply":"2022-01-09T13:02:27.333465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Display one val fold\n\n# df_val = val_df_list[0]\n\n# df_val.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.340962Z","iopub.execute_input":"2022-01-09T13:02:27.341574Z","iopub.status.idle":"2022-01-09T13:02:27.345758Z","shell.execute_reply.started":"2022-01-09T13:02:27.341546Z","shell.execute_reply":"2022-01-09T13:02:27.345012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Plot histogram of text lengths => optimize max tokens length\n# word_length_list = [len(x.split()) for x in df_train['prep_question_text'] if len(x.split()) < 60]\n# char_length_list = [len(x) for x in df_train['prep_question_text'] if len(x) < 100]\n# fig, axs = plt.subplots(1, 2, sharey=True, tight_layout=True)\n# axs[0].hist(word_length_list, bins=25)\n# axs[0].set_title('Number of Words')\n\n# axs[1].hist(char_length_list, bins=25)\n# axs[1].set_title('Number of Characters')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.347024Z","iopub.execute_input":"2022-01-09T13:02:27.3474Z","iopub.status.idle":"2022-01-09T13:02:27.353977Z","shell.execute_reply.started":"2022-01-09T13:02:27.347362Z","shell.execute_reply":"2022-01-09T13:02:27.353314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the above histograms, a maxlen of ~30-35 tokens should be fine for BERT.","metadata":{}},{"cell_type":"markdown","source":"**RoBERTa Implement**","metadata":{}},{"cell_type":"code","source":"# PRETRAINED_PATH = \"../input/xlmroberta/xlm-roberta-base\"\n\n\n# L_RATE = 1e-5 # Optimized lrate<paper>\n# MAX_LEN = 35\n\n# NUM_EPOCHS = 2\n# BATCH_SIZE_TRAIN = 32\n# BATCH_SIZE_TEST = 64\n# NUM_CORES = os.cpu_count()\n\n# NUM_CORES","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.35687Z","iopub.execute_input":"2022-01-09T13:02:27.357083Z","iopub.status.idle":"2022-01-09T13:02:27.364004Z","shell.execute_reply.started":"2022-01-09T13:02:27.357058Z","shell.execute_reply":"2022-01-09T13:02:27.363156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Tell PyTorch to use the TPU.    \n# device = xm.xla_device()\n\n# print(device)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.365206Z","iopub.execute_input":"2022-01-09T13:02:27.365906Z","iopub.status.idle":"2022-01-09T13:02:27.372539Z","shell.execute_reply.started":"2022-01-09T13:02:27.365869Z","shell.execute_reply":"2022-01-09T13:02:27.371831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Only use one fold\n# df_train = train_df_list[0]\n\n# df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.373725Z","iopub.execute_input":"2022-01-09T13:02:27.374553Z","iopub.status.idle":"2022-01-09T13:02:27.380725Z","shell.execute_reply.started":"2022-01-09T13:02:27.374515Z","shell.execute_reply":"2022-01-09T13:02:27.379972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_val = val_df_list[0]\n\n# df_val.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.383088Z","iopub.execute_input":"2022-01-09T13:02:27.383766Z","iopub.status.idle":"2022-01-09T13:02:27.38905Z","shell.execute_reply.started":"2022-01-09T13:02:27.383726Z","shell.execute_reply":"2022-01-09T13:02:27.388358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification\n\n\n# print('Loading XLMRoberta tokenizer...')\n# tokenizer = XLMRobertaTokenizer.from_pretrained(PRETRAINED_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.39164Z","iopub.execute_input":"2022-01-09T13:02:27.393299Z","iopub.status.idle":"2022-01-09T13:02:27.398034Z","shell.execute_reply.started":"2022-01-09T13:02:27.393249Z","shell.execute_reply":"2022-01-09T13:02:27.397223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train = df_train.reset_index(drop=True)\n# df_val = df_val.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.401055Z","iopub.execute_input":"2022-01-09T13:02:27.401307Z","iopub.status.idle":"2022-01-09T13:02:27.408792Z","shell.execute_reply.started":"2022-01-09T13:02:27.401273Z","shell.execute_reply":"2022-01-09T13:02:27.408098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class QuestionDataset(Dataset):\n\n#     def __init__(self, df):\n#         self.df_data = df\n\n\n\n#     def __getitem__(self, index):\n\n#         # get the sentence from the dataframe\n#         sentence1 = self.df_data.loc[index, 'prep_question_text']\n      \n#         # Process the sentence\n#         # ---------------------\n\n#         encoded_dict = tokenizer.encode_plus(\n#                     sentence1,        # Sentences to encode.\n#                     add_special_tokens = True,      # Add the special tokens.\n#                     max_length = MAX_LEN,           # Pad & truncate all sentences.\n#                     pad_to_max_length = True,\n#                     return_attention_mask = True,   # Construct attn. masks.\n#                     return_tensors = 'pt',          # Return pytorch tensors.\n#                )\n        \n#         # These are torch tensors.\n#         padded_token_list = encoded_dict['input_ids'][0]\n#         att_mask = encoded_dict['attention_mask'][0]\n        \n#         # Convert the target to a torch tensor\n#         target = torch.tensor(self.df_data.loc[index, 'target'])\n\n#         sample = (padded_token_list, att_mask, target)\n\n\n#         return sample\n\n\n#     def __len__(self):\n#         return len(self.df_data)\n    \n    \n    \n    \n    \n\n# class TestDataset(Dataset):\n\n#     def __init__(self, df):\n#         self.df_data = df\n\n\n\n#     def __getitem__(self, index):\n\n#         # get the sentence from the dataframe\n#         sentence1 = self.df_data.loc[index, 'prep_question_text']\n\n#         # Process the sentence\n#         # ---------------------\n\n#         encoded_dict = tokenizer.encode_plus(\n#                     sentence1,         # Sentence to encode.\n#                     add_special_tokens = True,      # Add the special tokens.\n#                     max_length = MAX_LEN,           # Pad & truncate all sentences.\n#                     pad_to_max_length = True,\n#                     return_attention_mask = True,   # Construct attn. masks.\n#                     return_tensors = 'pt',          # Return pytorch tensors.\n#                )\n        \n#         # These are torch tensors.\n#         padded_token_list = encoded_dict['input_ids'][0]\n#         att_mask = encoded_dict['attention_mask'][0]\n        \n               \n\n#         sample = (padded_token_list, att_mask)\n\n\n#         return sample\n\n\n#     def __len__(self):\n#         return len(self.df_data)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.410184Z","iopub.execute_input":"2022-01-09T13:02:27.410898Z","iopub.status.idle":"2022-01-09T13:02:27.418362Z","shell.execute_reply.started":"2022-01-09T13:02:27.410861Z","shell.execute_reply":"2022-01-09T13:02:27.417599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data = QuestionDataset(df_train)\n# val_data = QuestionDataset(df_val)\n# test_data = TestDataset(df_test)\n\n# train_dataloader = torch.utils.data.DataLoader(train_data,\n#                                         batch_size=BATCH_SIZE_TRAIN,\n#                                         shuffle=True,\n#                                        num_workers=NUM_CORES)\n\n# val_dataloader = torch.utils.data.DataLoader(val_data,\n#                                         batch_size=BATCH_SIZE_TRAIN,\n#                                         shuffle=True,\n#                                        num_workers=NUM_CORES)\n\n# test_dataloader = torch.utils.data.DataLoader(test_data,\n#                                         batch_size=BATCH_SIZE_TEST,\n#                                         shuffle=False,\n#                                        num_workers=NUM_CORES)\n\n\n\n# print(len(train_dataloader))\n# print(len(val_dataloader))\n# print(len(test_dataloader))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.41975Z","iopub.execute_input":"2022-01-09T13:02:27.420407Z","iopub.status.idle":"2022-01-09T13:02:27.428366Z","shell.execute_reply.started":"2022-01-09T13:02:27.42037Z","shell.execute_reply":"2022-01-09T13:02:27.42766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Get one train batch\n\n# padded_token_list, att_mask, target = next(iter(train_dataloader))\n\n# print(padded_token_list.shape)\n# print(att_mask.shape)\n# print(target.shape)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.429375Z","iopub.execute_input":"2022-01-09T13:02:27.432305Z","iopub.status.idle":"2022-01-09T13:02:27.439117Z","shell.execute_reply.started":"2022-01-09T13:02:27.43226Z","shell.execute_reply":"2022-01-09T13:02:27.438329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Get one val batch\n\n# padded_token_list, att_mask, target = next(iter(val_dataloader))\n\n# print(padded_token_list.shape)\n# print(att_mask.shape)\n# print(target.shape)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.44011Z","iopub.execute_input":"2022-01-09T13:02:27.440558Z","iopub.status.idle":"2022-01-09T13:02:27.447911Z","shell.execute_reply.started":"2022-01-09T13:02:27.440522Z","shell.execute_reply":"2022-01-09T13:02:27.447179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Get one test batch\n\n# padded_token_list, att_mask = next(iter(test_dataloader))\n\n# print(padded_token_list.shape)\n# print(att_mask.shape)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.449301Z","iopub.execute_input":"2022-01-09T13:02:27.449784Z","iopub.status.idle":"2022-01-09T13:02:27.455647Z","shell.execute_reply.started":"2022-01-09T13:02:27.449749Z","shell.execute_reply":"2022-01-09T13:02:27.454955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from transformers import XLMRobertaForSequenceClassification\n\n# model = XLMRobertaForSequenceClassification.from_pretrained(\n#     PRETRAINED_PATH, \n#     num_labels = 2, # The number of output labels. 2 for binary classification.\n# )\n\n# # Send the model to the device.\n# model.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.456642Z","iopub.execute_input":"2022-01-09T13:02:27.456828Z","iopub.status.idle":"2022-01-09T13:02:27.464116Z","shell.execute_reply.started":"2022-01-09T13:02:27.456806Z","shell.execute_reply":"2022-01-09T13:02:27.463408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Create a batch of train samples\n# # We will set a small batch size of 8 so that the model's output can be easily displayed.\n\n# train_dataloader = torch.utils.data.DataLoader(train_data,\n#                                         batch_size=8,\n#                                         shuffle=True,\n#                                        num_workers=NUM_CORES)\n\n# b_input_ids, b_input_mask, b_labels = next(iter(train_dataloader))\n\n# print(b_input_ids.shape)\n# print(b_input_mask.shape)\n# print(b_labels.shape)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.465047Z","iopub.execute_input":"2022-01-09T13:02:27.467919Z","iopub.status.idle":"2022-01-09T13:02:27.472878Z","shell.execute_reply.started":"2022-01-09T13:02:27.467886Z","shell.execute_reply":"2022-01-09T13:02:27.472083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Pass a batch of train samples to the model.\n\n# batch = next(iter(train_dataloader))\n\n# # Send the data to the device\n# b_input_ids = batch[0].to(device)\n# b_input_mask = batch[1].to(device)\n# b_labels = batch[2].to(device)\n\n# # Run the model\n# outputs = model(b_input_ids, \n#                         attention_mask=b_input_mask, \n#                         labels=b_labels)\n\n# # The ouput is a tuple (loss, preds).\n# outputs","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.474766Z","iopub.execute_input":"2022-01-09T13:02:27.475296Z","iopub.status.idle":"2022-01-09T13:02:27.481802Z","shell.execute_reply.started":"2022-01-09T13:02:27.475254Z","shell.execute_reply":"2022-01-09T13:02:27.481112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# outputs","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.484777Z","iopub.execute_input":"2022-01-09T13:02:27.485217Z","iopub.status.idle":"2022-01-09T13:02:27.490135Z","shell.execute_reply.started":"2022-01-09T13:02:27.485184Z","shell.execute_reply":"2022-01-09T13:02:27.489439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # The output is a tuple: (loss, preds)\n\n# len(outputs)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.491564Z","iopub.execute_input":"2022-01-09T13:02:27.49182Z","iopub.status.idle":"2022-01-09T13:02:27.498015Z","shell.execute_reply.started":"2022-01-09T13:02:27.491787Z","shell.execute_reply":"2022-01-09T13:02:27.497389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # This is the loss.\n\n# outputs[0]","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.499595Z","iopub.execute_input":"2022-01-09T13:02:27.500127Z","iopub.status.idle":"2022-01-09T13:02:27.506462Z","shell.execute_reply.started":"2022-01-09T13:02:27.50009Z","shell.execute_reply":"2022-01-09T13:02:27.505704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # These are the predictions.\n\n# outputs[1]","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.507423Z","iopub.execute_input":"2022-01-09T13:02:27.510925Z","iopub.status.idle":"2022-01-09T13:02:27.514865Z","shell.execute_reply.started":"2022-01-09T13:02:27.510898Z","shell.execute_reply":"2022-01-09T13:02:27.514211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds = outputs[1].detach().cpu().numpy()\n\n# y_true = b_labels.detach().cpu().numpy()\n# y_pred = np.argmax(preds, axis=1)\n\n# y_pred","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.516355Z","iopub.execute_input":"2022-01-09T13:02:27.516851Z","iopub.status.idle":"2022-01-09T13:02:27.523305Z","shell.execute_reply.started":"2022-01-09T13:02:27.516814Z","shell.execute_reply":"2022-01-09T13:02:27.522466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # This is the accuracy without fine tuning.\n\n# val_acc = accuracy_score(y_true, y_pred)\n\n# val_acc","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.525146Z","iopub.execute_input":"2022-01-09T13:02:27.525785Z","iopub.status.idle":"2022-01-09T13:02:27.531402Z","shell.execute_reply.started":"2022-01-09T13:02:27.525602Z","shell.execute_reply":"2022-01-09T13:02:27.530617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # The loss and preds are Torch tensors\n\n# print(type(outputs[0]))\n# print(type(outputs[1]))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.534071Z","iopub.execute_input":"2022-01-09T13:02:27.535752Z","iopub.status.idle":"2022-01-09T13:02:27.539798Z","shell.execute_reply.started":"2022-01-09T13:02:27.535716Z","shell.execute_reply":"2022-01-09T13:02:27.538953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Define the optimizer\n# optimizer = AdamW(model.parameters(),\n#               lr = 1e-3, \n#               eps = 1e-8 \n#             )","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.541634Z","iopub.execute_input":"2022-01-09T13:02:27.54233Z","iopub.status.idle":"2022-01-09T13:02:27.548109Z","shell.execute_reply.started":"2022-01-09T13:02:27.542305Z","shell.execute_reply":"2022-01-09T13:02:27.547368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Create the dataloaders.\n\n# train_data = QuestionDataset(df_train)\n# val_data = QuestionDataset(df_val)\n# test_data = TestDataset(df_test)\n\n# train_dataloader = torch.utils.data.DataLoader(train_data,\n#                                         batch_size=BATCH_SIZE_TRAIN,\n#                                         shuffle=True,\n#                                        num_workers=NUM_CORES)\n\n# val_dataloader = torch.utils.data.DataLoader(val_data,\n#                                         batch_size=BATCH_SIZE_TRAIN,\n#                                         shuffle=True,\n#                                        num_workers=NUM_CORES)\n\n# test_dataloader = torch.utils.data.DataLoader(test_data,\n#                                         batch_size=BATCH_SIZE_TEST,\n#                                         shuffle=False,\n#                                        num_workers=NUM_CORES)\n\n\n\n# print(len(train_dataloader))\n# print(len(val_dataloader))\n# print(len(test_dataloader))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.549592Z","iopub.execute_input":"2022-01-09T13:02:27.550085Z","iopub.status.idle":"2022-01-09T13:02:27.556387Z","shell.execute_reply.started":"2022-01-09T13:02:27.550048Z","shell.execute_reply":"2022-01-09T13:02:27.555695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n\n\n# # Set the seed.\n# seed_val = 101\n\n# random.seed(seed_val)\n# np.random.seed(seed_val)\n# torch.manual_seed(seed_val)\n# torch.cuda.manual_seed_all(seed_val)\n\n# # Store the average loss after each epoch so we can plot them.\n# loss_values = []\n\n\n# # For each epoch...\n# for epoch in range(0, NUM_EPOCHS):\n    \n#     print(\"\")\n#     print('======== Epoch {:} / {:} ========'.format(epoch + 1, NUM_EPOCHS))\n    \n\n#     stacked_val_labels = []\n#     targets_list = []\n\n#     # ========================================\n#     #               Training\n#     # ========================================\n    \n#     print('Training...')\n    \n#     # put the model into train mode\n#     model.train()\n    \n#     # This turns gradient calculations on and off.\n#     torch.set_grad_enabled(True)\n\n\n#     # Reset the total loss for this epoch.\n#     total_train_loss = 0\n\n#     for i, batch in enumerate(train_dataloader):\n        \n#         train_status = 'Batch ' + str(i) + ' of ' + str(len(train_dataloader))\n        \n#         print(train_status, end='\\r')\n\n\n#         b_input_ids = batch[0].to(device)\n#         b_input_mask = batch[1].to(device)\n#         b_labels = batch[2].to(device)\n\n#         model.zero_grad()        \n\n\n#         outputs = model(b_input_ids, \n#                     attention_mask=b_input_mask,\n#                     labels=b_labels)\n        \n#         # Get the loss from the outputs tuple: (loss, logits)\n#         loss = outputs[0]\n        \n#         # Convert the loss from a torch tensor to a number.\n#         # Calculate the total loss.\n#         total_train_loss = total_train_loss + loss.item()\n        \n#         # Zero the gradients\n#         optimizer.zero_grad()\n        \n#         # Perform a backward pass to calculate the gradients.\n#         loss.backward()\n        \n        \n#         # Clip the norm of the gradients to 1.0.\n#         # This is to help prevent the \"exploding gradients\" problem.\n#         torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n        \n        \n        \n#         # Use the optimizer to update the weights.\n        \n#         # Optimizer for GPU\n#         # optimizer.step() \n        \n#         # Optimizer for TPU\n#         # https://pytorch.org/xla/\n#         xm.optimizer_step(optimizer, barrier=True)\n\n    \n#     print('Train loss:' ,total_train_loss)\n\n\n#     # ========================================\n#     #               Validation\n#     # ========================================\n    \n#     print('\\nValidation...')\n\n#     # Put the model in evaluation mode.\n#     model.eval()\n\n#     # Turn off the gradient calculations.\n#     # This tells the model not to compute or store gradients.\n#     # This step saves memory and speeds up validation.\n#     torch.set_grad_enabled(False)\n    \n    \n#     # Reset the total loss for this epoch.\n#     total_val_loss = 0\n    \n\n#     for j, batch in enumerate(val_dataloader):\n        \n#         val_status = 'Batch ' + str(j) + ' of ' + str(len(val_dataloader))\n        \n#         print(val_status, end='\\r')\n\n#         b_input_ids = batch[0].to(device)\n#         b_input_mask = batch[1].to(device)\n#         b_labels = batch[2].to(device)      \n\n\n#         outputs = model(b_input_ids, \n#                 attention_mask=b_input_mask, \n#                 labels=b_labels)\n        \n#         # Get the loss from the outputs tuple: (loss, logits)\n#         loss = outputs[0]\n        \n#         # Convert the loss from a torch tensor to a number.\n#         # Calculate the total loss.\n#         total_val_loss = total_val_loss + loss.item()\n        \n\n#         # Get the preds\n#         preds = outputs[1]\n\n\n#         # Move preds to the CPU\n#         val_preds = preds.detach().cpu().numpy()\n        \n#         # Move the labels to the cpu\n#         targets_np = b_labels.to('cpu').numpy()\n\n#         # Append the labels to a numpy list\n#         targets_list.extend(targets_np)\n\n#         if j == 0:  # first batch\n#             stacked_val_preds = val_preds\n\n#         else:\n#             stacked_val_preds = np.vstack((stacked_val_preds, val_preds))\n\n    \n#     # Calculate the validation accuracy\n#     y_true = targets_list\n#     y_pred = np.argmax(stacked_val_preds, axis=1)\n    \n#     val_acc = accuracy_score(y_true, y_pred)\n#     val_f1 = f1_score(y_true, y_pred)\n    \n#     print('Val loss:' ,total_val_loss)\n#     print('Val acc: ', val_acc)\n#     print('Val F1: ', val_f1)\n\n\n#     # Save the Model\n#     torch.save(model.state_dict(), 'model.pt')\n    \n#     # Use the garbage collector to save memory.\n#     gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.557935Z","iopub.execute_input":"2022-01-09T13:02:27.558453Z","iopub.status.idle":"2022-01-09T13:02:27.568103Z","shell.execute_reply.started":"2022-01-09T13:02:27.558418Z","shell.execute_reply":"2022-01-09T13:02:27.567464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for j, batch in enumerate(test_dataloader):\n        \n#         inference_status = 'Batch ' + str(j+1) + ' of ' + str(len(test_dataloader))\n        \n#         print(inference_status, end='\\r')\n\n#         b_input_ids = batch[0].to(device)\n#         b_input_mask = batch[1].to(device)\n\n\n#         outputs = model(b_input_ids, \n#                 attention_mask=b_input_mask)\n        \n        \n#         # Get the preds\n#         preds = outputs[0]\n\n\n#         # Move preds to the CPU\n#         preds = preds.detach().cpu().numpy()\n        \n#         # Move the labels to the cpu\n#         targets_np = b_labels.to('cpu').numpy()\n\n#         # Append the labels to a numpy list\n#         targets_list.extend(targets_np)\n        \n#         # Stack the predictions.\n\n#         if j == 0:  # first batch\n#             stacked_preds = preds\n\n#         else:\n#             stacked_preds = np.vstack((stacked_preds, preds))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.569542Z","iopub.execute_input":"2022-01-09T13:02:27.570097Z","iopub.status.idle":"2022-01-09T13:02:27.578055Z","shell.execute_reply.started":"2022-01-09T13:02:27.57006Z","shell.execute_reply":"2022-01-09T13:02:27.57739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stacked_preds","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.579297Z","iopub.execute_input":"2022-01-09T13:02:27.580039Z","iopub.status.idle":"2022-01-09T13:02:27.588295Z","shell.execute_reply.started":"2022-01-09T13:02:27.580001Z","shell.execute_reply":"2022-01-09T13:02:27.587446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Take the argmax. This returns the column index of the max value in each row.\n\n# preds = np.argmax(stacked_preds, axis=1)\n\n# preds","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.589619Z","iopub.execute_input":"2022-01-09T13:02:27.590122Z","iopub.status.idle":"2022-01-09T13:02:27.596631Z","shell.execute_reply.started":"2022-01-09T13:02:27.590087Z","shell.execute_reply":"2022-01-09T13:02:27.595924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Load the sample submission.\n# # The row order in the test set and the sample submission is the same.\n\n# path = '../input/quora-insincere-questions-classification/sample_submission.csv'\n\n# df_sample = pd.read_csv(path)\n\n# print(df_sample.shape)\n\n# df_sample.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.598044Z","iopub.execute_input":"2022-01-09T13:02:27.598617Z","iopub.status.idle":"2022-01-09T13:02:27.605066Z","shell.execute_reply.started":"2022-01-09T13:02:27.59858Z","shell.execute_reply":"2022-01-09T13:02:27.604355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Assign the preds to the prediction column\n\n# df_sample['prediction'] = preds\n\n# df_sample.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.606533Z","iopub.execute_input":"2022-01-09T13:02:27.606803Z","iopub.status.idle":"2022-01-09T13:02:27.613614Z","shell.execute_reply.started":"2022-01-09T13:02:27.606769Z","shell.execute_reply":"2022-01-09T13:02:27.612798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Create a submission csv file\n# # Note that for this competition the submission file must be named submission.csv.\n# # Therefore, it won't be possible to submit this csv file for leaderboard scoring.\n# df_sample.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.616061Z","iopub.execute_input":"2022-01-09T13:02:27.616455Z","iopub.status.idle":"2022-01-09T13:02:27.622921Z","shell.execute_reply.started":"2022-01-09T13:02:27.616357Z","shell.execute_reply":"2022-01-09T13:02:27.622003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !ls","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.624593Z","iopub.execute_input":"2022-01-09T13:02:27.624966Z","iopub.status.idle":"2022-01-09T13:02:27.63157Z","shell.execute_reply.started":"2022-01-09T13:02:27.624926Z","shell.execute_reply":"2022-01-09T13:02:27.63058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Check the distribution of the predicted classes.\n\n# df_sample['prediction'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.633361Z","iopub.execute_input":"2022-01-09T13:02:27.63365Z","iopub.status.idle":"2022-01-09T13:02:27.640251Z","shell.execute_reply.started":"2022-01-09T13:02:27.63361Z","shell.execute_reply":"2022-01-09T13:02:27.639276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using GPU for RoBERTa","metadata":{}},{"cell_type":"code","source":"!pip install pytorch-lightning\n!pip install transformers\n!pip install sentencepiece\n!pip install fairseq","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:27.642168Z","iopub.execute_input":"2022-01-09T13:02:27.642534Z","iopub.status.idle":"2022-01-09T13:02:57.649192Z","shell.execute_reply.started":"2022-01-09T13:02:27.642493Z","shell.execute_reply":"2022-01-09T13:02:57.648336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# try plain data to reproduce error\nimport pytorch_lightning as pl\nfrom torch.utils.data import random_split\nfrom typing import Optional\n\ntrain_ratio = 0.8\nDATA_DIR = \"../input/quora-insincere-questions-classification/train.csv\"\ntrain = pd.read_csv(DATA_DIR)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:02:57.652527Z","iopub.execute_input":"2022-01-09T13:02:57.652759Z","iopub.status.idle":"2022-01-09T13:03:00.0707Z","shell.execute_reply.started":"2022-01-09T13:02:57.652728Z","shell.execute_reply":"2022-01-09T13:03:00.070003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class QuestionData(Dataset):\n    \"\"\"\n    Dataset class for Question analysis. \n    Every dataset using pytorch should be overwrite this class\n    This require 2 function, __len__ and __getitem__\n    \"\"\"\n    def __init__(self, data_dir):\n        \"\"\"\n        Args:\n            data_dir (string): Directory with the csv file\n        \"\"\"\n        self.df = pd.read_csv(data_dir, index_col=0)\n\n    def __len__(self):\n        \"\"\"\n        length of the dataset, i.e. number of rows in the csv file\n        Returns: int \n        \"\"\"\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        \"\"\"\n        given a row index, returns the corresponding row of the csv file\n        Returns: text (string), label (int) \n        \"\"\"\n        text = self.df[\"question_text\"][idx]\n        label = self.df[\"target\"][idx]\n\n        return text, label\n\n\nclass QuestionDataModule(pl.LightningDataModule):\n    \"\"\"\n    Module class for question analysis. this class is used to load the data to the model. \n    It is a subclass of LightningDataModule. \n    \"\"\"\n\n    def __init__(self, data_dir: str = DATA_DIR, batch_size: int = 32):\n        \"\"\"\n        Args:\n            data_dir (string): Directory with the csv file\n            batch_size (int): batch size for dataloader\n        \"\"\"\n        super().__init__()\n        self.data_dir = data_dir\n        self.batch_size = batch_size\n\n    def setup(self, stage: Optional[str] = None):\n        \"\"\"\n        Loads the data to the model. \n        the data is loaded in the setup function, so that it is loaded only once. \n        \"\"\"\n        data_full = QuestionData(self.data_dir)\n        train_size = round(len(data_full) * train_ratio)\n        val_size = len(data_full) - train_size\n        print(len(data_full), train_size, val_size)\n        self.data_train, self.data_val = random_split(data_full, [train_size, val_size])\n\n    def train_dataloader(self):\n        \"\"\"\n        Returns: dataloader for training\n        \"\"\"\n        return DataLoader(self.data_train, batch_size=self.batch_size, shuffle=True)\n\n    def val_dataloader(self):\n        \"\"\"\n        Returns: dataloader for validation\n        \"\"\"\n        return DataLoader(self.data_val, batch_size=self.batch_size, shuffle = True)\n\n# Do some Test with data\nif __name__ == \"__main__\":\n\tdm = QuestionDataModule(DATA_DIR)\n\tdm.setup()\n\tidx = 0\n\tfor item in (dm.train_dataloader()):\n\t\tprint(idx)\n\t\tprint(item)\n\t\tidx += 1\n\t\tif idx > 5: break","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:03:00.072113Z","iopub.execute_input":"2022-01-09T13:03:00.072678Z","iopub.status.idle":"2022-01-09T13:03:03.202825Z","shell.execute_reply.started":"2022-01-09T13:03:00.072639Z","shell.execute_reply":"2022-01-09T13:03:03.201511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fairseq.data import Dictionary\nimport sentencepiece as spm\nfrom os.path import join as pjoin\nfrom transformers import PreTrainedTokenizer\nimport sentencepiece as spm\n\n\nclass XLMRobertaTokenizer(PreTrainedTokenizer):\n    \"\"\"\n    XLM-RoBERTa tokenizer adapted from transformers.PreTrainedTokenizer. This helps to convert the input text into \n    tokenized format. eg, \n    \n    input: \"Hello, how are you?\" output: [\"1\", \"2\", \"3\", \"65\", \"2\", \"1\"]\n    \n    this class also provides the method to convert the tokenized format into the original text.\n    \n    eg, input: [\"1\", \"2\", \"3\", \"65\", \"2\", \"1\"] output: \"Hello, how are you?\"\n    \n    \"\"\"\n    def __init__(\n            self,\n            pretrained_file,\n            bos_token=\"<s>\",\n            eos_token=\"</s>\",\n            sep_token=\"</s>\",\n            cls_token=\"<s>\",\n            unk_token=\"<unk>\",\n            pad_token=\"<pad>\",\n            mask_token=\"<mask>\",\n            **kwargs\n    ):\n        \"\"\"\n        :param pretrained_file: path to the pretrained model file\n        :param bos_token: beginning of sentence token\n        :param eos_token: end of sentence token\n        :param sep_token: separation token\n        :param cls_token: classification token\n        :param unk_token: unknown token\n        :param pad_token: padding token\n        :param mask_token: mask token\n        \"\"\"\n        super().__init__(\n            bos_token=bos_token,\n            eos_token=eos_token,\n            unk_token=unk_token,\n            sep_token=sep_token,\n            cls_token=cls_token,\n            pad_token=pad_token,\n            mask_token=mask_token,\n            **kwargs,\n        )\n        # load bpe model and vocab file\n        sentencepiece_model = pjoin(pretrained_file, 'sentencepiece.bpe.model')\n        vocab_file = '../input/robertabase/roberta-base-dict.txt'\n        self.sp_model = spm.SentencePieceProcessor()\n        self.sp_model.Load(\n            sentencepiece_model)  # please dont use anything from sp_model bcz it makes everything goes wrong\n        self.bpe_dict = Dictionary().load(vocab_file)\n        # Mimic fairseq token-to-id alignment for the first 4 token\n        self.fairseq_tokens_to_ids = {\"<s>\": 0, \"<pad>\": 1, \"</s>\": 2, \"<unk>\": 3}\n        # The first \"real\" token \",\" has position 4 in the original fairseq vocab and position 3 in the spm vocab\n        self.fairseq_offset = 0\n        self.fairseq_tokens_to_ids[\"<mask>\"] = len(self.bpe_dict) + self.fairseq_offset\n        self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}\n\n    def _tokenize(self, text):\n        \"\"\" Tokenize a string. \"\"\"\n        return self.sp_model.EncodeAsPieces(text)\n\n    def _convert_token_to_id(self, token):\n        \"\"\" Converts a token (str) in an id using the vocab. \"\"\"\n        if token in self.fairseq_tokens_to_ids:\n            return self.fairseq_tokens_to_ids[token]\n        spm_id = self.bpe_dict.index(token)\n        return spm_id\n\n    def _convert_id_to_token(self, index):\n        \"\"\"Converts an index (integer) in a token (str) using the vocab.\"\"\"\n        if index in self.fairseq_ids_to_tokens:\n            return self.fairseq_ids_to_tokens[index]\n        return self.bpe_dict[index]\n\n    @property\n    def vocab_size(self):\n        \"\"\" Size of the base vocabulary (without the added tokens) \"\"\"\n        return len(self.bpe_dict) + self.fairseq_offset + 1  # Add the <mask> token\n\n    def get_vocab(self):\n        \"\"\" Returns the vocabulary as a list of tokens. \"\"\"\n        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}\n        vocab.update(self.added_tokens_encoder)\n        return vocab","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:03:03.204157Z","iopub.status.idle":"2022-01-09T13:03:03.204782Z","shell.execute_reply.started":"2022-01-09T13:03:03.204543Z","shell.execute_reply":"2022-01-09T13:03:03.204569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import XLMRobertaConfig, XLMRobertaForSequenceClassification\nimport torch\n\npretrained_path = '../input/xlmroberta/xlm-roberta-base' \n!ls $pretrained_path\n# load tokenizer\nroberta = XLMRobertaForSequenceClassification.from_pretrained(pretrained_path)\ntokenizer = XLMRobertaTokenizer(pretrained_path)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:03:03.20642Z","iopub.status.idle":"2022-01-09T13:03:03.206842Z","shell.execute_reply.started":"2022-01-09T13:03:03.206612Z","shell.execute_reply":"2022-01-09T13:03:03.206636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, classification_report, accuracy_score\n\n\nclass QuestionRoberta(pl.LightningModule):\n    \"\"\"\n    QuestionRoberta class inherits from LightningModule\n    This class is used to train a model using PyTorch Lightning\n    It overrides the following methods:\n        - forward : forward pass of the model\n        - training_step : training step of the model\n        - validation_step : validation step of the model\n        - validation_epoch_end : end of the validation epoch\n        - configure_optimizers : configure optimizers\n    \"\"\"\n    def __init__(self, lr_roberta, lr_classifier):\n        \"\"\"\n        Initialize the model with the following parameters:\n            - lr_roberta : learning rate of the roberta model\n            - lr_classifier : learning rate of the classifier model\n        \"\"\"\n        super().__init__()\n        self.roberta = XLMRobertaForSequenceClassification.from_pretrained(pretrained_path)\n        self.tokenizer = XLMRobertaTokenizer(pretrained_path)\n        self.lr_roberta = lr_roberta\n        self.lr_classifer = lr_classifier\n\n    def forward(self, texts, labels=None):\n        \"\"\"\n        Forward pass of the model\n        Args:\n            - texts : input texts\n            - labels : labels of the input texts\n        \"\"\"\n        inputs = self.tokenizer(texts, return_tensors='pt', padding=True, truncation=True, max_length=256)\n        for key in inputs:\n            inputs[key] = inputs[key].to(self.device)\n\n        outputs = self.roberta(**inputs, labels=labels)\n        return outputs\n\n    def configure_optimizers(self):\n        \"\"\"\n        Configure optimizers\n        This method is used to configure the optimizers of the model by using the learning rate\n        for specific parameter of the roberta model and the classifier model\n        \"\"\"\n        roberta_params = self.roberta.roberta.named_parameters()\n        classifier_params = self.roberta.classifier.named_parameters()\n\n        grouped_params = [\n            {\"params\": [p for n, p in roberta_params], \"lr\": self.lr_roberta},\n            {\"params\": [p for n, p in classifier_params], \"lr\": self.lr_classifer}\n        ]\n        optimizer = torch.optim.AdamW(\n            grouped_params\n        )\n        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1000, gamma=0.98)\n        return {\n            'optimizer': optimizer,\n            'lr_scheduler': {\n                'scheduler': scheduler,\n                'monitor': 'f1/val',\n            }\n        }\n\n    def training_step(self, batch, batch_idx):\n        \"\"\"\n        Training step of the model\n        Args:\n            - batch : batch of the data\n            - batch_idx : index of the batch\n        \"\"\"\n        texts, labels = batch\n        outputs = self(texts, labels=labels)\n\n        if len(outputs.values()) == 3:\n            loss, logits, _ = outputs.values()\n        else:\n            loss, logits = outputs.values()\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        \"\"\"\n        Validation step of the model, used to compute the metrics\n        Args:\n            - batch : batch of the data\n            - batch_idx : index of the batch\n        \"\"\"\n        texts, labels = batch\n        outputs = self(texts, labels=labels)\n\n        if len(outputs.values()) == 3:\n            loss, logits, _ = outputs.values()\n        else:\n            loss, logits = outputs.values()\n\n        output_scores = torch.softmax(logits, dim=-1)\n        return loss, output_scores, labels\n\n    def validation_epoch_end(self, validation_step_outputs):\n        \"\"\"\n        End of the validation epoch, this method will be called at the end of the validation epoch,\n        it will compute the multiple metrics of classification problem\n        Args:\n            - validation_step_outputs : outputs of the validation step\n        \"\"\"\n\n        val_preds = torch.tensor([], device=self.device)\n        val_scores = torch.tensor([], device=self.device)\n        val_labels = torch.tensor([], device=self.device)\n        val_loss = 0\n        total_item = 0\n\n        for idx, item in enumerate(validation_step_outputs):\n            loss, output_scores, labels = item\n\n            predictions = torch.argmax(output_scores, dim=-1)\n            val_preds = torch.cat((val_preds, predictions), dim=0)\n            val_scores = torch.cat((val_scores, output_scores[:, 1]), dim=0)\n            val_labels = torch.cat((val_labels, labels), dim=0)\n\n            val_loss += loss\n            total_item += 1\n\n        # print(\"VAL PREDS\", val_preds.shape)\n        # print(\"VAL SCORES\", val_scores.shape)\n        # print(\"VAL LABELS\", val_labels.shape)\n        val_preds = val_preds.cpu().numpy()\n        val_scores = val_scores.cpu().numpy()\n        val_labels = val_labels.cpu().numpy()\n\n        reports = classification_report(val_labels, val_preds, output_dict=True)\n        print(\"VAL LABELS\", val_labels)\n        print(\"VAL SCORES\", val_scores)\n        try:\n            auc = roc_auc_score(val_labels, val_scores)\n        except Exception as e:\n            print(e)\n            print(\"Cannot calculate AUC. Default to 0\")\n            auc = 0\n        accuracy = accuracy_score(val_labels, val_preds)\n\n        print(classification_report(val_labels, val_preds))\n\n        self.log(\"loss/val\", val_loss)\n        self.log(\"auc/val\", auc)\n        self.log(\"accuracy/val\", accuracy)\n        self.log(\"precision/val\", reports[\"weighted avg\"][\"precision\"])\n        self.log(\"recall/val\", reports[\"weighted avg\"][\"recall\"])\n        self.log(\"f1/val\", reports[\"weighted avg\"][\"f1-score\"])","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:03:03.208297Z","iopub.status.idle":"2022-01-09T13:03:03.208695Z","shell.execute_reply.started":"2022-01-09T13:03:03.208477Z","shell.execute_reply":"2022-01-09T13:03:03.2085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = pl.Trainer(\n    fast_dev_run=True,\n)\nmodel = QuestionRoberta(lr_roberta=1e-5, lr_classifier=3e-3)\ndm = QuestionDataModule()\n\ntrainer.fit(model, dm)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:03:03.210001Z","iopub.status.idle":"2022-01-09T13:03:03.210428Z","shell.execute_reply.started":"2022-01-09T13:03:03.210197Z","shell.execute_reply":"2022-01-09T13:03:03.21022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_lightning import loggers as pl_loggers\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint\n\ntorch.manual_seed(123)\n\ntb_logger = pl_loggers.TensorBoardLogger('/tb_logs/')\n\ntrainer = pl.Trainer(\n    min_epochs=1,\n    max_epochs=2,\n    gpus=1,\n    precision=16,\n    val_check_interval=0.5,\n    # check_val_every_n_epoch=1,\n    callbacks=[\n      ModelCheckpoint(\n          dirpath='/ckpt',\n          save_top_k=3,\n          monitor='f1/val',\n      ), \n      EarlyStopping('f1/val', patience=5)\n    ],\n    fast_dev_run=False,\n    logger=tb_logger\n)\n\ndm.setup(stage=\"fit\")\ntrainer.fit(model, dm)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:03:03.212823Z","iopub.status.idle":"2022-01-09T13:03:03.214492Z","shell.execute_reply.started":"2022-01-09T13:03:03.21424Z","shell.execute_reply":"2022-01-09T13:03:03.214267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Kaggle banned Tensorboard. Got my account locked for days.\n# %reload_ext tensorboard\n# %tensorboard --logdir '/tb_logs/'","metadata":{"execution":{"iopub.status.busy":"2022-01-09T13:03:03.215718Z","iopub.status.idle":"2022-01-09T13:03:03.216358Z","shell.execute_reply.started":"2022-01-09T13:03:03.216103Z","shell.execute_reply":"2022-01-09T13:03:03.216127Z"},"trusted":true},"execution_count":null,"outputs":[]}]}