{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About this notebook\nCopied from [Y.Nakama's Notebook](https://www.kaggle.com/yasufuminakama/jigsaw4-luke-base-starter-sub)\nThanks a lot!!\n\n- Add data\n    - measureing hate speech [André Moura's dataset](https://www.kaggle.com/andre112/measuring-hate-speech)\n    - toxic public dataframes [Deep Learner's dataset](https://www.kaggle.com/readoc/toxic-public-dataframes)\n\nCV: 0.8050, PrivateLeaderboard: 0.8011\n\nbelow is Nakama's references\n- [Luke](https://arxiv.org/pdf/2010.01057v1.pdf) - base starter notebook\n- [Inference notebook](https://www.kaggle.com/yasufuminakama/jigsaw4-luke-base-starter-sub)\n- Approach References\n    - https://www.kaggle.com/c/jigsaw-toxic-severity-rating/discussion/286471\n    - https://www.kaggle.com/debarshichanda/pytorch-w-b-jigsaw-starter\n    - https://www.kaggle.com/debarshichanda/0-816-jigsaw-inference\n    - Thanks for sharing @debarshichanda","metadata":{}},{"cell_type":"markdown","source":"# Directory settings","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Directory settings\n# ====================================================\nimport os\n\nOUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:44:52.161346Z","iopub.execute_input":"2022-02-06T11:44:52.161629Z","iopub.status.idle":"2022-02-06T11:44:52.185971Z","shell.execute_reply.started":"2022-02-06T11:44:52.161552Z","shell.execute_reply":"2022-02-06T11:44:52.185342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CFG","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    competition='Jigsaw4-2'\n    _wandb_kernel='HideBu'\n    debug=False\n    apex=True\n    print_freq=50\n    num_workers=4\n    model=\"studio-ousia/luke-base\"\n    scheduler='cosine' # ['linear', 'cosine']\n    batch_scheduler=True\n    num_cycles=0.55\n    num_warmup_steps=0\n    epochs=3 # 3\n    encoder_lr=1e-5\n    decoder_lr=1e-5\n    min_lr=1e-6\n    eps=1e-6\n    betas=(0.9, 0.999)\n    batch_size=64 # 64\n    fc_dropout=0.2 # 0\n    text=\"text\"\n    target=\"target\"\n    target_size=1\n    head=32 # 32\n    tail=32 # 32\n    max_len=head+tail\n    weight_decay=0.01\n    gradient_accumulation_steps=1\n    max_grad_norm=1000\n    margin=0.5\n    seed=42\n    n_fold=7 # 5\n    trn_fold=[0, 1, 2, 3, 4, 5, 6]\n    train=True","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:45:09.662815Z","iopub.execute_input":"2022-02-06T11:45:09.663494Z","iopub.status.idle":"2022-02-06T11:45:09.671288Z","shell.execute_reply.started":"2022-02-06T11:45:09.663456Z","shell.execute_reply":"2022-02-06T11:45:09.670067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# wandb\n# ====================================================\nimport wandb\n\ntry:\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    secret_value_0 = user_secrets.get_secret(\"dadf9abee2eb795f94d957e7136922bb0bcc3c6c\")\n    wandb.login(key=secret_value_0)\n    anony = None\nexcept:\n    anony = \"must\"\n    print('If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \\nGet your W&B access token from here: https://wandb.ai/authorize')\n\n    \ndef class2dict(f):\n    return dict((name, getattr(f, name)) for name in dir(f) if not name.startswith('__'))\n\nrun = wandb.init(project='Jigsaw4-Public2', \n                 name=CFG.model,\n                 config=class2dict(CFG),\n                 group=CFG.model,\n                 job_type=\"train\",\n                 anonymous=anony)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:45:14.993703Z","iopub.execute_input":"2022-02-06T11:45:14.994371Z","iopub.status.idle":"2022-02-06T11:45:24.230949Z","shell.execute_reply.started":"2022-02-06T11:45:14.994332Z","shell.execute_reply":"2022-02-06T11:45:24.230201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Library","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport os\nimport gc\nimport re\nimport sys\nimport json\nimport time\nimport math\nimport string\nimport pickle\nimport random\nimport joblib\nimport itertools\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn import Parameter\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.utils.data import DataLoader, Dataset\n\nos.system('pip uninstall -q transformers -y')\nos.system('pip uninstall -q tokenizers -y')\nos.system('pip uninstall -q huggingface_hub -y')\n\nos.system('mkdir -p /tmp/pip/cache-tokenizers/')\nos.system('cp ../input/tokenizers-0103/tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl /tmp/pip/cache-tokenizers/')\nos.system('pip install -q --no-index --find-links /tmp/pip/cache-tokenizers/ tokenizers')\n\nos.system('mkdir -p /tmp/pip/cache-huggingface-hub/')\nos.system('cp ../input/huggingface-hub-008/huggingface_hub-0.0.8-py3-none-any.whl /tmp/pip/cache-huggingface-hub/')\nos.system('pip install -q --no-index --find-links /tmp/pip/cache-huggingface-hub/ huggingface_hub')\n\nos.system('mkdir -p /tmp/pip/cache-transformers/')\nos.system('cp ../input/transformers-470/transformers-4.7.0-py3-none-any.whl /tmp/pip/cache-transformers/')\nos.system('pip install -q --no-index --find-links /tmp/pip/cache-transformers/ transformers')\n\nimport tokenizers\nimport transformers\nprint(f\"tokenizers.__version__: {tokenizers.__version__}\")\nprint(f\"transformers.__version__: {transformers.__version__}\")\nfrom transformers import LukeTokenizer, LukeModel, LukeConfig\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:45:24.2337Z","iopub.execute_input":"2022-02-06T11:45:24.233989Z","iopub.status.idle":"2022-02-06T11:45:54.738575Z","shell.execute_reply.started":"2022-02-06T11:45:24.23396Z","shell.execute_reply":"2022-02-06T11:45:54.737751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(df):\n    score = len(df[df['less_toxic_pred'] < df['more_toxic_pred']]) / len(df)\n    return score\n\n\ndef get_logger(filename=OUTPUT_DIR+'train'):\n    from logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=f\"{filename}.log\")\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = get_logger()\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(seed=42)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:45:54.741244Z","iopub.execute_input":"2022-02-06T11:45:54.74169Z","iopub.status.idle":"2022-02-06T11:45:54.754764Z","shell.execute_reply.started":"2022-02-06T11:45:54.741649Z","shell.execute_reply":"2022-02-06T11:45:54.754056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# text cleaning (add: 22-1-16)\n# ====================================================\nfrom bs4 import BeautifulSoup\ndef text_cleaning(text):\n    '''\n    Cleans text into a basic form for NLP. Operations include the following:-\n    1. Remove special charecters like &, #, etc\n    2. Removes extra spaces\n    3. Removes embedded URL links\n    4. Removes HTML tags\n    5. Removes emojis\n    \n    text - Text piece to be cleaned.\n    '''\n    template = re.compile(r'https?://\\S+|www\\.\\S+') #Removes website links\n    text = template.sub(r'', text)\n    \n    soup = BeautifulSoup(text, 'lxml') #Removes HTML tags\n    only_text = soup.get_text()\n    text = only_text\n    text = re.sub(r\"[^a-zA-Z\\d]\", \" \", text) #Remove special Charecters\n    text = re.sub(' +', ' ', text) #Remove Extra Spaces\n    text = text.strip() # remove spaces at the beginning and at the end of string\n\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:45:54.756394Z","iopub.execute_input":"2022-02-06T11:45:54.75694Z","iopub.status.idle":"2022-02-06T11:45:54.921601Z","shell.execute_reply.started":"2022-02-06T11:45:54.756896Z","shell.execute_reply":"2022-02-06T11:45:54.9208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FEATURE_WTS = {\n#     'severe_toxic': 1.5, 'identity_hate': 1.0, 'threat': 1.0, \n#     'insult': 1.0, 'toxic': 1.0, 'obscene': 1.0, \n# }\nFEATURE_WTS = {\n    'severe_toxic': 1.5, 'identity_hate': 1.5, 'threat': 1.5, \n    'insult': 0.64, 'toxic': 0.32, 'obscene': 0.16, \n}\nPSEUDO_LABEL_WEIGHT = 0.033\n\nFEATURES = list(FEATURE_WTS.keys())\nFEATURES","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:45:54.924144Z","iopub.execute_input":"2022-02-06T11:45:54.924513Z","iopub.status.idle":"2022-02-06T11:45:54.934138Z","shell.execute_reply.started":"2022-02-06T11:45:54.924476Z","shell.execute_reply":"2022-02-06T11:45:54.933395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport nltk\nnltk.download('omw-1.4')\nfrom nltk.stem import WordNetLemmatizer\n\nwordnet_lemmatizer = WordNetLemmatizer()\ndef replaceURL(text):\n    \"\"\" Replaces url address with \"url\" \"\"\"\n    text = re.sub('((www\\.[^\\s]+)|(https?://[^\\s]+))','url',text)\n    text = re.sub(r'#([^\\s]+)', r'\\1', text)\n    return text\n\ndef replaceAbbrev(text):\n    text = re.sub(r\"what's\", \"what is \",text)    \n    text = re.sub(r\"\\'ve\", \" have \",text)\n    text = re.sub(r\"can't\", \"cannot \",text)\n    text = re.sub(r\"n't\", \" not \",text)\n    text = re.sub(r\"i'm\", \"i am \",text)\n    text = re.sub(r\"\\'re\", \" are \",text)\n    text = re.sub(r\"\\'d\", \" would \",text)\n    text = re.sub(r\"\\'ll\", \" will \",text)\n    text = re.sub(r\"\\'scuse\", \" excuse \",text)\n    text = re.sub(r\"\\'s\", \" \",text)\n    text = re.sub(r\"FC\", \"FUCK\",text)\n    text = re.sub(r\"fc\", \"FUCK\",text) \n    return text\n\ndef removeUnicode(text):\n    \"\"\" Removes unicode strings like \"\\u002c\" and \"x96\" \"\"\"\n    text = re.sub(r'(\\\\u[0-9A-Fa-f]+)',r' ', text)       \n    text = re.sub(r'[^\\x00-\\x7f]',r' ',text)\n    return text\ndef removeRepeatPattern(text):\n    text=re.sub(r'([a-zA-Z])\\1{2,}\\b',r'\\1\\1',text)\n    text=re.sub(r'([a-zA-Z])\\1\\1{2,}\\B',r'\\1\\1\\1',text)\n    text=re.sub(r'[ ]{2,}',' ',text)\n    return text\n\ndef replaceAtUser(text):\n    \"\"\" Replaces \"@user\" with \"atUser\" \"\"\"\n    text = re.sub('@[^\\s]+','atUser',text)\n    return text\n\ndef replaceMultiToxicWords(text):\n    text = re.sub(r'(fuckfuck)','fuck fuck ',text)\n    text = re.sub(r'(f+)( *)([u|*|_]+)( *)([c|*|_]+)( *)(k)+','fuck',text)\n    text = re.sub(r'(h+)(a+)(h+)(a+)','ha ha ',text)\n    text = re.sub(r'(s+ *h+ *[i|!]+ *t+)','shit',text)\n    text = re.sub(r'\\b(n+)(i+)(g+)(a+)\\b','nigga',text)\n    text = re.sub(r'\\b(n+)([i|!]+)(g+)(e+)(r+)\\b','nigger',text)\n    text = re.sub(r'\\b(d+)(o+)(u+)(c+)(h+)(e+)( *)(b+)(a+)(g+)\\b','douchebag',text)\n    text = re.sub(r'([a|@][$|s][s|$])','ass',text)\n    text = re.sub(r'(\\bfuk\\b)','fuck',text)\n    return text\n\ndef removeNumbers(text):\n    \"\"\" Removes integers \"\"\"\n    text = re.sub(r\"(^|\\W)\\d+\", \" \", text)\n    text = re.sub(\"5\",\"s\",text)\n    text = re.sub(\"1\",\"i\",text)\n    text = re.sub(\"0\",\"o\",text)\n    return text\n                  \ndef replaceMultiPunc(text):\n    text=re.sub(r'([!])\\1\\1{2,}',r' mxm ',text)\n    text=re.sub(r'([?])\\1\\1{2,}',r' mqm ',text)\n    text=re.sub(r'([*])\\1\\1{2,}',r'*',text)\n    return text\n\n\nreplace_pun = {}\nseparators = set('\"%&\\'()+,-./:;<=>@[\\\\]^_`{|}~')\nfor punc in separators:\n    replace_pun[punc] = ' '\nreplace_pun['&']=' and '\n\ndef my_cleaner(s):\n    #s = s.lower()\n    s=replaceURL(s)\n    s=removeUnicode(s)\n    s=removeNumbers(s)\n    s=replaceAbbrev(s)\n    s=replaceMultiToxicWords(s)\n    s=replaceMultiPunc(s)\n    s=removeRepeatPattern(s)\n    \n    for punc in separators:\n        s= s.replace(punc,replace_pun[punc])                   # remove & replace punctuations\n    tokens = nltk.tokenize.word_tokenize(s)                    # split a string into words (tokens)\n    tokens = [wordnet_lemmatizer.lemmatize(t) for t in tokens]\n    return ' '.join(tokens)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:45:54.935624Z","iopub.execute_input":"2022-02-06T11:45:54.936013Z","iopub.status.idle":"2022-02-06T11:45:56.459029Z","shell.execute_reply.started":"2022-02-06T11:45:54.935984Z","shell.execute_reply":"2022-02-06T11:45:56.458271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loading","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Data Loading\n# ====================================================\njigsaw_train = pd.read_csv('../input/jigsaw-toxic-severity-rating/validation_data.csv')\nif CFG.debug:\n    igsaw_train = igsaw_train.sample(n=100, random_state=CFG.seed).reset_index(drop=True)\ntest = pd.read_csv('../input/jigsaw-toxic-severity-rating/comments_to_score.csv')\nsubmission = pd.read_csv('../input/jigsaw-toxic-severity-rating/sample_submission.csv')\n\nprint(jigsaw_train.shape)\ndisplay(jigsaw_train.head())\nprint(test.shape, submission.shape)\ndisplay(test.head())\ndisplay(submission.head())","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:45:56.460906Z","iopub.execute_input":"2022-02-06T11:45:56.461164Z","iopub.status.idle":"2022-02-06T11:45:57.06238Z","shell.execute_reply.started":"2022-02-06T11:45:56.461128Z","shell.execute_reply":"2022-02-06T11:45:57.06175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_duplicate = jigsaw_train.duplicated(subset=['less_toxic', 'more_toxic'])\nprint(\"Duplicates: {} %\".format(round(is_duplicate.mean() * 100, 2)))","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:47:02.480397Z","iopub.execute_input":"2022-02-06T11:47:02.480945Z","iopub.status.idle":"2022-02-06T11:47:02.543973Z","shell.execute_reply.started":"2022-02-06T11:47:02.480912Z","shell.execute_reply":"2022-02-06T11:47:02.543232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jigsaw_train[is_duplicate].head(6)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:47:53.241153Z","iopub.execute_input":"2022-02-06T11:47:53.241778Z","iopub.status.idle":"2022-02-06T11:47:53.260407Z","shell.execute_reply.started":"2022-02-06T11:47:53.241741Z","shell.execute_reply":"2022-02-06T11:47:53.259759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jigsaw_train.drop_duplicates(subset=['less_toxic', 'more_toxic'], keep='last', inplace=True)\njigsaw_train.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:51:43.020326Z","iopub.execute_input":"2022-02-06T11:51:43.020623Z","iopub.status.idle":"2022-02-06T11:51:43.077176Z","shell.execute_reply.started":"2022-02-06T11:51:43.020593Z","shell.execute_reply":"2022-02-06T11:51:43.076481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jigsaw_train[jigsaw_train.worker==313].head()","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:51:50.442269Z","iopub.execute_input":"2022-02-06T11:51:50.442929Z","iopub.status.idle":"2022-02-06T11:51:50.458022Z","shell.execute_reply.started":"2022-02-06T11:51:50.442884Z","shell.execute_reply":"2022-02-06T11:51:50.456279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jigsaw_train.groupby(\"worker\").count()","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:51:58.23909Z","iopub.execute_input":"2022-02-06T11:51:58.239625Z","iopub.status.idle":"2022-02-06T11:51:58.261375Z","shell.execute_reply.started":"2022-02-06T11:51:58.239584Z","shell.execute_reply":"2022-02-06T11:51:58.260701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Old_Train(jigsaw-toxic-comment-classification-challenge)\n# ====================================================\nold_train = pd.read_csv('../input/d/julian3833/jigsaw-toxic-comment-classification-challenge/train.csv')\ndisplay(old_train.shape)\ndisplay(old_train.head(3))","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:52:08.204496Z","iopub.execute_input":"2022-02-06T11:52:08.204767Z","iopub.status.idle":"2022-02-06T11:52:09.895376Z","shell.execute_reply.started":"2022-02-06T11:52:08.204737Z","shell.execute_reply":"2022-02-06T11:52:09.894627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"old_train['y'] = 0\nfor feat, wt in FEATURE_WTS.items(): \n    old_train.y += wt*old_train[feat]\nold_train.y = old_train.y/old_train.y.max()\ndisplay(old_train.head(3))","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:52:11.965205Z","iopub.execute_input":"2022-02-06T11:52:11.965459Z","iopub.status.idle":"2022-02-06T11:52:11.994347Z","shell.execute_reply.started":"2022-02-06T11:52:11.96543Z","shell.execute_reply":"2022-02-06T11:52:11.9937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_duplicate = old_train.duplicated(subset='comment_text')\nprint(\"Duplicates: {} %\".format(round(is_duplicate.mean() * 100, 2)))","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:53:02.247634Z","iopub.execute_input":"2022-02-06T11:53:02.247915Z","iopub.status.idle":"2022-02-06T11:53:02.458346Z","shell.execute_reply.started":"2022-02-06T11:53:02.247881Z","shell.execute_reply":"2022-02-06T11:53:02.45684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"old_train.y.hist()","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:53:19.783785Z","iopub.execute_input":"2022-02-06T11:53:19.784216Z","iopub.status.idle":"2022-02-06T11:53:20.157313Z","shell.execute_reply.started":"2022-02-06T11:53:19.784173Z","shell.execute_reply":"2022-02-06T11:53:20.15666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pos = old_train[old_train.y>0]\nneg = old_train[old_train.y==0].sample(len(pos), random_state=201)\nold_train = pd.concat([pos, neg])\ndisplay(old_train.shape)\ndisplay(old_train.head(3))\nold_train.y.hist()","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:53:33.678915Z","iopub.execute_input":"2022-02-06T11:53:33.679641Z","iopub.status.idle":"2022-02-06T11:53:33.945569Z","shell.execute_reply.started":"2022-02-06T11:53:33.679604Z","shell.execute_reply":"2022-02-06T11:53:33.944915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# concat dataframes\n# ====================================================\nold_df = pd.DataFrame(columns=[\"worker\", \"less_toxic\", \"more_toxic\"])\nold_df[\"less_toxic\"] = neg.reset_index(drop=True, inplace=False)[\"comment_text\"]\nold_df[\"more_toxic\"] = pos.reset_index(drop=True, inplace=False)[\"comment_text\"]\n\nold_df.reset_index(drop=True, inplace=True)\nold_df[\"worker\"] = old_df.index//16 + 753\nold_df","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:53:41.595781Z","iopub.execute_input":"2022-02-06T11:53:41.596058Z","iopub.status.idle":"2022-02-06T11:53:41.626281Z","shell.execute_reply.started":"2022-02-06T11:53:41.596027Z","shell.execute_reply":"2022-02-06T11:53:41.62562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pos[pos.id==\"0002bcb3da6cb337\"]","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:53:43.583202Z","iopub.execute_input":"2022-02-06T11:53:43.583867Z","iopub.status.idle":"2022-02-06T11:53:43.604448Z","shell.execute_reply.started":"2022-02-06T11:53:43.583818Z","shell.execute_reply":"2022-02-06T11:53:43.603722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# measuring_hate_speech\n# ====================================================\nhspeech = pd.read_csv('../input/measuring-hate-speech/measuring_hate_speech.csv')\ndisplay(hspeech.shape)\ndisplay(hspeech.head(3))","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:53:48.604945Z","iopub.execute_input":"2022-02-06T11:53:48.605503Z","iopub.status.idle":"2022-02-06T11:53:52.824407Z","shell.execute_reply.started":"2022-02-06T11:53:48.605466Z","shell.execute_reply":"2022-02-06T11:53:52.823694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get mean scores for each comment_id\nscores_dict = hspeech.groupby('comment_id')['hate_speech_score'].apply(np.mean).to_dict()\n\n# drop duplicate comment_ids\nhspeech = hspeech.drop_duplicates(subset='comment_id')\nhspeech['hate_speech_score'] = hspeech['comment_id'].map(scores_dict)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:54:06.606423Z","iopub.execute_input":"2022-02-06T11:54:06.606685Z","iopub.status.idle":"2022-02-06T11:54:10.043742Z","shell.execute_reply.started":"2022-02-06T11:54:06.606658Z","shell.execute_reply":"2022-02-06T11:54:10.043012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hspeech['hate_speech_score'].plot.hist(bins=100, title='Hate speech scores')","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:54:11.441103Z","iopub.execute_input":"2022-02-06T11:54:11.441373Z","iopub.status.idle":"2022-02-06T11:54:11.831157Z","shell.execute_reply.started":"2022-02-06T11:54:11.441344Z","shell.execute_reply":"2022-02-06T11:54:11.830414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hspeech = hspeech[['comment_id', 'text','hate_speech_score']]\nhspeech.rename(columns={\n    \"comment_id\":\"id\", \n    \"text\":\"comment_text\", \n    \"hate_speech_score\":\"y\"\n},inplace=True)\n\nhspeech.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:54:21.723249Z","iopub.execute_input":"2022-02-06T11:54:21.72402Z","iopub.status.idle":"2022-02-06T11:54:21.742752Z","shell.execute_reply.started":"2022-02-06T11:54:21.723968Z","shell.execute_reply":"2022-02-06T11:54:21.741685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(hspeech.shape)\nhspeech.y.hist()","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:55:19.46846Z","iopub.execute_input":"2022-02-06T11:55:19.468715Z","iopub.status.idle":"2022-02-06T11:55:19.898415Z","shell.execute_reply.started":"2022-02-06T11:55:19.468685Z","shell.execute_reply":"2022-02-06T11:55:19.89776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pos = hspeech[hspeech.y<(-2)]\nneg = hspeech[hspeech.y>0].sample(len(pos), random_state=201)\nhspeech = pd.concat([pos, neg])\ndisplay(hspeech.shape)\ndisplay(hspeech.head(3))\nhspeech.y.hist()","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:55:20.798979Z","iopub.execute_input":"2022-02-06T11:55:20.799471Z","iopub.status.idle":"2022-02-06T11:55:21.05239Z","shell.execute_reply.started":"2022-02-06T11:55:20.799436Z","shell.execute_reply":"2022-02-06T11:55:21.051722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# concat dataframes\n# ====================================================\nhate_df = pd.DataFrame(columns=[\"worker\", \"less_toxic\", \"more_toxic\"])\nhate_df[\"less_toxic\"] = neg.reset_index(drop=True, inplace=False)[\"comment_text\"]\nhate_df[\"more_toxic\"] = pos.reset_index(drop=True, inplace=False)[\"comment_text\"]\n\nhate_df.reset_index(drop=True, inplace=True)\nhate_df[\"worker\"] = hate_df.index//16 + 1768\nhate_df","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:55:32.315357Z","iopub.execute_input":"2022-02-06T11:55:32.315609Z","iopub.status.idle":"2022-02-06T11:55:32.344416Z","shell.execute_reply.started":"2022-02-06T11:55:32.315581Z","shell.execute_reply":"2022-02-06T11:55:32.343752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.concat([jigsaw_train, old_df])\ntrain = pd.concat([train, hate_df])\n\ntrain.reset_index(drop=True, inplace=True)\ntrain[\"less_toxic\"] = train[\"less_toxic\"].apply(my_cleaner)\ntrain[\"more_toxic\"] = train[\"more_toxic\"].apply(my_cleaner)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:55:33.049017Z","iopub.execute_input":"2022-02-06T11:55:33.049538Z","iopub.status.idle":"2022-02-06T11:57:12.4566Z","shell.execute_reply.started":"2022-02-06T11:55:33.049501Z","shell.execute_reply":"2022-02-06T11:57:12.455918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del jigsaw_train, old_df, pos, neg, hspeech, hate_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:57:12.458225Z","iopub.execute_input":"2022-02-06T11:57:12.458625Z","iopub.status.idle":"2022-02-06T11:57:12.705071Z","shell.execute_reply.started":"2022-02-06T11:57:12.458587Z","shell.execute_reply":"2022-02-06T11:57:12.704427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CV split","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# CV split\n# ====================================================\nFold = GroupKFold(n_splits=CFG.n_fold)\nfor n, (trn_index, val_index) in enumerate(Fold.split(train, train, train['worker'])):\n    train.loc[val_index, 'fold'] = int(n)\ntrain['fold'] = train['fold'].astype(int)\ndisplay(train.groupby('fold').size())","metadata":{"execution":{"iopub.status.busy":"2022-02-06T11:57:12.706467Z","iopub.execute_input":"2022-02-06T11:57:12.706971Z","iopub.status.idle":"2022-02-06T11:57:12.760029Z","shell.execute_reply.started":"2022-02-06T11:57:12.706934Z","shell.execute_reply":"2022-02-06T11:57:12.759393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tokenizer","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# tokenizer\n# ====================================================\ntokenizer = LukeTokenizer.from_pretrained(CFG.model, lowercase=True)\ntokenizer.save_pretrained(OUTPUT_DIR+'tokenizer/')\nCFG.tokenizer = tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-01-30T13:10:27.595592Z","iopub.execute_input":"2022-01-30T13:10:27.595846Z","iopub.status.idle":"2022-01-30T13:10:34.544339Z","shell.execute_reply.started":"2022-01-30T13:10:27.595812Z","shell.execute_reply":"2022-01-30T13:10:34.5436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\ndef prepare_input(text, cfg):\n    if cfg.tail == 0:\n        inputs = cfg.tokenizer.encode_plus(text, \n                                           return_tensors=None, \n                                           add_special_tokens=True, \n                                           max_length=cfg.max_len,\n                                           pad_to_max_length=True,\n                                           truncation=True)\n        for k, v in inputs.items():\n            inputs[k] = torch.tensor(v, dtype=torch.long)\n    else:\n        inputs = cfg.tokenizer.encode_plus(text,\n                                           return_tensors=None, \n                                           add_special_tokens=True, \n                                           truncation=True)\n        for k, v in inputs.items():\n            v_length = len(v)\n            if v_length > cfg.max_len:\n                v = np.hstack([v[:cfg.head], v[-cfg.tail:]])\n            if k == 'input_ids':\n                new_v = np.ones(cfg.max_len) * cfg.tokenizer.pad_token_id\n            else:\n                new_v = np.zeros(cfg.max_len)\n            new_v[:v_length] = v \n            inputs[k] = torch.tensor(new_v, dtype=torch.long)\n    return inputs\n\n\nclass TrainDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.less_toxic = df['less_toxic'].fillna(\"none\").values\n        self.more_toxic = df['more_toxic'].fillna(\"none\").values\n\n    def __len__(self):\n        return len(self.less_toxic)\n\n    def __getitem__(self, item):\n        less_toxic_inputs = prepare_input(str(self.less_toxic[item]), self.cfg)\n        more_toxic_inputs = prepare_input(str(self.more_toxic[item]), self.cfg)\n        label = torch.tensor(1, dtype=torch.float)\n        return less_toxic_inputs, more_toxic_inputs, label\n\n\nclass TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.text = df[cfg.text].fillna(\"none\").values\n\n    def __len__(self):\n        return len(self.text)\n\n    def __getitem__(self, item):\n        text = str(self.text[item])\n        inputs = prepare_input(text, self.cfg)\n        return inputs","metadata":{"execution":{"iopub.status.busy":"2022-01-30T13:10:34.545697Z","iopub.execute_input":"2022-01-30T13:10:34.545957Z","iopub.status.idle":"2022-01-30T13:10:34.561609Z","shell.execute_reply.started":"2022-01-30T13:10:34.545924Z","shell.execute_reply":"2022-01-30T13:10:34.559906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Model\n# ====================================================\nclass CustomModel(nn.Module):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__()\n        self.cfg = cfg\n        if config_path is None:\n            self.config = LukeConfig.from_pretrained(cfg.model, output_hidden_states=True)\n        else:\n            self.config = torch.load(config_path)\n        if pretrained:\n            self.model = LukeModel.from_pretrained(cfg.model, config=self.config)\n        else:\n            self.model = LukeModel(self.config)\n        self.fc_dropout = nn.Dropout(cfg.fc_dropout)\n        self.fc = nn.Linear(self.config.hidden_size, cfg.target_size)\n        \n    def feature(self, inputs):\n        outputs = self.model(**inputs)\n        last_hidden_states = outputs[0]\n        feature = torch.mean(last_hidden_states, 1)\n        return feature\n\n    def forward(self, inputs):\n        feature = self.feature(inputs)\n        output = self.fc(self.fc_dropout(feature))\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-01-30T13:10:34.563153Z","iopub.execute_input":"2022-01-30T13:10:34.563418Z","iopub.status.idle":"2022-01-30T13:10:34.575031Z","shell.execute_reply.started":"2022-01-30T13:10:34.563384Z","shell.execute_reply":"2022-01-30T13:10:34.574364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helpler functions","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n\n\ndef train_fn(fold, train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    model.train()\n    scaler = torch.cuda.amp.GradScaler(enabled=CFG.apex)\n    losses = AverageMeter()\n    start = end = time.time()\n    global_step = 0\n    for step, (less_toxic_inputs, more_toxic_inputs, labels) in enumerate(train_loader):\n        for k, v in less_toxic_inputs.items():\n            less_toxic_inputs[k] = v.to(device)\n        for k, v in more_toxic_inputs.items():\n            more_toxic_inputs[k] = v.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        with torch.cuda.amp.autocast(enabled=CFG.apex):\n            less_toxic_y_preds = model(less_toxic_inputs)\n            more_toxic_y_preds = model(more_toxic_inputs)\n            loss = criterion(more_toxic_y_preds, less_toxic_y_preds, labels)\n        losses.update(loss.item(), batch_size)\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        scaler.scale(loss).backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)\n        if (step + 1) % CFG.gradient_accumulation_steps == 0:\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            global_step += 1\n            if CFG.batch_scheduler:\n                scheduler.step()\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n            print('Epoch: [{0}][{1}/{2}] '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  'Grad: {grad_norm:.4f}  '\n                  'LR: {lr:.8f}  '\n                  .format(epoch+1, step, len(train_loader), \n                          remain=timeSince(start, float(step+1)/len(train_loader)),\n                          loss=losses,\n                          grad_norm=grad_norm,\n                          lr=scheduler.get_lr()[0]))\n        wandb.log({f\"[fold{fold}] loss\": losses.val,\n                   f\"[fold{fold}] lr\": scheduler.get_lr()[0]})\n    return losses.avg\n\n\ndef inference_fn(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    tk0 = tqdm(test_loader, total=len(test_loader))\n    for inputs in tk0:\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n        with torch.no_grad():\n            y_preds = model(inputs)\n        preds.append(y_preds.sigmoid().to('cpu').numpy())\n    predictions = np.concatenate(preds)\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-01-30T13:10:34.576492Z","iopub.execute_input":"2022-01-30T13:10:34.57689Z","iopub.status.idle":"2022-01-30T13:10:34.597743Z","shell.execute_reply.started":"2022-01-30T13:10:34.576856Z","shell.execute_reply":"2022-01-30T13:10:34.597096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# train loop\n# ====================================================\ndef train_loop(folds, fold):\n    \n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n\n    # ====================================================\n    # loader\n    # ====================================================\n    \n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n    \n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    validation = folds.loc[val_idx].reset_index(drop=True)\n    \n    valid_folds = sorted(set(validation['less_toxic'].unique()) | set(validation['more_toxic'].unique()))\n    valid_folds = pd.DataFrame({'text': valid_folds}).reset_index()\n    \n    train_dataset = TrainDataset(CFG, train_folds)\n    valid_dataset = TestDataset(CFG, valid_folds)\n\n    train_loader = DataLoader(train_dataset,\n                              batch_size=CFG.batch_size,\n                              shuffle=True,\n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset,\n                              batch_size=CFG.batch_size,\n                              shuffle=False,\n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n\n    # ====================================================\n    # model & optimizer\n    # ====================================================\n    model = CustomModel(CFG, config_path=None, pretrained=True)\n    torch.save(model.config, OUTPUT_DIR+'config.pth')\n    model.to(device)\n    \n    def get_optimizer_params(model, encoder_lr, decoder_lr, weight_decay=0.0):\n        param_optimizer = list(model.named_parameters())\n        no_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\n        optimizer_parameters = [\n            {'params': [p for n, p in model.model.named_parameters() if not any(nd in n for nd in no_decay)],\n             'lr': encoder_lr, 'weight_decay': weight_decay},\n            {'params': [p for n, p in model.model.named_parameters() if any(nd in n for nd in no_decay)],\n             'lr': encoder_lr, 'weight_decay': 0.0},\n            {'params': [p for n, p in model.named_parameters() if \"model\" not in n],\n             'lr': decoder_lr, 'weight_decay': 0.0}\n        ]\n        return optimizer_parameters\n\n    optimizer_parameters = get_optimizer_params(model,\n                                                encoder_lr=CFG.encoder_lr, \n                                                decoder_lr=CFG.decoder_lr,\n                                                weight_decay=CFG.weight_decay)\n    optimizer = AdamW(optimizer_parameters, lr=CFG.encoder_lr, eps=CFG.eps, betas=CFG.betas)\n    \n    # ====================================================\n    # scheduler\n    # ====================================================\n    def get_scheduler(cfg, optimizer, num_train_steps):\n        if cfg.scheduler=='linear':\n            scheduler = get_linear_schedule_with_warmup(\n                optimizer, num_warmup_steps=cfg.num_warmup_steps, num_training_steps=num_train_steps\n            )\n        elif cfg.scheduler=='cosine':\n            scheduler = get_cosine_schedule_with_warmup(\n                optimizer, num_warmup_steps=cfg.num_warmup_steps, num_training_steps=num_train_steps, num_cycles=cfg.num_cycles\n            )\n        return scheduler\n    \n    num_train_steps = int(len(train_folds) / CFG.batch_size * CFG.epochs)\n    scheduler = get_scheduler(CFG, optimizer, num_train_steps)\n\n    # ====================================================\n    # loop\n    # ====================================================\n    criterion = nn.MarginRankingLoss(margin=CFG.margin)\n    \n    best_score = 0.\n\n    for epoch in range(CFG.epochs):\n\n        start_time = time.time()\n\n        # train\n        avg_loss = train_fn(fold, train_loader, model, criterion, optimizer, epoch, scheduler, device)\n\n        # eval\n        preds = inference_fn(valid_loader, model, device)\n        \n        # scoring\n        valid_folds['pred'] = preds\n        if 'less_toxic_pred' in validation.columns:\n            validation = validation.drop(columns='less_toxic_pred')\n        if 'more_toxic_pred' in validation.columns:\n            validation = validation.drop(columns='more_toxic_pred')\n        rename_cols = {CFG.text: 'less_toxic', 'pred': 'less_toxic_pred'}\n        validation = validation.merge(valid_folds[[CFG.text, 'pred']].rename(columns=rename_cols), \n                                      on='less_toxic', how='left')\n        rename_cols = {CFG.text: 'more_toxic', 'pred': 'more_toxic_pred'}\n        validation = validation.merge(valid_folds[[CFG.text, 'pred']].rename(columns=rename_cols), \n                                      on='more_toxic', how='left')\n        score = get_score(validation)\n\n        elapsed = time.time() - start_time\n\n        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  time: {elapsed:.0f}s')\n        LOGGER.info(f'Epoch {epoch+1} - Score: {score:.4f}')\n        wandb.log({f\"[fold{fold}] epoch\": epoch+1, \n                   f\"[fold{fold}] avg_train_loss\": avg_loss, \n                   f\"[fold{fold}] score\": score})\n        \n        if score > best_score:\n            best_score = score\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {score:.4f} Model')\n            torch.save({'model': model.state_dict(),\n                        'preds': preds},\n                        OUTPUT_DIR+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\")\n\n    preds = torch.load(OUTPUT_DIR+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\", \n                       map_location=torch.device('cpu'))['preds']\n    valid_folds['pred'] = preds\n    if 'less_toxic_pred' in validation.columns:\n        validation = validation.drop(columns='less_toxic_pred')\n    if 'more_toxic_pred' in validation.columns:\n        validation = validation.drop(columns='more_toxic_pred')\n    rename_cols = {CFG.text: 'less_toxic', 'pred': 'less_toxic_pred'}\n    validation = validation.merge(valid_folds[[CFG.text, 'pred']].rename(columns=rename_cols), \n                                  on='less_toxic', how='left')\n    rename_cols = {CFG.text: 'more_toxic', 'pred': 'more_toxic_pred'}\n    validation = validation.merge(valid_folds[[CFG.text, 'pred']].rename(columns=rename_cols), \n                                  on='more_toxic', how='left')\n\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    return validation","metadata":{"execution":{"iopub.status.busy":"2022-01-30T13:10:34.599125Z","iopub.execute_input":"2022-01-30T13:10:34.599633Z","iopub.status.idle":"2022-01-30T13:10:34.63858Z","shell.execute_reply.started":"2022-01-30T13:10:34.599596Z","shell.execute_reply":"2022-01-30T13:10:34.637662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"code","source":"if __name__ == '__main__':\n    \n    def get_result(oof_df):\n        score = get_score(oof_df)\n        LOGGER.info(f'Score: {score:<.4f}')\n    \n    if CFG.train:\n        # train \n        oof_df = pd.DataFrame()\n        for fold in range(CFG.n_fold):\n            if fold in CFG.trn_fold:\n                _oof_df = train_loop(train, fold)\n                oof_df = pd.concat([oof_df, _oof_df])\n                LOGGER.info(f\"========== fold: {fold} result ==========\")\n                get_result(_oof_df)\n        oof_df = oof_df.reset_index(drop=True)\n        # CV result\n        LOGGER.info(f\"========== CV ==========\")\n        get_result(oof_df)\n        # save result\n        oof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)\n    \n    wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T13:10:34.640181Z","iopub.execute_input":"2022-01-30T13:10:34.640772Z","iopub.status.idle":"2022-01-30T13:11:15.416093Z","shell.execute_reply.started":"2022-01-30T13:10:34.640655Z","shell.execute_reply":"2022-01-30T13:11:15.414993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}