{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"- Multi Label Learning using jigsaw-toxic-comment-classification-challenge training dataset\n- reference\n    - https://www.kaggle.com/yasufuminakama/jigsaw4-luke-base-starter-train/notebook\n    - https://www.kaggle.com/quincyqiang/download-huggingface-pretrain-for-kaggle\n    - https://www.kaggle.com/kishalmandal/jigsaw-fit-multi-label-comment-classifier\n \n \ninference notebook is [here](https://www.kaggle.com/mst8823/roberta-base-with-jigsaw-toxic-comment-infer)\n","metadata":{}},{"cell_type":"code","source":"# ========================================\n# Config\n# ========================================\nclass Config:\n    name = \"roberta-base-with-Jigsaw-Toxic-Comments\"  \n    only_inference = False\n    model_name = \"roberta-base\"\n    verbose = 200\n    max_length = 256\n    lr = 1e-5\n    weight_decay = 1e-5\n    epochs = 2\n    gradient_accumulation_steps = 1\n    max_grad_norm = 1000 \n    batch_scheduler = True\n    \n    scheduler = dict(\n        scheduler=\"cosine_scedule_with_warmup\", \n        num_warmup_steps=100, \n        num_training_steps=None)\n\n    train_batch_size = 32\n    valid_batch_size = 256\n    test_batch_size = 256\n    \n    num_workers = 2\n    is_higher_score_better = True  #  Jigsaw01 metrics\n    n_fold = 2\n    trn_fold = [0, 1]\n    seed = 2022\n    target_cols = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n    debug = False\n\n    # Colab Env\n    submit_from_colab = False\n    upload_from_colab = False\n    api_path = \"/content/drive/MyDrive/competition/kaggle.json\"\n    drive_path = \"/content/drive/MyDrive/competition/Jigsaw-Rate-Severity-of-Toxic-Comments\"\n    \n    # Kaggle Env\n    kaggle_dataset_path = None\n\nif Config.debug:\n    Config.epochs = 2\n    Config.trn_fold = [0]","metadata":{"execution":{"iopub.status.busy":"2021-11-18T03:46:19.724909Z","iopub.execute_input":"2021-11-18T03:46:19.726282Z","iopub.status.idle":"2021-11-18T03:46:19.754555Z","shell.execute_reply.started":"2021-11-18T03:46:19.726151Z","shell.execute_reply":"2021-11-18T03:46:19.753370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ========================================\n# Library\n# ========================================\nimport os\nimport json\nimport warnings\nimport shutil\nimport logging\nimport joblib\nimport random\nimport datetime\nimport sys\nimport math\nimport time\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import (\n    CosineAnnealingWarmRestarts,\n    CosineAnnealingLR, \n    ReduceLROnPlateau\n    )","metadata":{"execution":{"iopub.status.busy":"2021-11-18T03:46:19.756644Z","iopub.execute_input":"2021-11-18T03:46:19.756994Z","iopub.status.idle":"2021-11-18T03:46:22.266994Z","shell.execute_reply.started":"2021-11-18T03:46:19.756950Z","shell.execute_reply":"2021-11-18T03:46:22.265750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ========================================\n# Utils\n# ========================================\nclass Logger:\n    \"\"\"save log\"\"\"\n    def __init__(self, path):\n        self.general_logger = logging.getLogger(path)\n        stream_handler = logging.StreamHandler()\n        file_general_handler = logging.FileHandler(os.path.join(path, 'Experiment.log'))\n        if len(self.general_logger.handlers) == 0:\n            self.general_logger.addHandler(stream_handler)\n            self.general_logger.addHandler(file_general_handler)\n            self.general_logger.setLevel(logging.INFO)\n\n    def info(self, message):\n        # display time\n        self.general_logger.info('[{}] - {}'.format(self.now_string(), message))\n\n    @staticmethod\n    def now_string():\n        return str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))\n    \n    \ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\ndef read_csv(filepath, **kwargs):\n    \n    if os.path.isdir(filepath):\n        filename = filepath.split(\"/\")[-1]\n        filepath = os.path.join(filepath, filename)\n        \n    try:\n        csv_data = pd.read_csv(filepath,  **kwargs)\n    except:\n        csv_data = pd.read_csv(filepath + \".zip\",  **kwargs)\n\n    return csv_data","metadata":{"execution":{"iopub.status.busy":"2021-11-18T03:46:22.268891Z","iopub.execute_input":"2021-11-18T03:46:22.269251Z","iopub.status.idle":"2021-11-18T03:46:22.281173Z","shell.execute_reply.started":"2021-11-18T03:46:22.269205Z","shell.execute_reply":"2021-11-18T03:46:22.280055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ========================================\n# SetUp\n# ========================================\nCOLAB = \"google.colab\" in sys.modules\n\nif COLAB:\n    print(\"This environment is Google Colab\")\n    # import library\n    ! pip install --quiet transformers\n    ! pip install --quiet iterative-stratification\n\n    # mount\n    from google.colab import drive\n    if not os.path.isdir(\"/content/drive\"):\n        drive.mount('/content/drive') \n\n    # use kaggle api (need kaggle token)\n    f = open(Config.api_path, 'r')\n    json_data = json.load(f) \n    os.environ[\"KAGGLE_USERNAME\"] = json_data[\"username\"]\n    os.environ[\"KAGGLE_KEY\"] = json_data[\"key\"]\n\n    DRIVE = Config.drive_path\n    EXP = (Config.name if Config.name is not None \n           else get(\"http://172.28.0.2:9000/api/sessions\").json()[0][\"name\"][:-6])  # get notebook name\n    INPUT = os.path.join(DRIVE, \"Input\")\n    OUTPUT = os.path.join(DRIVE, \"Output\")\n    SUBMISSION = os.path.join(DRIVE, \"Submission\")\n    OUTPUT_EXP = os.path.join(OUTPUT, EXP) \n    EXP_MODEL = os.path.join(OUTPUT_EXP, \"model\")\n    EXP_FIG = os.path.join(OUTPUT_EXP, \"fig\")\n    EXP_PREDS = os.path.join(OUTPUT_EXP, \"preds\")\n\n    # all jigsaw input data\n    INPUT_JIGSAW_01 = os.path.join(INPUT, \"jigsaw-toxic-comment-classification-challenge\")\n    INPUT_JIGSAW_02 = os.path.join(INPUT, \"jigsaw-unintended-bias-in-toxicity-classification\")\n    INPUT_JIGSAW_03 = os.path.join(INPUT, \"jigsaw-multilingual-toxic-comment-classification\")\n    INPUT_JIGSAW_04 = os.path.join(INPUT, \"jigsaw-toxic-severity-rating\")\n    jigsaw_inputs = [INPUT_JIGSAW_01, INPUT_JIGSAW_02, INPUT_JIGSAW_03, INPUT_JIGSAW_04]\n\n    # make dirs\n    for d in [INPUT, SUBMISSION, EXP_MODEL, EXP_FIG, EXP_PREDS] + jigsaw_inputs:\n        os.makedirs(d, exist_ok=True)\n\n    if not os.path.isfile(os.path.join(INPUT_JIGSAW_04, \"comments_to_score.csv.zip\")):\n        # load dataset\n        ! kaggle competitions download -c jigsaw-toxic-comment-classification-challenge -p $INPUT_JIGSAW_01 \n        ! kaggle competitions download -c jigsaw-unintended-bias-in-toxicity-classification -p $INPUT_JIGSAW_02 \n        ! kaggle competitions download -c jigsaw-multilingual-toxic-comment-classification -p $INPUT_JIGSAW_03 \n        ! kaggle competitions download -c jigsaw-toxic-severity-rating -p $INPUT_JIGSAW_04 \n    \n    # utils\n    logger = Logger(OUTPUT_EXP)\n\nelse:\n    print(\"This environment is Kaggle Kernel\")\n    ! pip install --quiet ../input/iterative-stratification/iterative-stratification-master\n    \n    # download pretrain weight(https://www.kaggle.com/quincyqiang/download-huggingface-pretrain-for-kaggle/notebook)\n    if not Config.only_inference:\n        !curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash\n        !apt-get install -y --allow-unauthenticated git-lfs\n        !git lfs install\n        !git clone https://huggingface.co/$Config.model_name\n        !GIT_LFS_SKIP_SMUDGE=1\n\n    INPUT = \"../input\"\n    INPUT_JIGSAW_01 = os.path.join(INPUT, \"jigsaw-toxic-comment-classification-challenge\")\n    INPUT_JIGSAW_02 = os.path.join(INPUT, \"jigsaw-unintended-bias-in-toxicity-classification\")\n    INPUT_JIGSAW_03 = os.path.join(INPUT, \"jigsaw-multilingual-toxic-comment-classification\")\n    INPUT_JIGSAW_04 = os.path.join(INPUT, \"jigsaw-toxic-severity-rating\")\n\n    EXP, OUTPUT, SUBMISSION = \"./\", \"./\", \"./\"\n    EXP_MODEL = os.path.join(EXP, \"model\")\n    EXP_FIG = os.path.join(EXP, \"fig\")\n    EXP_PREDS = os.path.join(EXP, \"preds\")\n\n    if Config.kaggle_dataset_path is not None:\n        KD_MODEL = os.path.join(Config.kaggle_dataset_path, \"model\")\n        KD_EXP_PREDS = os.path.join(Config.kaggle_dataset_path, \"preds\")\n        shutil.copytree(KD_MODEL, EXP_MODEL)\n        shutil.copytree(KD_EXP_PREDS, EXP_PREDS)\n\n    # make dirs\n    for d in [EXP_MODEL, EXP_FIG, EXP_PREDS]:\n        os.makedirs(d, exist_ok=True)\n        \n    # utils\n    logger = Logger(EXP)\n\n# utils\nwarnings.filterwarnings(\"ignore\")\nsns.set(style='whitegrid')\nseed_everything(seed=Config.seed)\n\n# 2nd import\nfrom transformers import AutoTokenizer, AutoModel, AdamW\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nfrom transformers import (\n    get_cosine_schedule_with_warmup, \n    get_cosine_with_hard_restarts_schedule_with_warmup,\n    get_linear_schedule_with_warmup\n)\n\n# set device\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2021-11-18T03:46:22.284045Z","iopub.execute_input":"2021-11-18T03:46:22.284915Z","iopub.status.idle":"2021-11-18T03:47:34.342245Z","shell.execute_reply.started":"2021-11-18T03:46:22.284855Z","shell.execute_reply":"2021-11-18T03:47:34.341129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ========================================\n# Load Data\n# ========================================\ncomments_to_score = read_csv(os.path.join(INPUT_JIGSAW_04 , \"comments_to_score.csv\"))\nvalidation_data = read_csv(os.path.join(INPUT_JIGSAW_04 , \"validation_data.csv\"))\nsample_submission = read_csv(os.path.join(INPUT_JIGSAW_04 , \"sample_submission.csv\"))\ntrain_jigsaw_01 = read_csv(os.path.join(INPUT_JIGSAW_01 , \"train.csv\"))\n\nif Config.debug:\n    train_jigsaw_01 = train_jigsaw_01.sample(1000).reset_index(drop=True)\n\n# fisrt, add fold index\ntrain_jigsaw_01[\"fold\"] = -1\nfor i, lst in enumerate(\n    MultilabelStratifiedKFold(\n        n_splits=Config.n_fold, \n        shuffle=True,\n        random_state=Config.seed)\n    .split(X=train_jigsaw_01, y=train_jigsaw_01[Config.target_cols])):\n\n    if i in Config.trn_fold:\n        train_jigsaw_01.loc[lst[1].tolist(), \"fold\"] = i\n    \ndisplay(train_jigsaw_01)","metadata":{"execution":{"iopub.status.busy":"2021-11-18T03:52:13.480170Z","iopub.execute_input":"2021-11-18T03:52:13.481289Z","iopub.status.idle":"2021-11-18T03:52:19.605181Z","shell.execute_reply.started":"2021-11-18T03:52:13.481250Z","shell.execute_reply":"2021-11-18T03:52:19.604043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ========================================\n# Dataset\n# ========================================\nclass TrainDataset(Dataset):\n    def __init__(self, data, tokenizer, text_col):\n        self.comment_text = data[text_col].values\n        self.targets = data[Config.target_cols].values\n        self.tokenizer = tokenizer\n    \n    def __len__(self):\n        return len(self.comment_text)\n    \n    def __getitem__(self, idx):\n\n        tokenized_dict = self.tokenizer.encode_plus(\n            self.comment_text[idx],\n            truncation=True,\n            max_length=Config.max_length,\n            padding='max_length'\n        )\n        \n        input_ids = torch.LongTensor(tokenized_dict[\"input_ids\"])\n        attention_mask = torch.BoolTensor(tokenized_dict[\"attention_mask\"])\n        targets = torch.tensor(self.targets[idx]).float()\n\n        return input_ids, attention_mask, targets\n\n\nclass TestDataset(Dataset):\n    def __init__(self, data, tokenizer, text_col):\n        self.comment_text = data[text_col].values\n        self.tokenizer = tokenizer\n    \n    def __len__(self):\n        return len(self.comment_text)\n    \n    def __getitem__(self, idx):\n\n        tokenized_dict = self.tokenizer.encode_plus(\n            self.comment_text[idx],\n            truncation=True,\n            max_length=Config.max_length,\n            padding='max_length'\n        )\n        \n        input_ids = torch.LongTensor(tokenized_dict[\"input_ids\"])\n        attention_mask = torch.BoolTensor(tokenized_dict[\"attention_mask\"])\n\n        return input_ids, attention_mask","metadata":{"execution":{"iopub.status.busy":"2021-11-17T06:09:26.3815Z","iopub.execute_input":"2021-11-17T06:09:26.382272Z","iopub.status.idle":"2021-11-17T06:09:26.39252Z","shell.execute_reply.started":"2021-11-17T06:09:26.382232Z","shell.execute_reply":"2021-11-17T06:09:26.391557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ========================================\n# Model\n# ========================================\nclass Jigsaw01Model(nn.Module):\n    def __init__(self, ):\n        super(Jigsaw01Model, self).__init__()\n        \n        self.model = AutoModel.from_pretrained(Config.model_name)    \n        self.drop = nn.Dropout(p=0.2)\n        self.head = nn.Linear(768, len(Config.target_cols))\n    \n    def forward(self, input_ids, attention_mask):\n        x = self.model(input_ids=input_ids, attention_mask=attention_mask)\n        x = x[0][:, 0, :]\n        x = self.drop(x)\n        x = self.head(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-11-17T06:09:26.393871Z","iopub.execute_input":"2021-11-17T06:09:26.3947Z","iopub.status.idle":"2021-11-17T06:09:26.407561Z","shell.execute_reply.started":"2021-11-17T06:09:26.394659Z","shell.execute_reply":"2021-11-17T06:09:26.406774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ========================================\n# Funcs\n# ========================================\ndef get_score(y_true, y_pred, verbose=False):\n    print(y_true.shape, y_pred.shape)\n    scores = []\n    if Config.debug:\n        y_true[0] = 1\n        \n    for i, col_name in enumerate(Config.target_cols):\n        score_i = roc_auc_score(y_true[:, i], y_pred[:, i])\n        if verbose:\n            print(f\"{col_name}={score_i:.4f}\")\n        scores.append(score_i)\n    score = np.mean(scores)\n    return score\n\n\nclass Meter(object):\n    def __init__(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n    \n    def time_since(self, since, percent):\n        now = time.time()\n        s = now - since\n        es = s / percent\n        rs = es - s\n        return f\"{self.as_minutes(s)} (remain {self.as_minutes(rs)})\"\n    \n    @staticmethod\n    def as_minutes(s):\n        m = math.floor(s / 60)\n        s -= m * 60\n        return f\"{int(m)}m {s:.1f}s\"\n\n\ndef train_fn(\n    train_loader, \n    model, \n    criterion, \n    optimizer, \n    epoch, \n    scheduler, \n    device):\n\n    model.train()\n    losses = Meter()\n    start = end = time.time()\n    global_step = 0\n    for step, (input_ids, attention_mask, targets) in enumerate(train_loader):\n        input_ids, attention_mask = input_ids.to(device), attention_mask.to(device)\n        targets = targets.to(device)\n        batch_size = targets.size(0)\n\n        y_preds = model(input_ids=input_ids, attention_mask=attention_mask)\n        loss = criterion(y_preds, targets)\n\n        # record loss\n        losses.update(loss.item(), batch_size)\n        if Config.gradient_accumulation_steps > 1:\n            loss = loss / Config.gradient_accumulation_steps\n        \n        # backward propagation\n        loss.backward() \n        grad_norm = nn.utils.clip_grad_norm_(model.parameters(), Config.max_grad_norm)\n\n        if (step + 1) % Config.gradient_accumulation_steps == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n            global_step += 1\n            if Config.batch_scheduler:\n                scheduler.step()\n        \n        end = time.time()\n\n        if step % Config.verbose == 0 or step == (len(train_loader) - 1):\n            print(f\"Epoch: [{epoch+1}][{step}/{len(train_loader)}] \"\n                  f\"Elapsed {losses.time_since(start, float(step+1)/len(train_loader))} \"\n                  f\"Loss: {losses.val:.4f}({losses.avg:.4f}) \"\n                  f\"Grad: {grad_norm:.4f} \"\n                  f\"LR: {scheduler.get_lr()[0]:4f}\"\n                  )\n    return losses.avg\n\n\n\ndef valid_fn(valid_loader, model, criterion, device):\n    model.eval()\n    losses = Meter()\n    preds = []\n    start = end = time.time()\n    for step, (input_ids, attention_mask, targets) in enumerate(valid_loader):\n        input_ids = input_ids.to(device)\n        attention_mask = attention_mask.to(device)\n        targets = targets.to(device)\n        batch_size = targets.size(0)\n\n        # compute loss\n        with torch.no_grad():\n            y_preds = model(input_ids=input_ids, attention_mask=attention_mask)\n        loss = criterion(y_preds, targets)\n        losses.update(loss.item(), batch_size)\n\n        # record predictions\n        preds.append(y_preds.sigmoid().to(\"cpu\").numpy())\n        if Config.gradient_accumulation_steps > 1:\n            loss = loss / Config.gradient_accumulation_steps\n        end = time.time()\n\n        if step % Config.verbose == 0 or step == (len(valid_loader) - 1):\n            print(f\"Eval: [{step}/{len(valid_loader)}] \"\n                  f\"Elapsed {losses.time_since(start, float(step+1)/len(valid_loader))} \"\n                  f\"Loss: {losses.val:.4f}({losses.avg:.4f}) \"\n                  )\n    predictions = np.concatenate(preds)\n    return losses.avg, predictions","metadata":{"execution":{"iopub.status.busy":"2021-11-17T06:09:26.409Z","iopub.execute_input":"2021-11-17T06:09:26.409377Z","iopub.status.idle":"2021-11-17T06:09:26.432624Z","shell.execute_reply.started":"2021-11-17T06:09:26.409338Z","shell.execute_reply":"2021-11-17T06:09:26.431651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training(train_df, valid_df, model, filepath, text_col):\n\n    # model & tokenizer\n    model.to(DEVICE)\n    tokenizer = AutoTokenizer.from_pretrained(Config.model_name)\n\n    va_targets = valid_df[Config.target_cols].values\n    train_dataset = TrainDataset(train_df, tokenizer, text_col)\n    valid_dataset = TrainDataset(valid_df, tokenizer, text_col)\n\n    train_loader = DataLoader(\n        train_dataset, \n        batch_size=Config.train_batch_size, \n        shuffle=True, \n        num_workers=Config.num_workers, \n        pin_memory=True, \n        drop_last=True)\n    \n    valid_loader = DataLoader(\n        valid_dataset,\n        batch_size=Config.valid_batch_size,\n        shuffle=False,\n        num_workers=Config.num_workers, \n        pin_memory=True, \n        drop_last=False\n    )\n\n    # optimizer & scheduler\n    optimizer = AdamW(\n        model.parameters(),\n        lr=Config.lr, \n        weight_decay=Config.weight_decay)\n\n    # set warmup steps if it is None\n    if \"num_training_steps\" in list(Config.scheduler.keys()):\n        if Config.scheduler[\"num_training_steps\"] is None:\n            Config.scheduler[\"num_training_steps\"] = ((len(train_loader) * Config.epochs) -\n                                                      Config.scheduler[\"num_warmup_steps\"])\n    scheduler = get_scheduler(optimizer)\n\n    # loop\n    criterion = nn.BCEWithLogitsLoss()\n    best_score = -np.inf if Config.is_higher_score_better else np.inf\n    best_loss = np.inf\n    \n    for epoch in range(Config.epochs):\n        start_time = time.time()\n\n        # train\n        avg_loss = train_fn(\n            train_loader=train_loader,\n            model=model,\n            criterion=criterion,\n            optimizer=optimizer,\n            epoch=epoch, \n            scheduler=scheduler,\n            device=DEVICE)\n        \n        # eval\n        avg_val_loss, preds = valid_fn(\n            valid_loader=valid_loader, \n            model=model, \n            criterion=criterion, \n            device=DEVICE)\n        \n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(avg_val_loss)\n        else:\n            scheduler.step() \n        \n        # scoring\n        score = get_score(va_targets, preds)\n        \n        elapsed = time.time() - start_time\n        print(f\"Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f} \"\n              f\"avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s\")\n        print(f\"Epoch {epoch+1} - Score: {score:.4f}\")\n\n        judge = (score >= best_score if Config.is_higher_score_better\n                 else score < best_score)\n        if judge:\n            best_score = score\n            print(f\"Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model\")\n            torch.save(model.state_dict(), filepath)\n\n\ndef inference(test_df, model, filepath, text_col):\n    model.eval()\n    state = torch.load(filepath, map_location=torch.device(\"cpu\"))\n    model.load_state_dict(state)\n    model.to(DEVICE)\n\n    tokenizer = AutoTokenizer.from_pretrained(Config.model_name)\n    test_dataset = TestDataset(test_df, tokenizer, text_col)\n    test_loader = DataLoader(\n        test_dataset,\n        batch_size=Config.test_batch_size, \n        shuffle=False, \n        num_workers=Config.num_workers, \n        pin_memory=True, \n        drop_last=False)\n    \n    preds = []\n    bar = tqdm(enumerate(test_loader), total=len(test_loader))\n    for step, (input_ids, attention_mask) in bar:\n        input_ids = input_ids.to(DEVICE)\n        attention_mask = attention_mask.to(DEVICE)\n        with torch.no_grad():\n            pred = model(input_ids, attention_mask)\n\n        preds.append(pred.sigmoid().detach().cpu().numpy())\n    preds = np.concatenate(preds)\n    return preds\n\n\ndef get_scheduler(optimizer):\n    if Config.scheduler[\"scheduler\"] == \"cosine_scedule_with_warmup\":\n        scheduler = get_cosine_schedule_with_warmup(\n            optimizer,\n            num_warmup_steps=Config.scheduler[\"num_warmup_steps\"],\n            num_training_steps=Config.scheduler[\"num_training_steps\"],\n            )\n        \n    elif Config.scheduler[\"scheduler\"] == \"get_linear_schedule_with_warmup\":\n        scheduler = get_linear_schedule_with_warmup(\n            optimizer,\n            num_warmup_steps=Config.scheduler[\"num_warmup_steps\"],\n            num_training_steps=Config.scheduler[\"num_training_steps\"],\n            )\n    \n    elif Config.scheduler[\"scheduler\"] == \"CosineAnnealingWarmRestarts\":\n        scheduler = CosineAnnealingWarmRestarts(\n            optimizer, \n            T_0=Config.scheduler[\"T_0\"], \n            T_mult=1,\n            eta_min=Config.scheduler[\"min_lr\"], \n            last_epoch=-1)\n    \n    elif Config.scheduler[\"scheduler\"] == \"ReduceLROnPlateau\":\n        scheduler = ReduceLROnPlateau(\n            optimizer, \n            mode=\"min\",\n            factor=Config.scheduler[\"factor\"], \n            patience=Config.scheduler[\"patience\"],\n            verbose=True, \n            eps=Config.scheduler[\"eps\"])\n    \n    else:\n        raise NotImplementedError\n    \n    return scheduler\n\n\ndef train_cv(train, text_col):\n\n    oof_df = pd.DataFrame(np.zeros((len(train), len(Config.target_cols))), columns=Config.target_cols)\n    for i_fold in range(Config.n_fold):\n        if i_fold in Config.trn_fold:\n            filepath = os.path.join(\n                        EXP_MODEL,\n                        f\"{Config.name}-seed{Config.seed}-fold{i_fold}.pth\")\n            \n            tr_df, va_df = (train[train[\"fold\"] != i_fold].reset_index(drop=True),\n                            train[train[\"fold\"] == i_fold].reset_index(drop=True))\n                        \n            if not os.path.isfile(filepath):  # if trained model, no training\n                model = Jigsaw01Model()\n                training(tr_df, va_df, model, filepath, text_col)\n            \n            model = Jigsaw01Model()\n            preds = inference(va_df, model, filepath, text_col)\n\n            oof_df.loc[train[\"fold\"] == i_fold, Config.target_cols] = preds \n\n            # fold score\n            score = get_score(va_df[Config.target_cols].values, preds)\n            logger.info(f\"{Config.name}-seed{Config.seed}-fold{i_fold} >>>>> Score={score:.4f}\")\n\n    # overall score\n    score = get_score(va_df[Config.target_cols].values, preds)\n    logger.info(f\"{Config.name}-seed{Config.seed}-OOF-Score >>>>> Score={score:.4f}\")\n\n    return oof_df.reset_index(drop=True)\n\n\ndef predict_cv(test, text_col):\n    model = Jigsaw01Model()\n\n    preds_fold = []\n    preds_fold_df = pd.DataFrame()\n    for i_fold in range(Config.n_fold):\n        if i_fold in Config.trn_fold:\n            filepath = os.path.join(\n                EXP_MODEL,\n                f\"{Config.name}-seed{Config.seed}-fold{i_fold}.pth\")\n            \n            preds = inference(test, model, filepath, text_col)\n            _df = (pd.DataFrame(preds, columns=Config.target_cols).\n                   add_prefix(f\"FOLD{i_fold:02}=\"))\n            \n            preds_fold.append(preds)\n            preds_fold_df = pd.concat([preds_fold_df, _df], axis=1)\n    \n    preds = np.mean(preds_fold, axis=0)\n    return preds, preds_fold_df\n\n\ndef predict_cv_jigsaw04_validation(validation_data):\n    text_cols = [\"less_toxic\", \"more_toxic\"]\n    for text_col in text_cols:\n\n        preds, preds_fold_df = predict_cv(validation_data, text_col)\n\n        # my toxic score = sum of target_cols(fold prediction)\n        toxic = preds_fold_df.sum(axis=1)\n        validation_data[f\"preds={text_col}\"] = toxic\n\n        # to scv [preds: fold average, preds_fold_df: all fold]\n        pd.DataFrame(preds, columns=Config.target_cols).to_csv(os.path.join(EXP_PREDS, f\"{text_col}-preds.csv\"), index=False)\n        preds_fold_df.to_csv(os.path.join(EXP_PREDS, f\"{text_col}-preds_fold_df.csv\"), index=False)\n\n    return validation_data\n\n\ndef jigsaw04_metrics(preds_less_toxic, preds_more_toxic):\n    scores = np.zeros(len(preds_less_toxic))\n    for i in range(len(scores)):\n        if preds_less_toxic[i] < preds_more_toxic[i]:\n            scores[i] = 1\n    return scores","metadata":{"execution":{"iopub.status.busy":"2021-11-17T06:17:39.8012Z","iopub.execute_input":"2021-11-17T06:17:39.801483Z","iopub.status.idle":"2021-11-17T06:17:39.837406Z","shell.execute_reply.started":"2021-11-17T06:17:39.801451Z","shell.execute_reply":"2021-11-17T06:17:39.836364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ========================================\n# Main\n# ========================================\nif not Config.only_inference:\n    # training\n    print(\"# ---------- # Start Training # ---------- #\")\n    oof_df = train_cv(train_jigsaw_01, text_col=\"comment_text\")\n    oof_df.to_csv(os.path.join(EXP_PREDS, \"oof.csv\"), index=False)\n\n    # score (Jigsaw01)\n    fold_mask = train_jigsaw_01[\"fold\"].isin(Config.trn_fold)\n    score = get_score(\n        train_jigsaw_01.loc[fold_mask, Config.target_cols].values, \n        oof_df.loc[fold_mask, Config.target_cols].values, \n        verbose=True)\n    logger.info(f\"Jigsaw01-MCWROC={score:.4f}\")\n\n    # validation (Jigsaw04[This Compe])\n    print(\"# ---------- # Start Validation # ---------- #\")\n    validation_preds_df = predict_cv_jigsaw04_validation(validation_data)\n    validation_preds_df.to_csv(os.path.join(EXP_PREDS, f\"validation_preds_df.csv\"), index=False)\n    scores = jigsaw04_metrics(preds_less_toxic=validation_preds_df[\"preds=less_toxic\"], \n                            preds_more_toxic=validation_preds_df[\"preds=more_toxic\"])\n    score = np.mean(scores)\n    logger.info(f\"Jigsaw04-Jigsaw-Rate-Severity={score:.4f}\")\n\n# prediction\nprint(\"# ---------- # Start Inference # ---------- #\")\npreds, fold_preds_df = predict_cv(comments_to_score, text_col=\"text\")\n(pd.DataFrame(preds, columns=Config.target_cols).\n to_csv(os.path.join(EXP_PREDS, f\"comments_to_score_preds_df.csv\"), index=False))\nfold_preds_df.to_csv(os.path.join(EXP_PREDS, f\"comments_to_score_fold_preds_df.csv\"), index=False)\n\ncomments_to_score[\"preds\"] = fold_preds_df.sum(axis=1)\ncomments_to_score.to_csv(os.path.join(EXP_PREDS, f\"comments_to_score_df.csv\"), index=False)\n\n# make submission\nprint(\"# ---------- # Make Submission # ---------- #\")\nsample_submission[\"score\"] = comments_to_score[\"preds\"].rank(method='first')\ndisplay(sample_submission)\nsample_submission.to_csv(os.path.join(SUBMISSION, Config.name+\".csv\"), index=False)\nprint(\"# ---------- # Finish Experiment!! # ---------- #\")","metadata":{"execution":{"iopub.status.busy":"2021-11-17T06:17:40.398455Z","iopub.execute_input":"2021-11-17T06:17:40.399154Z","iopub.status.idle":"2021-11-17T06:18:40.552198Z","shell.execute_reply.started":"2021-11-17T06:17:40.399117Z","shell.execute_reply":"2021-11-17T06:18:40.551326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# upload output folder to kaggle dataset\nif Config.upload_from_colab:\n    from kaggle.api.kaggle_api_extended import KaggleApi\n\n    def dataset_create_new(dataset_name, upload_dir):\n        dataset_metadata = {}\n        dataset_metadata['id'] = f'{os.environ[\"KAGGLE_USERNAME\"]}/{dataset_name}'\n        dataset_metadata['licenses'] = [{'name': 'CC0-1.0'}]\n        dataset_metadata['title'] = dataset_name\n        with open(os.path.join(upload_dir, 'dataset-metadata.json'), 'w') as f:\n            json.dump(dataset_metadata, f, indent=4)\n        api = KaggleApi()\n        api.authenticate()\n        api.dataset_create_new(folder=upload_dir, convert_to_csv=False, dir_mode='tar')\n\n    if len(EXP) >= 50:\n        dataset_name = EXP[:7]\n\n    dataset_create_new(dataset_name=dataset_name, upload_dir=OUTPUT_EXP)","metadata":{"execution":{"iopub.status.busy":"2021-11-17T06:11:00.855905Z","iopub.status.idle":"2021-11-17T06:11:00.856561Z","shell.execute_reply.started":"2021-11-17T06:11:00.856311Z","shell.execute_reply":"2021-11-17T06:11:00.85634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}