{"cells":[{"metadata":{"_uuid":"529f1f49-ec0c-4fb6-9302-e5c9b9d3dc60","_cell_guid":"a2267eff-0148-4711-ae9a-729377045221","trusted":true},"cell_type":"code","source":"package_path = '../input/pytorch-image-models/pytorch-image-models-master' #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ca640bde-0a66-43b8-8c44-47e32c825bc3","_cell_guid":"13c9995a-63c8-4b8a-8b9b-985380500a4b","trusted":true},"cell_type":"code","source":"!pip install pretrainedmodels\n!pip install pytorch-lightning==0.7.3\n!git clone https://github.com/NVIDIA/apex\n%cd apex\n!pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./\n%cd ..","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0629fb26-236a-440a-9841-8f4420f76c8e","_cell_guid":"48194c38-1ad4-49d5-985d-ba32def61d29","trusted":true},"cell_type":"code","source":"import logging\nimport os\nimport random\nfrom logging import Logger\nfrom logging.handlers import TimedRotatingFileHandler\nimport timm\n\n# Third party libraries\n# Third party libraries\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport math\n\n# Third party libraries\nimport gc\nfrom time import time\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint\n\n# Third party libraries\nimport torch\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import KFold\nimport pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1e8b83fc-9c53-4027-b0f2-cdbb309b5b34","_cell_guid":"6df27962-aec2-4cfc-8e2d-ba0b205f7447","trusted":true},"cell_type":"code","source":"IMG_SHAPE = (600, 800, 3)\n# IMAGE_FOLDER = \"/home/public_data_center/kaggle/plant_pathology_2020/images\"\nIMAGE_FOLDER = \"../input/cassava-leaf-disease-classification/train_images\"\nNPY_FOLDER = \"/home/public_data_center/kaggle/plant_pathology_2020/npys\"\nLOG_FOLDER = \"logs\"\nEPOCHS = 7\n\n\ndef mkdir(path: str):\n    \"\"\"Create directory.\n     Create directory if it is not exist, else do nothing.\n     Parameters\n     ----------\n     path: str\n        Path of your directory.\n     Examples\n     --------\n     mkdir(\"data/raw/train/\")\n     \"\"\"\n    try:\n        if path is None:\n            pass\n        else:\n            os.stat(path)\n    except Exception:\n        os.makedirs(path)\n\n\ndef seed_reproducer(seed=2020):\n    \"\"\"Reproducer for pytorch experiment.\n    Parameters\n    ----------\n    seed: int, optional (default = 2019)\n        Radnom seed.\n    Example\n    -------\n    seed_reproducer(seed=2019).\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n        torch.backends.cudnn.enabled = True\n\n\ndef init_hparams():\n    parser = ArgumentParser(add_help=False)\n    parser.add_argument(\"-backbone\", \"--backbone\", type=str, default=\"se_resnext50_32x4d\")\n    parser.add_argument(\"-tbs\", \"--train_batch_size\", type=int, default=32 * 1)\n    parser.add_argument(\"-vbs\", \"--val_batch_size\", type=int, default=32 * 1)\n    parser.add_argument(\"--num_workers\", type=int, default=8)\n    parser.add_argument(\"--image_size\", nargs=\"+\", default=[450, 600])\n    parser.add_argument(\"--seed\", type=int, default=2020)\n    parser.add_argument(\"--max_epochs\", type=int, default=EPOCHS)\n    parser.add_argument(\"--gpus\", nargs=\"+\", default=[0])  # 输入1 2 3\n    parser.add_argument(\"--precision\", type=int, default=16)\n    parser.add_argument(\"--gradient_clip_val\", type=float, default=1)\n    parser.add_argument(\"--soft_labels_filename\", type=str, default=\"\")#\"soft_labels.csv\")\n    parser.add_argument(\"--log_dir\", type=str, default=\"logs_submit\")\n    try:\n        hparams = parser.parse_args()\n    except:\n        hparams = parser.parse_args([])\n    print(type(hparams.gpus), hparams.gpus)\n    if len(hparams.gpus) == 1:\n        hparams.gpus = [int(hparams.gpus[0])]\n    else:\n        hparams.gpus = [int(gpu) for gpu in hparams.gpus]\n\n    hparams.image_size = [int(size) for size in hparams.image_size]\n    return hparams\n\n\ndef load_data(logger, frac=1):\n    data, test_data = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\"), pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\n    # Do fast experiment\n    if frac < 1:\n        logger.info(f\"use frac : {frac}\")\n        data = data.sample(frac=frac).reset_index(drop=True)\n        test_data = test_data.sample(frac=frac).reset_index(drop=True)\n    return data, test_data\n\n\ndef init_logger(log_name, log_dir=None):\n    \"\"\"日志模块\n    Reference: https://juejin.im/post/5bc2bd3a5188255c94465d31\n    日志器初始化\n    日志模块功能:\n        1. 日志同时打印到到屏幕和文件\n        2. 默认保留近一周的日志文件\n    日志等级:\n        NOTSET（0）、DEBUG（10）、INFO（20）、WARNING（30）、ERROR（40）、CRITICAL（50）\n    如果设定等级为10, 则只会打印10以上的信息\n    Parameters\n    ----------\n    log_name : str\n        日志文件名\n    log_dir : str\n        日志保存的目录\n    Returns\n    -------\n    RootLogger\n        Python日志实例\n    \"\"\"\n\n    mkdir(log_dir)\n\n    # 若多处定义Logger，根据log_name确保日志器的唯一性\n    if log_name not in Logger.manager.loggerDict:\n        logging.root.handlers.clear()\n        logger = logging.getLogger(log_name)\n        logger.setLevel(logging.DEBUG)\n\n        # 定义日志信息格式\n        datefmt = \"%Y-%m-%d %H:%M:%S\"\n        format_str = \"[%(asctime)s] %(filename)s[%(lineno)4s] : %(levelname)s  %(message)s\"\n        formatter = logging.Formatter(format_str, datefmt)\n\n        # 日志等级INFO以上输出到屏幕\n        console_handler = logging.StreamHandler()\n        console_handler.setLevel(logging.INFO)\n        console_handler.setFormatter(formatter)\n        logger.addHandler(console_handler)\n\n        if log_dir is not None:\n            # 日志等级INFO以上输出到{log_name}.log文件\n            file_info_handler = TimedRotatingFileHandler(\n                filename=os.path.join(log_dir, \"%s.log\" % log_name), when=\"D\", backupCount=7\n            )\n            file_info_handler.setFormatter(formatter)\n            file_info_handler.setLevel(logging.INFO)\n            logger.addHandler(file_info_handler)\n\n    logger = logging.getLogger(log_name)\n\n    return logger\n\n\ndef read_image(image_path):\n    \"\"\" 读取图像数据，并转换为RGB格式\n        32.2 ms ± 2.34 ms -> self\n        48.7 ms ± 2.24 ms -> plt.imread(image_path)\n    \"\"\"\n    return cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"193ea390-0e16-43fa-859b-8975bfe2cf58","_cell_guid":"7fa2c2e4-9084-4583-a061-5ebece673723","trusted":true},"cell_type":"markdown","source":"# DataSet"},{"metadata":{"_uuid":"82d179a1-0096-4294-9aaf-82a55641d00d","_cell_guid":"1a44ce08-e6e8-4cad-9c1b-a0d9edc59baf","trusted":true},"cell_type":"code","source":"\nfrom albumentations import (\n    Compose,\n    GaussianBlur,\n    HorizontalFlip,\n    MedianBlur,\n    MotionBlur,\n    Normalize,\n    OneOf,\n    RandomBrightness,\n    RandomContrast,\n    Resize,\n    ShiftScaleRotate,\n    VerticalFlip,\n    RandomResizedCrop,\n)\nfrom torch.utils.data import DataLoader, Dataset\n\n# User defined libraries\n\n# for fast read data\n# from utils import NPY_FOLDER\n\n\nclass PlantDataset(Dataset):\n    \"\"\" Do normal training\n    \"\"\"\n\n    def __init__(self, data, soft_labels_filename=None, transforms=None):\n        self.data = data\n        self.transforms = transforms\n        if soft_labels_filename == \"\":\n            print(\"soft_labels is None\")\n            self.soft_labels = None\n        else:\n            self.soft_labels = pd.read_csv(soft_labels_filename)\n\n    def __getitem__(self, index):\n        start_time = time()\n        # Read image\n        # solution-1: read from raw image\n        image = cv2.cvtColor(\n            cv2.imread(os.path.join(IMAGE_FOLDER, self.data.iloc[index, 0] )), cv2.COLOR_BGR2RGB\n        )\n        # solution-2: read from npy file which can speed the data load time.\n        # image = np.load(os.path.join(NPY_FOLDER, \"raw\", self.data.iloc[index, 0] + \".npy\"))\n\n        # Convert if not the right shape\n        if image.shape != IMG_SHAPE:\n            image = image.transpose(1, 0, 2)\n\n        # Do data augmentation\n        if self.transforms is not None:\n            image = self.transforms(image=image)[\"image\"].transpose(2, 0, 1)\n        \n        label_value = [0,0,0,0,0]\n        label_value[int(self.data.iloc[index, 1:].values.astype(np.int64))] = 1\n        #label_value = self.data.iloc[index, 1:].values.astype(np.int64)\n        # Soft label\n        if self.soft_labels is not None:\n            label = torch.FloatTensor(\n                (self.data.iloc[index, 1:].values * 0.7).astype(np.float)\n                + (self.soft_labels.iloc[index, 1:].values * 0.3).astype(np.float)\n            )\n        else:\n            label = torch.FloatTensor(label_value)\n        #print(label)\n        #image = image/255.\n\n        return image, label, time() - start_time\n\n    def __len__(self):\n        return len(self.data)\n\n\ndef generate_transforms(image_size):\n\n    train_transform = Compose(\n        [\n            OneOf([RandomResizedCrop(height=image_size[0], width=image_size[1]),Resize(height=image_size[0], width=image_size[1])],p=1.0),\n            OneOf([RandomBrightness(limit=0.1, p=1), RandomContrast(limit=0.1, p=1)]),\n            OneOf([MotionBlur(blur_limit=3), MedianBlur(blur_limit=3), GaussianBlur(blur_limit=3)], p=0.5),\n            VerticalFlip(p=0.5),\n            HorizontalFlip(p=0.5),\n            ShiftScaleRotate(\n                shift_limit=0.2,\n                scale_limit=0.2,\n                rotate_limit=20,\n                interpolation=cv2.INTER_LINEAR,\n                border_mode=cv2.BORDER_REFLECT_101,\n                p=1,\n            ),\n            Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), max_pixel_value=255.0, p=1.0),\n        ]\n    )\n\n    val_transform = Compose(\n        [\n            Resize(height=image_size[0], width=image_size[1]),\n            Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), max_pixel_value=255.0, p=1.0),\n        ]\n    )\n\n    return {\"train_transforms\": train_transform, \"val_transforms\": val_transform}\n\n\ndef generate_dataloaders(hparams, train_data, val_data, transforms):\n    train_dataset = PlantDataset(\n        data=train_data, transforms=transforms[\"train_transforms\"], soft_labels_filename=hparams.soft_labels_filename\n    )\n    val_dataset = PlantDataset(\n        data=val_data, transforms=transforms[\"val_transforms\"], soft_labels_filename=hparams.soft_labels_filename\n    )\n    train_dataloader = DataLoader(\n        train_dataset,\n        batch_size=hparams.train_batch_size,\n        shuffle=True,\n        num_workers=hparams.num_workers,\n        pin_memory=True,\n        drop_last=True,\n    )\n    val_dataloader = DataLoader(\n        val_dataset,\n        batch_size=hparams.val_batch_size,\n        shuffle=False,\n        num_workers=hparams.num_workers,\n        pin_memory=True,\n        drop_last=False,\n    )\n\n    return train_dataloader, val_dataloader","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5257a0f7-d49b-45a2-96b8-0f546a412d69","_cell_guid":"19268dd4-12d7-4794-9f3d-fb9353cbdb4a","trusted":true},"cell_type":"markdown","source":"# lr_scheduler"},{"metadata":{"_uuid":"71e11eae-9e0f-4865-90b5-14ca4dd8efb2","_cell_guid":"5b97dcb7-2ba3-4155-a4d7-d6679ec408f8","trusted":true},"cell_type":"code","source":"class WarmRestart(lr_scheduler.CosineAnnealingLR):\n    \"\"\"This class implements Stochastic Gradient Descent with Warm Restarts(SGDR): https://arxiv.org/abs/1608.03983.\n    Set the learning rate of each parameter group using a cosine annealing schedule,\n    When last_epoch=-1, sets initial lr as lr.\n    This can't support scheduler.step(epoch). please keep epoch=None.\n    \"\"\"\n\n    def __init__(self, optimizer, T_max=10, T_mult=2, eta_min=0, last_epoch=-1):\n        \"\"\"implements SGDR\n        Parameters:\n        ----------\n        T_max : int\n            Maximum number of epochs.\n        T_mult : int\n            Multiplicative factor of T_max.\n        eta_min : int\n            Minimum learning rate. Default: 0.\n        last_epoch : int\n            The index of last epoch. Default: -1.\n        \"\"\"\n        self.T_mult = T_mult\n        super().__init__(optimizer, T_max, eta_min, last_epoch)\n\n    def get_lr(self):\n        if self.last_epoch == self.T_max:\n            self.last_epoch = 0\n            self.T_max *= self.T_mult\n        return [\n            self.eta_min + (base_lr - self.eta_min) * (1 + math.cos(math.pi * self.last_epoch / self.T_max)) / 2\n            for base_lr in self.base_lrs\n        ]\n\n\ndef warm_restart(scheduler, T_mult=2):\n    \"\"\"warm restart policy\n    Parameters:\n    ----------\n    T_mult: int\n        default is 2, Stochastic Gradient Descent with Warm Restarts(SGDR): https://arxiv.org/abs/1608.03983.\n    Examples:\n    --------\n    >>> # some other operations(note the order of operations)\n    >>> scheduler.step()\n    >>> scheduler = warm_restart(scheduler, T_mult=2)\n    >>> optimizer.step()\n    \"\"\"\n    if scheduler.last_epoch == scheduler.T_max:\n        scheduler.last_epoch = -1\n        scheduler.T_max *= T_mult\n    return scheduler","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c8dc76af-0d97-48d1-814e-2582aed6185a","_cell_guid":"4d669c7b-4795-46fd-850b-7b3d2c3c681e","trusted":true},"cell_type":"markdown","source":"# Loss_function"},{"metadata":{"_uuid":"5a1a7320-7918-4cfe-be81-5eff6e1094e6","_cell_guid":"778e721e-e4cd-445b-bd49-a1c9e7e5755d","trusted":true},"cell_type":"code","source":"\n    \nclass LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=5, smoothing=0.25, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist[torch.arange(target.size(0)),torch.argmax(target,1)] = self.confidence\n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"931c3017-7d1e-4b3b-b2a1-6fcdb0227acd","_cell_guid":"ac1c32db-e22e-407d-98dd-c659deeb6910","trusted":true},"cell_type":"markdown","source":"# models"},{"metadata":{"_uuid":"6b1b49f4-275b-4288-922d-614f4579d6e3","_cell_guid":"f4ed8288-3460-48f1-b389-4f024ebab78b","trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=True):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6cffc050-d39c-42d2-8993-10a0c1a67506","_cell_guid":"bf88d966-12cb-4403-a737-c500f9f462b5","trusted":true},"cell_type":"code","source":"class CoolSystem(pl.LightningModule):\n    def __init__(self, hparams):\n        super().__init__()\n        self.hparams = hparams\n\n        # 让每次模型初始化一致, 不让只要中间有再次初始化的情况, 结果立马跑偏\n        seed_reproducer(self.hparams.seed)\n\n        self.model = CassvaImgClassifier('tf_efficientnet_b2_ns', 5)\n        self.criterion = LabelSmoothingLoss()#CrossEntropyLossOneHot()\n        self.logger_kun = init_logger(\"kun_in\", hparams.log_dir)\n\n    def forward(self, x):\n        return self.model(x)\n\n    def configure_optimizers(self):\n        self.optimizer = torch.optim.Adam(self.parameters(), lr=0.001, betas=(0.9, 0.999), eps=1e-08, weight_decay=0)\n        self.scheduler = WarmRestart(self.optimizer, T_max=10, T_mult=1, eta_min=1e-5)\n        return [self.optimizer], [self.scheduler]\n\n    def training_step(self, batch, batch_idx):\n        step_start_time = time()\n        images, labels, data_load_time = batch\n\n        scores = self(images)\n        loss = self.criterion(scores, labels)\n        #print(torch.sum(torch.argmax(scores,dim=1) == torch.argmax(labels,dim=1))/float(len(labels)))\n        # self.logger_kun.info(f\"loss : {loss.item()}\")\n        # ! can only return scalar tensor in training_step\n        # must return key -> loss\n        # optional return key -> progress_bar optional (MUST ALL BE TENSORS)\n        # optional return key -> log optional (MUST ALL BE TENSORS)\n        data_load_time = torch.sum(data_load_time)\n\n        return {\n            \"loss\": loss,\n            \"data_load_time\": data_load_time,\n            \"batch_run_time\": torch.Tensor([time() - step_start_time + data_load_time]).to(data_load_time.device),\n        }\n\n    def training_epoch_end(self, outputs):\n        # outputs is the return of training_step\n        train_loss_mean = torch.stack([output[\"loss\"] for output in outputs]).mean()\n        self.data_load_times = torch.stack([output[\"data_load_time\"] for output in outputs]).sum()\n        self.batch_run_times = torch.stack([output[\"batch_run_time\"] for output in outputs]).sum()\n\n        self.current_epoch += 1\n        if self.current_epoch < (self.trainer.max_epochs - 4):\n            self.scheduler = warm_restart(self.scheduler, T_mult=2)\n\n        return {\"train_loss\": train_loss_mean}\n\n    def validation_step(self, batch, batch_idx):\n        step_start_time = time()\n        images, labels, data_load_time = batch\n        data_load_time = torch.sum(data_load_time)\n        scores = self(images)\n        loss = self.criterion(scores, labels)\n\n        # must return key -> val_loss\n        return {\n            \"val_loss\": loss,\n            \"scores\": scores,\n            \"labels\": labels,\n            \"data_load_time\": data_load_time,\n            \"batch_run_time\": torch.Tensor([time() - step_start_time + data_load_time]).to(data_load_time.device),\n        }\n\n    def validation_epoch_end(self, outputs):\n        # compute loss\n        val_loss_mean = torch.stack([output[\"val_loss\"] for output in outputs]).mean()\n        self.data_load_times = torch.stack([output[\"data_load_time\"] for output in outputs]).sum()\n        self.batch_run_times = torch.stack([output[\"batch_run_time\"] for output in outputs]).sum()\n\n        # compute roc_auc\n        scores_all = torch.cat([output[\"scores\"] for output in outputs]).cpu()\n        labels_all = torch.round(torch.cat([output[\"labels\"] for output in outputs]).cpu())\n        #val_roc_auc = roc_auc_score(labels_all, scores_all)\n        val_acc = torch.sum(torch.argmax(scores_all,dim=1) == torch.argmax(labels_all,dim=1))/float(len(scores_all))\n\n        # terminal logs\n        self.logger_kun.info(\n            f\"{self.hparams.fold_i}-{self.current_epoch} | \"\n            f\"lr : {self.scheduler.get_lr()[0]:.6f} | \"\n            f\"val_loss : {val_loss_mean:.4f} | \"\n            f\"val_acc : {val_acc:.4f} | \"\n            f\"data_load_times : {self.data_load_times:.2f} | \"\n            f\"batch_run_times : {self.batch_run_times:.2f}\"\n        )\n        # f\"data_load_times : {self.data_load_times:.2f} | \"\n        # f\"batch_run_times : {self.batch_run_times:.2f}\"\n        # must return key -> val_loss\n        return {\"val_loss\": val_loss_mean, \"val_acc\": val_acc}\n\n\nif __name__ == \"__main__\":\n    from argparse import ArgumentParser\n\n    # Make experiment reproducible\n    seed_reproducer(2020)\n\n    # Init Hyperparameters\n    hparams = init_hparams()\n\n    # init logger\n    logger = init_logger(\"kun_out\", log_dir=hparams.log_dir)\n\n    # Load data\n    data, test_data = load_data(logger)\n\n    # Generate transforms\n    transforms = generate_transforms(hparams.image_size)\n\n    # Do cross validation\n    valid_roc_auc_scores = []\n    folds = KFold(n_splits=5, shuffle=True, random_state=hparams.seed)\n    for fold_i, (train_index, val_index) in enumerate(folds.split(data)):\n        hparams.fold_i = fold_i\n        train_data = data.iloc[train_index, :].reset_index(drop=True)\n        val_data = data.iloc[val_index, :].reset_index(drop=True)\n\n        train_dataloader, val_dataloader = generate_dataloaders(hparams, train_data, val_data, transforms)\n\n        # Define callbacks\n        checkpoint_callback = ModelCheckpoint(\n            monitor=\"val_acc\",#\"val_roc_auc\",\n            save_top_k=1,\n            mode=\"max\",\n            filepath=os.path.join(hparams.log_dir, f\"fold={fold_i}\" + \"-{epoch}-{val_loss:.4f}-{val_acc:.4f}\"),\n        )\n        early_stop_callback = EarlyStopping(monitor=\"acc\", patience=10, mode=\"max\", verbose=True)\n\n        # Instance Model, Trainer and train model\n        model = CoolSystem(hparams)\n        trainer = pl.Trainer(\n            gpus=hparams.gpus,\n            min_epochs=1,\n            max_epochs=hparams.max_epochs,\n            #early_stop_callback=early_stop_callback,\n            checkpoint_callback=checkpoint_callback,\n            progress_bar_refresh_rate=0,\n            precision=hparams.precision,\n            num_sanity_val_steps=0,\n            profiler=False,\n            weights_summary=None,\n            #use_dp=True,\n            gradient_clip_val=1,\n        )\n        trainer.fit(model, train_dataloader, val_dataloader)\n\n        valid_roc_auc_scores.append(round(checkpoint_callback.best, 5))\n        logger.info(valid_roc_auc_scores)\n\n        del model\n        gc.collect()\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0ad3b900-b8f8-4448-87e0-6627a6e1c9e5","_cell_guid":"a830d485-b07c-4303-be9b-53febe2d49ea","trusted":true},"cell_type":"code","source":"\ndel model\ngc.collect()\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f6643d3-34f4-4e55-ab56-336d738a144d","_cell_guid":"73f2081a-cbd9-4839-a551-406fe672bc90","trusted":true},"cell_type":"code","source":"!rm -r apex\nimport os,shutil,glob\nfor path in glob.glob('logs_submit/*.ckpt'):\n    shutil.move(path,path.split('/')[-1].replace('=','-'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5905b09d-7596-43c1-9d70-5432de418289","_cell_guid":"4fd776e4-d45e-4d06-93cb-01950c009db5","trusted":true},"cell_type":"code","source":"open('multi.txt','w')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}