{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# Uninstall fastai for solving dependence problems\n!pip uninstall fastai -y\n# Install packages without internet\n!pip install ../input/packages/torch-1.7.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/packages/torchvision-0.8.2-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/byol-pytorch/byol-pytorch-master","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/repvgg/')\nsys.path.append('../input/repvggmodels/')\nsys.path.append('../input/pytorch-optimizers/')\n\nfrom torch_optimizer.radam import RAdam\nfrom repvgg import RepVGG, create_RepVGG_B2, create_RepVGG_B3g4, create_RepVGG_B3, repvgg_model_convert","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport time\nimport copy\nimport random\nimport joblib\nimport sklearn\nimport warnings\nimport multiprocessing\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom glob import glob\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom datetime import datetime\nfrom skimage import io\nfrom sklearn import metrics\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom IPython.display import display\n\nimport torch\nimport torch.nn.functional as F\nimport torchvision\nfrom torch import nn\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.optim.lr_scheduler import StepLR\nfrom torch.nn.modules.loss import _WeightedLoss\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torchvision import models\nfrom torchvision import transforms\nfrom byol_pytorch import BYOL\n\nfrom albumentations.pytorch import ToTensor, ToTensorV2\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,\n    ShiftScaleRotate, CenterCrop, Resize, GaussianBlur)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train      = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\nsubmission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ndisplay(train.head(2))\ndisplay(submission.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE  = 64\nEPOCHS      = 40\nLR          = 0.001\nIMAGE_SIZE  = 224\nEARLY_STOP  = 10\nNUM_WORKERS = multiprocessing.cpu_count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SelfSupervisedLearner(nn.Module):\n    \n    def __init__(self, net, **kwargs):\n        super().__init__()\n        self.learner = BYOL(net, **kwargs)\n\n    def forward(self, images):\n        return self.learner(images)\n\nclass ImagesDataset(Dataset):\n    \n    def __init__(self, df, image_size):\n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.data_root = '../input/cassava-leaf-disease-classification/train_images/'\n        self.transform = Compose([ToTensor()], p=1.)\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, index : int):\n        path = f\"{self.data_root}/{self.df.iloc[index]['image_id']}\"\n        img  = get_img(path)\n        return self.transform(image=img)['image']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = ImagesDataset(train, IMAGE_SIZE)\ntrain_loader = DataLoader(ds, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=True, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#backbone = models.resnet50(pretrained=True)\nbackbone = create_RepVGG_B2(deploy=False)\nbackbone.load_state_dict(torch.load('../input/repvgg/RepVGG-B2-train.pth'))\n\nmodel    = SelfSupervisedLearner(\n    backbone,\n    image_size = IMAGE_SIZE,\n    #hidden_layer = 'avgpool',  # ResNet\n    hidden_layer = 'gap',\n    projection_size = 256,\n    projection_hidden_size = 2056,\n    moving_average_decay = 0.99,\n    use_momentum = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sgd_optimizer(model, lr, momentum, weight_decay):\n    params = []\n    for key, value in model.named_parameters():\n        if not value.requires_grad:\n            continue\n        apply_weight_decay = weight_decay\n        apply_lr = lr\n        if 'bias' in key or 'bn' in key:\n            apply_weight_decay = 0\n        if 'bias' in key:\n            apply_lr = 2 * lr       #   Just a Caffe-style common practice. Made no difference.\n        params += [{'params': [value], 'lr': apply_lr, 'weight_decay': apply_weight_decay}]\n    optimizer = torch.optim.SGD(params, lr, momentum=momentum)\n    return optimizer\n\ndef train_epoch(model, dataloader, optim, device=\"cpu\", scheduler=None):\n    model.train()\n    \n    running_loss = None\n    losses = []\n    scaler = torch.cuda.amp.GradScaler()\n    \n    tbar   = tqdm(dataloader)\n    for item in tbar:\n        x = item.to(device).float()\n        optim.zero_grad()\n        # Runs the forward pass with autocasting.\n        with autocast():\n            loss = model(x)\n        scaler.scale(loss).backward()\n        scaler.step(optim)\n        scaler.update()\n        if running_loss is None:\n            running_loss = loss.item()\n        else:\n            running_loss = running_loss * .99 + loss.item() * .01\n        tbar.set_description('loss - {:.4f}'.format(running_loss))\n        losses.append(loss.item())\n        \n    if scheduler is not None:\n        scheduler.step()\n\n    return losses\n        \ndef do_train():\n    not_improved_cnt = 0\n    best_loss = 999\n    \n    #optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n    #scheduler_steplr = StepLR(optimizer, step_size=5, gamma=0.9)\n    #scheduler_warmup = GradualWarmupScheduler(optimizer, multiplier=1, total_epoch=10, after_scheduler=scheduler_steplr)\n    #optimizer = sgd_optimizer(model, LR, 0.9, 1e-4)\n    #scheduler = CosineAnnealingLR(optimizer=optimizer, T_max=100)\n    optimizer = RAdam(model.parameters(), lr=LR, weight_decay=1e-4)\n\n    model.to(device)\n    \n    for epoch in range(EPOCHS):\n        losses = train_epoch(model, train_loader, optimizer, device, None)\n        print(f\"epoch - {epoch + 1} mean loss - {np.mean(losses):.5f}\")\n        \n        if best_loss > np.mean(losses):\n            print('Best model will be saved to output path')\n            best_loss = np.mean(losses)\n            torch.save(backbone.state_dict(), \"./byol.pt\")\n            not_improved_cnt = 0\n        elif EARLY_STOP == not_improved_cnt:\n            print(\"Met early stopping.\")\n            break\n        else:\n            not_improved_cnt += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndo_train()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}