{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install torch==1.1.0\n#!pip install pretrainedmodels\n#!pip install pytorchcv\n#!pip install albumentations\n!pip install efficientnet_pytorch\nfrom efficientnet_pytorch import EfficientNet\n#from pretrainedmodels import se_resnext101_32x4d\n#from pytorchcv.model_provider import get_model as ptcv_get_model\n#import albumentations\n#from albumentations import torch as AT","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!git clone https://github.com/NVIDIA/apex\n!pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./apex","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nfrom os.path import isfile\nimport torch.nn.init as init\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport pandas as pd \nimport os\nfrom PIL import Image, ImageFilter\n#print(os.listdir(\"../input\"))\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom torch.optim import Adam, SGD, RMSprop\nimport time\nfrom torch.autograd import Variable\nimport torch.functional as F\nfrom tqdm import tqdm\nfrom sklearn import metrics\nimport urllib\nimport pickle\nimport cv2\nimport torch.nn.functional as F\nfrom torchvision import models\nimport seaborn as sns\nimport random\nfrom PIL import Image\nimport sys\nfrom apex import amp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"766f44c87272f67d632e519dce11cf54a3382696"},"cell_type":"code","source":"#train = '../input/hgarbage/garbage_classify/train_data/'\n#train_csv = pd.read_csv(\"../input/hgarbage/garbage_classify/img_info.csv\")\n\n#train_df, val_df = train_test_split(train_csv, test_size=0.1, random_state=24, stratify=train_csv.cls)\n#train_df.reset_index(drop=True, inplace=True)\n#val_df.reset_index(drop=True, inplace=True)\n#num_cls = len(train_csv['cls'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train= \"../input/hwdataset/my_data/my_data/\"\ntrn_csv = pd.read_csv(\"../input/hwdataset/train.csv\")\ntrn_csv = trn_csv[:14803]\nfold = 4\ntrain_df, val_df = train_test_split(trn_csv, test_size=0.1, random_state=24, stratify=trn_csv.cls)\n#trn_idx = list(StratifiedKFold(n_splits=5, random_state=24, shuffle=True).split(trn_csv, trn_csv.cls))[fold][0]\n#val_idx = list(StratifiedKFold(n_splits=5, random_state=24, shuffle=True).split(trn_csv, trn_csv.cls))[fold][1]\n#train_df = trn_csv.iloc[trn_idx]\n#val_df = trn_csv.iloc[val_idx]\ntrain_df.reset_index(drop=True, inplace=True)\nval_df.reset_index(drop=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"groupDataFrame=train_df.groupby(by=['cls'])\nlabels=groupDataFrame.size()\nprint(\"length of label is \",len(labels))\nmaxNum=max(labels)\nlst=pd.DataFrame(columns=[\"cls\",\"id\"])\nfor i in range(len(labels)):\n    #print(\"Processing label  :\",i)\n    tmpGroupBy=groupDataFrame.get_group(i)\n    createdShuffleLabels=np.random.permutation(np.array(range(maxNum)))%labels[i]\n    #print(\"Num of the label is : \",labels[i])\n    lst=lst.append(tmpGroupBy.iloc[createdShuffleLabels],ignore_index=True)\n    #print(\"Done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lst = lst.sample(frac=1, random_state=24)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = 40\nseed_everything(24)\nlr          = 1e-3\nIMG_SIZE    = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21908baa8df4e398b0d49a5146ce544504637c5a"},"cell_type":"code","source":"class MyDataset(Dataset):\n    \n    def __init__(self, dataframe, transform=None):\n        self.df = dataframe\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_name = self.df['id'].iloc[idx]\n        image = cv2.imread(os.path.join(train, img_name)+'.jpg')\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n        #image = Image.open(os.path.join(train, img_name))\n        #image = image.convert(\"RGB\")\n        label = self.df['cls'].iloc[idx]\n        image = transforms.ToPILImage()(image)\n        \n        if self.transform:\n            image = self.transform(image)\n            \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f590638fd07b9aefe2210a39612ac77e0689c0c1"},"cell_type":"code","source":"train_transform = transforms.Compose([\n    transforms.Resize((300, 300)),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\n\ntrainset     = MyDataset(lst, transform =train_transform)\ntrain_loader = torch.utils.data.DataLoader(lst, batch_size=48, shuffle=True, num_workers=4)\nvalset       = MyDataset(val_df, transform = train_transform)\nval_loader   = torch.utils.data.DataLoader(valset, batch_size=48, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5600b405f51d623922c315eef30612e91205bfff","_kg_hide-output":true},"cell_type":"code","source":"model = EfficientNet.from_name('efficientnet-b4')\nmodel.load_state_dict(torch.load('../input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth'))\nin_features = model._fc.in_features\nmodel._fc = nn.Linear(in_features, 40)\nmodel.cuda();\n\n\n#refinelabelmodel = EfficientNet.from_name('efficientnet-b3')\n#in_features = refinelabelmodel._fc.in_features\n#refinelabelmodel._fc = nn.Linear(in_features, 40)\n#refinelabelmodel.load_state_dict(torch.load('../input/garbage9291/acc_weight_best.pt'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\n\nclass LSR(nn.Module):\n\n    def __init__(self, e=0.1, reduction='mean'):\n        super().__init__()\n\n        self.log_softmax = nn.LogSoftmax(dim=1)\n        self.e = e\n        self.reduction = reduction\n    \n    def _one_hot(self, labels, classes, value=1):\n        \"\"\"\n            Convert labels to one hot vectors\n        \n        Args:\n            labels: torch tensor in format [label1, label2, label3, ...]\n            classes: int, number of classes\n            value: label value in one hot vector, default to 1\n        \n        Returns:\n            return one hot format labels in shape [batchsize, classes]\n        \"\"\"\n\n        one_hot = torch.zeros(labels.size(0), classes)\n\n        #labels and value_added  size must match\n        labels = labels.view(labels.size(0), -1)\n        value_added = torch.Tensor(labels.size(0), 1).fill_(value)\n\n        value_added = value_added.to(labels.device)\n        one_hot = one_hot.to(labels.device)\n\n        one_hot.scatter_add_(1, labels.long(), value_added)\n\n        return one_hot\n\n    def _smooth_label(self, target, length, smooth_factor):\n        \"\"\"convert targets to one-hot format, and smooth\n        them.\n        Args:\n            target: target in form with [label1, label2, label_batchsize]\n            length: length of one-hot format(number of classes)\n            smooth_factor: smooth factor for label smooth\n        \n        Returns:\n            smoothed labels in one hot format\n        \"\"\"\n        one_hot = self._one_hot(target, length, value=1 - smooth_factor)\n        one_hot += smooth_factor / length\n\n        return one_hot.to(target.device)\n\n    def forward(self, x, target):\n\n        if x.size(0) != target.size(0):\n            raise ValueError('Expected input batchsize ({}) to match target batch_size({})'\n                    .format(x.size(0), target.size(0)))\n\n        if x.dim() < 2:\n            raise ValueError('Expected input tensor to have least 2 dimensions(got {})'\n                    .format(x.size(0)))\n\n        if x.dim() != 2:\n            raise ValueError('Only 2 dimension tensor are implemented, (got {})'\n                    .format(x.size()))\n\n\n        smoothed_target = self._smooth_label(target, x.size(1), self.e)\n        x = self.log_softmax(x)\n        loss = torch.sum(- x * smoothed_target, dim=1)\n\n        if self.reduction == 'none':\n            return loss\n        \n        elif self.reduction == 'sum':\n            return torch.sum(loss)\n        \n        elif self.reduction == 'mean':\n            return torch.mean(loss)\n        \n        else:\n            raise ValueError('unrecognized option, expect reduction to be one of none, mean, sum')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_weights(net):\n    \"\"\"split network weights into to categlories,\n    one are weights in conv layer and linear layer,\n    others are other learnable paramters(conv bias, \n    bn weights, bn bias, linear bias)\n    Args:\n        net: network architecture\n    \n    Returns:\n        a dictionary of params splite into to categlories\n    \"\"\"\n\n    decay = []\n    no_decay = []\n\n    for m in net.modules():\n        if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):\n            decay.append(m.weight)\n\n            if m.bias is not None:\n                no_decay.append(m.bias)\n        \n        else: \n            if hasattr(m, 'weight'):\n                no_decay.append(m.weight)\n            if hasattr(m, 'bias'):\n                no_decay.append(m.bias)\n        \n    assert len(list(net.parameters())) == len(decay) + len(no_decay)\n\n    return [dict(params=decay), dict(params=no_decay, weight_decay=0)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\nimport itertools as it\n#from torch.optim import Optimizer\n#credit - Lookahead implementation from LonePatient - https://github.com/lonePatient/lookahead_pytorch/blob/master/optimizer.py\n#credit2 - RAdam code by https://github.com/LiyuanLucasLiu/RAdam/blob/master/radam.py\n\n\nclass Ranger(Optimizer):\n    \n    def __init__(self, params, lr=1e-3, alpha=0.5, k=6, betas=(.9,0.999), eps=1e-8, weight_decay=0):\n        #parameter checks\n        if not 0.0 <= alpha <= 1.0:\n            raise ValueError(f'Invalid slow update rate: {alpha}')\n        if not 1 <= k:\n            raise ValueError(f'Invalid lookahead steps: {k}')\n        if not lr > 0:\n            raise ValueError(f'Invalid Learning Rate: {lr}')\n        if not eps > 0:\n            raise ValueError(f'Invalid eps: {eps}')\n        \n        #prep defaults and init torch.optim base\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        super().__init__(params,defaults)\n        \n        #now we can get to work...\n        for group in self.param_groups:\n            group[\"step_counter\"] = 0\n            #print(\"group step counter init\")\n                      \n        #look ahead params\n        self.alpha = alpha\n        self.k = k \n        \n        #radam buffer for state\n        self.radam_buffer = [[None,None,None] for ind in range(10)]\n        \n        #lookahead weights\n        self.slow_weights = [[p.clone().detach() for p in group['params']]\n                                for group in self.param_groups]\n        \n        #don't use grad for lookahead weights\n        for w in it.chain(*self.slow_weights):\n            w.requires_grad = False\n        \n    def __setstate__(self, state):\n        print(\"set state called\")\n        super(Ranger, self).__setstate__(state)\n       \n        \n    def step(self, closure=None):\n        loss = None\n        #note - below is commented out b/c I have other work that passes back the loss as a float, and thus not a callable closure.  \n        #Uncomment if you need to use the actual closure...\n        \n        #if closure is not None:\n            #loss = closure()\n            \n        #------------ radam\n        for group in self.param_groups:\n    \n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n    \n                p_data_fp32 = p.data.float()\n    \n                state = self.state[p]\n    \n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n    \n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n    \n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n    \n                state['step'] += 1\n                buffered = self.radam_buffer[int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n                    if N_sma > 5:\n                        step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = group['lr'] / (1 - beta1 ** state['step'])\n                    buffered[2] = step_size\n    \n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n    \n                if N_sma > 5:                    \n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n                else:\n                    p_data_fp32.add_(-step_size, exp_avg)\n    \n                p.data.copy_(p_data_fp32)\n        \n        \n        #---------------- end radam step\n        \n        #look ahead tracking and updating if latest batch = k\n        for group,slow_weights in zip(self.param_groups,self.slow_weights):\n            group['step_counter'] += 1\n            if group['step_counter'] % self.k != 0:\n                continue\n            for p,q in zip(group['params'],slow_weights):\n                if p.grad is None:\n                    continue\n                q.data.add_(self.alpha,p.data - q.data)\n                p.data.copy_(q.data)\n            \n        \n            \n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parameter = split_weights(model)\n#optimizer = torch.optim.SGD(parameter, lr = 2e-2, momentum=0.9, weight_decay=1e-4, nesterov=True)\n#optimizer = torch.optim.Adam(parameter, lr=lr, weight_decay=1e-5)\noptimizer = Ranger(parameter, lr=1e-3, weight_decay=5e-4)\ncriterion =  LSR()\nscheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[1, 3, 5, 7, 9, 20, 25, 30], gamma=0.5)\n#scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode=\"min\", factor=0.5, patience=3)\n#scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[1, 2, 3, 4, 5,7, 8, 9, 10], gamma=0.5)\n#scheduler = torch.optim.lr_scheduler.StepLR(optimizer,step_size=2, gamma=0.5)\n#scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.5)\nmodel, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\",verbosity=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c338feda0eee741964b4c3d736c30b1e0a7e3ace"},"cell_type":"code","source":"def train_model(epoch):\n    model.train() \n    avg_loss = 0.\n    correct = 0\n    total = 0\n    for idx, (imgs, labels) in enumerate(train_loader):\n        imgs_train, labels_train = imgs.cuda(), labels.cuda()\n        output_train = model(imgs_train)\n        loss = criterion(output_train,labels_train)\n        with amp.scale_loss(loss, optimizer) as scaled_loss:\n            scaled_loss.backward()\n        #loss.backward()\n        avg_loss += loss.item()\n        optimizer.step() \n        optimizer.zero_grad()\n        prediction = torch.argmax(output_train, 1)\n        correct += (prediction == labels_train).sum().float()\n        total += len(labels_train)\n    acc = (correct/total).cpu().detach().data.numpy()\n    avg_loss = avg_loss / (len(train_loader))\n    return acc, avg_loss\n\ndef test_model():\n    correct = 0\n    avg_val_loss = 0.\n    total = 0\n    model.eval()\n    with torch.no_grad():\n        for idx, (imgs, labels) in enumerate(val_loader):\n            imgs_vaild, labels_vaild = imgs.cuda(), labels.cuda()\n            output_test = model(imgs_vaild)\n            val_loss = criterion(output_test, labels_vaild)\n            avg_val_loss += val_loss.item() \n            prediction = torch.argmax(output_test, 1)\n            correct += (prediction == labels_vaild).sum().float()\n            total += len(labels_vaild)\n        val_acc = (correct/total).cpu().detach().data.numpy()\n        avg_val_loss = avg_val_loss/ (len(val_loader)) \n    return val_acc, avg_val_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3562bd2ec1b0650519ca196bfc0e60eb139ca180"},"cell_type":"code","source":"best_avg_loss = 100.0\nbest_avg_acc  = 0\nn_epochs      = 50\n\nfor epoch in range(n_epochs):\n    start_time   = time.time()\n    acc, avg_loss = train_model(epoch)\n    val_acc, avg_val_loss = test_model()\n    elapsed_time = time.time() - start_time \n    print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t acc={:.4f} \\t val_acc={:.4f} \\t time={:.2f}s'.format(\n        epoch + 1, n_epochs, avg_loss, avg_val_loss, acc, val_acc, elapsed_time))\n    \n    if avg_val_loss < best_avg_loss:\n        best_avg_loss = avg_val_loss\n        torch.save(model.state_dict(), 'loss_weight_best.pt')\n    if val_acc>=best_avg_acc:\n        best_avg_acc = val_acc\n        torch.save(model.state_dict(), 'acc_weight_best.pt')\n        \n    scheduler.step()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -r ./apex","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}