{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\n\nfrom torchvision import models, transforms\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom matplotlib import pyplot as plt\n\nfrom PIL import Image\n\nfrom pathlib import Path\nPath.ls = lambda x: list(x.iterdir())\nimport re\nfrom functools import partial\n\nfrom typing import *\nimport math\nfrom IPython.core.debugger import set_trace\nfrom sklearn.metrics import roc_auc_score\nimport time\nfrom fastprogress.fastprogress import format_time\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def listify(o):\n    if o is None: return []\n    if isinstance(o, list): return o\n    if isinstance(o, str): return [o]\n    if isinstance(o, Iterable): return list(o)\n    return [o]\n\ndef compose(x, funcs, *args, order_key='_order', **kwargs):\n    key = lambda o: getattr(o, order_key,0)\n    for f in sorted(listify(funcs), key=key): x = f(x,**kwargs)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def annealer(f):\n    def _inner(start,end): return partial(f,start,end)\n    return _inner\n\n@annealer\ndef sched_lin(start,end,pos): return start+pos*(end-start)\n\n@annealer\ndef sched_cos(start,end,pos): return start + (1+math.cos(math.pi*(1-pos))) * (end-start) / 2\n@annealer\ndef sched_no(start,end,pos): return start\n@annealer\ndef sched_exp(start, end, pos): return start * (end/start) ** pos\n\ndef cos_1cycle_anneal(start, high, end):\n    return [sched_cos(start,high), sched_cos(high,end)]\n\ndef combine_scheds(pcts, scheds):\n    assert sum(pcts)==1.\n    pcts = torch.tensor([0]+listify(pcts))\n    assert torch.all(pcts >= 0)\n    pcts = torch.cumsum(pcts, 0)\n    def _inner(pos):\n        idx = (pos >= pcts).nonzero().max()\n        actual_pos = (pos-pcts[idx])/ (pcts[idx+1] - pcts[idx])\n        return scheds[idx](actual_pos)\n    return _inner","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass CancelBatchException(Exception): pass\nclass CancelTrainException(Exception): pass\nclass CancelEpochException(Exception): pass\n\n\n_camel_re1 = re.compile('(.)([A-Z][a-z]+)')\n_camel_re2 = re.compile('([a-z0-9])([A-Z])')\n\ndef camel2snake(name):\n    s1 = re.sub(_camel_re1, r'\\1_\\2',name)\n    return re.sub(_camel_re2,r'\\1_\\2',s1).lower()\n\nclass Callback():\n    _order = 0\n    def set_runner(self,run):self.run=run\n    def __getattr__(self,k): return getattr(self.run, k)\n    @property\n    def name(self):\n        name = re.sub(r'Callback$','',self.__class__.__name__)\n        return camel2snake(name or 'callback')\n\n    def __call__(self,cb_name):\n        f = getattr(self, cb_name, None)\n        if f and f(): return True\n        return False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"###########\n# AvgStats\n###########\nclass AvgStats():\n    def __init__(self,metrics,in_train): self.metrics,self.in_train = listify(metrics),in_train\n\n    def reset(self):\n        self.tot_loss, self.count = 0., 0\n        self.tot_mets = [0.] * len(self.metrics)\n\n    @property\n    def all_stats(self): return [self.tot_loss.item()] + self.tot_mets\n    @property\n    def avg_stats(self): return [o/self.count for o in self.all_stats]\n\n    def __repr__(self):\n        if not self.count: return \"\"\n        return f\"{'train' if self.in_train else 'valid'}: {self.avg_stats}\"\n\n    def accumulate(self,run):\n        bn = run.xb.shape[0]\n        self.tot_loss += run.loss *bn\n        self.count += bn\n        for i,m in enumerate(self.metrics):\n            self.tot_mets[i]+=m(run.yb.cpu(),run.pred.detach().cpu())*bn\n\n########################\n#BatchTransformXCallback\n########################\nclass BatchTransformXCallback(Callback):\n    def __init__(self,tfm): self.tfm = tfm\n    \n    def begin_batch(self): self.run.xb = self.tfm(self.xb)\n#######################\n#CudaCallback\n#######################\nclass CudaCallback(Callback):\n    def begin_fit(self): self.model.cuda()\n    \n    def begin_batch(self): self.run.xb, self.run.yb = self.xb.cuda(), self.yb.cuda()\n##################\n#AvgStatsCallback\n##################\nclass AvgStatsCallback(Callback):\n    def __init__(self, metrics):\n        self.train_stats, self.valid_stats = AvgStats(metrics, True), AvgStats(metrics, False)\n        \n    def begin_fit(self):\n        met_names = ['loss'] + [m.__name__ for m in self.train_stats.metrics]\n        names = ['epoch'] + [f'train_{n}' for n in met_names] + [\n            f'valid_{n}' for n in met_names] + ['time']\n        self.logger(names)\n    \n    def begin_epoch(self):\n        self.train_stats.reset()\n        self.valid_stats.reset()\n        self.start_time = time.time()\n    \n    def after_loss(self):\n        set_trace()\n        stats = self.train_stats if self.in_train else self.valid_stats\n        with torch.no_grad(): stats.accumulate(self.run)\n            \n    def after_epoch(self):\n        stats = [str(self.epoch)]\n        for o in [self.train_stats, self.valid_stats]:\n            stats += [f'{v:6f}' for v in o.avg_stats]\n        stats += [format_time(time.time() - self.start_time)]\n        self.logger(stats)\n\n##################\n#LogStatsCallback\n##################\nclass LogStatsCallback(Callback):\n    def __init__(self, metrics):\n        self.train_stats, self.valid_stats = AccumulateStats(metrics, True), AccumulateStats(metrics, False)\n        \n    def begin_fit(self):\n        met_names = ['loss'] + [m.__name__ for m in self.train_stats.metrics]\n        names = ['epoch'] + [f'train_{n}' for n in met_names] + [\n            f'valid_{n}' for n in met_names] + ['time']\n        self.logger(names)\n    \n    def begin_epoch(self):\n        self.train_stats.reset()\n        self.valid_stats.reset()\n        self.start_time = time.time()\n    \n    def after_loss(self):\n        stats = self.train_stats if self.in_train else self.valid_stats\n        with torch.no_grad(): stats.accumulate(self.run)\n            \n    def after_epoch(self):\n        stats = [str(self.epoch)]\n        for o in [self.train_stats, self.valid_stats]:\n            stats += [f'{v:6f}' for v in o.avg_stats]\n        stats += [format_time(time.time() - self.start_time)]\n        self.logger(stats)\n        \n#############\n#Recorder\n#############\n\nclass Recorder(Callback):\n    def __init__(self,pnames):\n        self.pnames = listify(pnames)\n        self.hps = []\n\n    def begin_fit(self):\n        for pname in self.pnames:\n            self.hps.append([])\n        self.losses = []\n\n    def after_batch(self):\n        if not self.in_train: return\n        for hp,pname in zip(self.hps,self.pnames):\n            hp.append(self.opt.hypers[-1][pname])\n\n        self.losses.append(self.loss.detach().cpu())\n\n    def plot_lr  (self): plt.plot(self.hps[self.pnames.index('lr')])\n    def plot_mom (self): plt.plot(self.hps[self.pnames.index('mom')])\n    def plot_loss(self): plt.plot(self.losses)\n\n    def plot(self,skip_last=0,pgid=-1):\n        lrs = self.hps[self.pnames.index('lr')]\n        losses = [o.item() for o in self.losses]\n        n = len(self.losses) - skip_last\n        plt.xscale('log')\n        plt.plot(lrs[:n],losses[:n])\n\n\n######################\n#TrainEvalCallback\n######################\n\nclass TrainEvalCallback(Callback):\n    def begin_fit(self):\n        self.run.n_epochs=0.\n        self.run.n_iter =0.\n    \n    def begin_epoch(self):\n        self.run.n_epochs = self.epoch\n        self.model.train()\n        self.run.in_train = True\n        \n    def begin_validate(self):\n        self.model.eval()\n        self.run.in_train=False\n    \n    def after_batch(self):\n        if not self.in_train: return\n        self.run.n_epochs += 1./self.iters\n        self.run.n_iter += 1\n\n######################\n#LR_Find\n######################\n\nclass LR_Find(Callback):\n    _order = 1\n    def __init__(self,max_iter=100,min_lr=1e-6,max_lr=10):\n        self.max_iter,self.min_lr,self.max_lr = max_iter,min_lr,max_lr\n        self.best_loss = 1e9\n        \n    def begin_batch(self):\n        if not self.in_train: return\n        pos = self.n_iter/self.max_iter\n        lr = self.min_lr * (self.max_lr/self.min_lr) ** pos\n        for pg in self.opt.hypers: pg['lr'] = lr\n    \n    def after_step(self):\n        if self.n_iter >= self.max_iter or self.loss > self.best_loss*10:\n            raise CancelTrainException()\n        if self.loss < self.best_loss: self.best_loss = self.loss\n\n##################\n#ParamScheduler\n################## \n\nclass ParamScheduler(Callback):\n    _order=1\n    def __init__(self, pname, sched_func):\n        self.pname,self.sched_func = pname, sched_func\n        \n    def set_param(self):\n        for h in self.opt.hypers:\n            h[self.pname] = self.sched_func(self.n_epochs/self.epochs)\n    \n    def begin_batch(self):\n        if self.in_train: self.set_param()\n\n##################\n#ProgressCallback\n##################\nfrom fastprogress.fastprogress import master_bar, progress_bar,ProgressBar\n\nclass ProgressCallback(Callback):\n    _order=-1\n    def begin_fit(self):\n        self.mbar = master_bar(range(self.epochs))\n        self.mbar.on_iter_begin()\n        self.run.logger = partial(self.mbar.write, table=True)\n        \n    def after_fit(self): self.mbar.on_iter_end()\n    def after_batch(self): self.pb.update(self.iter)\n    def begin_epoch   (self): self.set_pb()\n    def begin_validate(self): self.set_pb()\n    \n    def set_pb(self):\n        self.pb = progress_bar(self.dl, parent=self.mbar)\n        self.mbar.update(self.epoch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sgd_step(p, lr, **kwargs):\n    p.data.add_(p.grad.data, alpha=-lr)\n    return p\n\ndef weight_decay(p, lr, wd, **kwargs):\n    p.data.mul_(1-lr*wd)\n    return p\nweight_decay._defaults = dict(wd=0.)\n\ndef maybe_update(os, dest, f):\n    for o in os:\n        for k,v in f(o).items():\n            if k not in dest: dest[k] = v\ndef get_defaults(d): return getattr(d, '_defaults', {})\n\nclass Optimizer():\n    def __init__(self, params, steppers, **defaults):\n        self.steppers = listify(steppers)\n        maybe_update(self.steppers,defaults, get_defaults)\n        self.param_groups = list(params)\n        if not isinstance(self.param_groups[0], list): self.param_groups = [self.param_groups]\n        self.hypers = [{**defaults} for p in self.param_groups]\n       \n    def grad_params(self):\n        return [(p,hyper) for pg,hyper in zip(self.param_groups,self.hypers) \n               for p in pg if p.grad is not None]\n    \n    def zero_grad(self):\n        for p,hyper in self.grad_params():\n            p.grad.detach_()\n            p.grad.zero_()\n            \n    def step(self):\n        for p,hyper in self.grad_params(): compose(p, self.steppers, **hyper)\n\nsgd_opt = partial(Optimizer, steppers=[weight_decay,sgd_step])\n\nclass StatefulOptimizer(Optimizer):\n    def __init__(self, params, steppers, stats=None, **defaults):\n        self.stats = listify(stats)\n        maybe_update(self.stats, defaults, get_defaults)\n        super().__init__(params, steppers, **defaults)\n        self.state = {}\n    \n    def step(self):\n        for p,hyper in self.grad_params():\n            if p not in self.state:\n                #Create a state for p and call all the statistics to initialize it.\n                self.state[p] = {}\n                maybe_update(self.stats, self.state[p], lambda o: o.init_state(p))\n            state = self.state[p]\n            for stat in self.stats: state = stat.update(p, state, **hyper)\n            compose(p, self.steppers, **state, **hyper)\n            self.state[p] = state\n\nclass Stat():\n    _defaults = {}\n    def init_state(self,p): raise NotImplementedError\n    def update(self,p,state,**kwargs): raise NotImplementedError\n\nclass AverageGrad(Stat):\n    _defaults = dict(mom=0.9)\n    \n    def __init__(self, dampening:bool=False): self.dampening=dampening\n    def init_state(self,p): return {'grad_avg': torch.zeros_like(p.grad.data)}\n    def update(self,p,state,mom,**kwargs):\n        state['mom_damp'] = 1-mom if self.dampening else 1.\n        state['grad_avg'].mul_(mom).add_(state['mom_damp'],p.grad.data)\n        return state\n\ndef momentum_step(p, lr, grad_avg, **kwargs):\n    p.data.add_(grad_avg,alpha=-lr)\n    return p\n\nsgd_mom_opt = partial(StatefulOptimizer, steppers=[momentum_step, weight_decay], \n                     stats=AverageGrad(), wd=0.01)\n\n\nclass AverageSqrGrad(Stat):\n    _defaults = dict(sqr_mom=0.99)\n    \n    def __init__(self, dampening:bool=True): self.dampening=dampening\n    def init_state(self,p): return {'sqr_avg': torch.zeros_like(p.grad.data)}\n    def update(self, p, state, sqr_mom, **kwargs):\n        state['sqr_damp'] = 1-sqr_mom if self.dampening else 1.\n        state['sqr_avg'].mul_(sqr_mom).addcmul_(state['sqr_damp'], p.grad.data, p.grad.data)\n        return state\n\nclass StepCount(Stat):\n    def init_state(self, p): return {'step': 0}\n    def update(self, p, state, **kwargs):\n        state['step'] += 1\n        return state\n\ndef debias(mom, damp,step): return damp * (1-mom**step) / (1-mom)\n\ndef adam_step(p,lr,mom,mom_damp,step,sqr_mom, sqr_damp, grad_avg, sqr_avg, eps, **kwargs):\n    debias1 = debias(mom,     mom_damp, step)\n    debias2 = debias(sqr_mom, sqr_damp, step)\n    p.data.addcdiv_(-lr/debias1, grad_avg, (sqr_avg/debias2).sqrt()+eps)\n    return p\nadam_step._defaults = dict(eps=1e-5)\n\ndef adam_opt(xtra_step=None, **kwargs):\n    return partial(StatefulOptimizer, steppers=[adam_step, weight_decay]+listify(xtra_step),\n                  stats=[AverageGrad(dampening=True), AverageSqrGrad(), StepCount()],**kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def param_getter(m): return m.parameters()\n\n#Exceptions\n\nclass Learner():\n    def __init__(self, model, data, loss_func, opt_func, lr=1e-2, splitter=param_getter,\n                cbs=None, cb_funcs=None):\n        self.model, self.data, self.loss_func, self.opt_func, self.lr, self.splitter = model,data,loss_func,opt_func, lr, splitter\n        self.in_train, self.logger, self.opt = False,print,None\n        \n        self.cbs = []\n        self.add_cb(TrainEvalCallback())\n        self.add_cbs(cbs)\n        self.add_cbs(cbf() for cbf in listify(cb_funcs))\n        \n    def add_cbs(self,cbs):\n        for cb in listify(cbs): self.add_cb(cb)\n            \n    def add_cb(self,cb):\n        cb.set_runner(self)\n        setattr(self,cb.name,cb)\n        self.cbs.append(cb)\n        \n    def remove_cbs(self,cbs):\n        for cb in listify(cbs): self.cbs.remove(cb)\n            \n    def one_batch(self,i,xb,yb):\n        try: \n            self.iter = i\n            self.xb,self.yb = xb,yb;                        self('begin_batch')\n            self.pred = self.model(self.xb);                self('after_pred')\n            self.loss = self.loss_func(self.pred,self.yb);  self('after_loss')\n            if not self.in_train: return\n            self.loss.backward();                           self('after_backward')\n            self.opt.step();                                self('after_step')\n            self.opt.zero_grad()\n        except CancelBatchException:                        self('after_cancel_batch')\n        finally:                                            self('after_batch')\n            \n    def all_batches(self):\n        self.iters = len(self.dl)\n        try:\n            for i, (xb,yb) in enumerate(self.dl): self.one_batch(i,xb,yb)\n        except CancelEpochException: self('after_cancel_epoch')\n    \n    def do_begin_fit(self, epochs):\n        self.epochs, self.loss = epochs, torch.tensor(0.)\n        self('begin_fit')\n    \n    def do_begin_epoch(self,epoch):\n        self.epoch, self.dl = epoch, self.data.train_dl\n        return self('begin_epoch')\n    \n    def fit(self,epochs,cbs=None,reset_opt=False):\n        self.add_cbs(cbs)\n        \n        if reset_opt or not self.opt: self.opt = self.opt_func(self.splitter(self.model), lr=self.lr)\n            \n        try:\n            self.do_begin_fit(epochs)\n            for epoch in range(epochs):\n                if not self.do_begin_epoch(epoch): self.all_batches()\n                    \n                with torch.no_grad():\n                    self.dl = self.data.valid_dl\n                    if not self('begin_validate'): self.all_batches()\n                self('after_epoch')\n        except CancelTrainException: self('after_cancel_train')\n        finally:\n            self('after_fit')\n            self.remove_cbs(cbs)\n            \n    ALL_CBS = {'begin_batch','after_pred','after_loss','after_backward','after_step',\n              'after_cancel_batch', 'after_batch','after_cancel_epoch','begin_fit','begin_epoch',\n              'begin_validate','after_epoch','after_cancel_train','after_fit'}\n    \n    def __call__(self,cb_name):\n        res= False\n        assert cb_name in self.ALL_CBS\n        for cb in sorted(self.cbs, key=lambda x: x._order): res = cb(cb_name) and res\n        return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input/siim-isic-melanoma-classification/')\npath.ls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stratified Split for train and valid set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nkf = StratifiedKFold(n_splits=5)\n\ndf['valid'] = 0\n\nfor train_idx, valid_idx in kf.split(df, df['target']):\n    df.loc[valid_idx, 'valid'] = 1\n    break\n\ndf['valid'] = df['valid'].astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_train = Path('/kaggle/input/siic-isic-224x224-images')\npath_train.ls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset:\n    def __init__(self,path,df,folder='train',extension='.png',is_test=False,size=224):\n        self.path = Path(path)\n        self.df = df\n        self.folder = folder\n        self.extension = extension\n        self.is_test = is_test\n        self.size = size if isinstance(size,tuple) else (size,size)\n        self.tfms = transforms.Compose([transforms.Resize(self.size),\n            transforms.ToTensor(),transforms.Normalize(mean = [0.485, 0.456, 0.406],std = [0.229, 0.224, 0.225])])\n        \n    def __len__(self): return len(self.df)\n    def __getitem__(self,idx):\n        row = self.df.loc[idx,:]\n        file_name = row['image_name']+self.extension\n        \n        img = Image.open(self.path/self.folder/file_name)\n        img = self.tfms(img)\n        \n        if not self.is_test:\n            target = torch.tensor(row['target'],dtype=torch.long)\n            return img,target\n        return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataBunch():\n    def __init__(self, train_dl, valid_dl):\n        self.train_dl, self.valid_dl = train_dl, valid_dl\n        \n    @property\n    def train_ds(self): return self.train_dl.dataset\n    @property\n    def valid_ds(self): return self.valid_dl.dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, pretrained=False):\n        super().__init__()\n        self.model = models.densenet121(pretrained=pretrained).features\n        self.out = nn.Linear(1024,1)\n        \n    def forward(self, x):\n        x = F.adaptive_avg_pool2d(self.model(x),1)\n        x = self.out(x.view(x.shape[0],-1))\n        return x        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = df.loc[df['valid']==0, :].reset_index(drop=True)\nvalid_df = df.loc[df['valid']==1, :].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_ds = Dataset(path_train, train_df,size=128)\n# valid_ds = Dataset(path_train, valid_df,size=128)\n\n# train_dl = torch.utils.data.DataLoader(\n#     train_ds, \n#     batch_size=256,\n#     shuffle=True,\n#     num_workers=4\n# )\n# valid_dl = torch.utils.data.DataLoader(\n#     valid_ds,\n#     batch_size=256,\n#     shuffle=False,\n#     num_workers=4\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data = DataBunch(train_dl, valid_dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def loss_func(outputs, targets,**kwargs):\n    loss = nn.BCEWithLogitsLoss(**kwargs)(outputs.view(-1), targets.float())\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sched_lr = combine_scheds([0.3,0.7],cos_1cycle_anneal(4e-5,4e-4,4e-6))\nsched_mom = combine_scheds([0.3,0.7],cos_1cycle_anneal(0.94,0.85,0.94))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lin_comb (v1,v2,beta): return beta*v1 + (1-beta)*v2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NoneReduce():\n    def __init__(self, loss_func):\n        self.loss_func, self.old_red = loss_func, None\n    \n    def __enter__(self):\n        if hasattr(self.loss_func, 'reduction'):\n            self.old_red = getattr(self.loss_func, 'reduction')\n            setattr(self.loss_func,'reduction', 'none')\n            return self.loss_func\n        else: return partial(self.loss_func, reduction='none')\n    \n    def __exit__(self, type, value, traceback):\n        if self.old_red is not None: setattr(self.loss_func, 'reduction', self.old_red)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.distributions.beta import Beta\n\ndef unsqueeze(input, dims):\n    for dim in listify(dims): input = torch.unsqueeze(input,dim)\n    return input\n\ndef reduce_loss(loss, reduction='mean'):\n    return loss.mean() if reduction=='mean' else loss.sum() if reduction=='sum' else loss\n\nclass Mixup(Callback):\n    _order = 90 #Runs after normalization and cuda\n    def __init__(self, alpha:float=0.4): self.distrib = Beta(torch.tensor([alpha]), torch.tensor([alpha]))\n    \n    def begin_fit(self): self.old_loss_func, self.run.loss_func = self.run.loss_func, self.loss_func\n        \n    def begin_batch(self):\n        if not self.in_train: return #Only mixup things during training\n        L = self.distrib.sample((self.yb.size(0),)).squeeze().to(self.xb.device)\n        L = torch.stack([L, 1-L], 1)\n        self.L = unsqueeze(L.max(1)[0], (1,2,3))\n        shuffle = torch.randperm(self.yb.size(0)).to(self.xb.device)\n        xb1, self.yb1 = self.xb[shuffle], self.yb[shuffle]\n        self.run.xb = lin_comb(self.xb, xb1, self.L)\n    \n    def after_fit(self): self.run.loss_func = self.old_loss_func\n        \n    def loss_func(self, pred, yb):\n        if not self.in_train: return self.old_loss_func(pred,yb)\n        with NoneReduce(self.old_loss_func) as loss_func:\n            loss1 = loss_func(pred,yb)\n            loss2 = loss_func(pred, self.yb1)\n        loss = lin_comb(loss1, loss2, self.L)\n        return reduce_loss(loss, getattr(self.old_loss_func, 'reduction', 'mean'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cbfs=[partial(Recorder,['lr','mom']),CudaCallback,ProgressCallback]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_learner(data, loss_func, opt_func, lr=1e-3, cbfs=None, pretrained=False):\n    model = Model(pretrained)\n    learner = Learner(model, data, loss_func, opt_func=opt_func,lr=1e-3,cb_funcs=cbfs)\n    return learner","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learner = get_learner(data, loss_func, adam_opt(), cbfs=cbfs,pretrained=True)\n# learner.fit(1,cbs=[LR_Find(),Mixup()])\n# learner.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner = None\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##################\n#AccumulateStats\n##################\nclass AccumulateStats():\n    def __init__(self, metrics, in_train): self.metrics, self.in_train = listify(metrics), in_train\n    \n    def reset(self):\n        self.loss_list = []\n        self.met_list = [[]]*len(self.metrics)\n        self.preds_list = []\n        self.yb_list = []\n    \n    def accumulate(self, run):\n        self.loss_list.append(run.loss)\n        self.yb_list.append(run.yb.cpu())\n        for i,m in enumerate(self.metrics):\n            self.met_list[i].append(run.pred.detach().cpu())\n\n    def get_stats(self):\n        loss = torch.tensor(self.loss_list).mean()\n        metrics=[]\n        for i,m in enumerate(self.metrics):\n            metrics.append(m(torch.cat(self.yb_list),torch.cat(self.met_list[i])))\n        \n        return [loss]+metrics\n    @property\n    def avg_stats(self): return self.get_stats()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = Dataset(path_train, train_df,size=224)\nvalid_ds = Dataset(path_train, valid_df,size=224)\n\ntrain_dl = torch.utils.data.DataLoader(\n    train_ds, \n    batch_size=96,\n    shuffle=True\n)\nvalid_dl = torch.utils.data.DataLoader(\n    valid_ds,\n    batch_size=128,\n    shuffle=False\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = DataBunch(train_dl,valid_dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner = get_learner(data, loss_func, adam_opt(),cbfs=cbfs,pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.fit(5, cbs=[ParamScheduler('lr',sched_lr),ParamScheduler('mom',sched_mom),LogStatsCallback(roc_auc_score),Mixup()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learner.fit(1,cbs=[LR_Find()])\n# learner.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(path/'test.csv')\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = Dataset(path_train, test_df, folder='test',is_test=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dl = torch.utils.data.DataLoader(\n    test_ds,\n    batch_size=128,\n    shuffle=False\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(dataloader, model, device):\n    \n    predictions = []\n    model.eval()\n    model.to(device)\n    with torch.no_grad():\n        for x in tqdm(dataloader, total=len(dataloader)):\n            x = x.to(device)\n            preds = model(x)\n            predictions.append(preds.detach().cpu())\n\n    return torch.cat(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions =predict(test_dl,learner.model, 'cuda')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submissions = pd.read_csv(path/'sample_submission.csv')\nsubmissions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submissions['target'] = predictions.numpy()\nsubmissions.to_csv('submission.csv',index=False)\n\n\nsubmissions['target'] = torch.sigmoid(predictions).numpy()\nsubmissions.to_csv('submission_sigmoid.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}