{"cells":[{"metadata":{},"cell_type":"markdown","source":"## APTOS2019 train kernel (PyTorch)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Flags for training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os \nprint(os.listdir('../input/xczdatasets'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# -*- coding: utf-8 -*-\nfrom __future__ import unicode_literals\nimport gc\nimport os\nimport random\nimport time\nfrom contextlib import contextmanager\nfrom pathlib import Path\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\nimport sys\nfrom functools import partial\nfrom sklearn.metrics import cohen_kappa_score\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.utils.data import DataLoader, Dataset\nsys.path.append('../input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/')\nfrom efficientnet_pytorch import EfficientNet\ntest_batch_size = 16\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nweights_path =  '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth'\nclass OptimizedRounder():\n    def __init__(self):\n        self.coef_ = 0\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef=[0.5, 1.5, 2.5, 3.5]):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']\nimport matplotlib.pyplot as plt\n\ndef predict( X, coef=[0.5, 1.5, 2.5, 3.5]):\n    X_p = np.copy(X)\n    for i, pred in enumerate(X_p):\n        if pred < coef[0]:\n            X_p[i] = 0\n        elif pred >= coef[0] and pred < coef[1]:\n            X_p[i] = 1\n        elif pred >= coef[1] and pred < coef[2]:\n            X_p[i] = 2\n        elif pred >= coef[2] and pred < coef[3]:\n            X_p[i] = 3\n        else:\n            X_p[i] = 4\n    return X_p\ndef crop_image1(img,tol=7):\n    # img is image data\n    # tol  is tolerance\n\n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]\n\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n\n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n        #         print(img.shape)\n        return img\ndef preprocess_image(image_path, desired_size=256):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = crop_image_from_gray(img)\n    img = cv2.resize(img, (desired_size,desired_size))\n    img = cv2.addWeighted(img,4,cv2.GaussianBlur(img, (0,0), desired_size/30) ,-4 ,128)\n\n    return img\n\ndef preprocess_image_old(image_path, desired_size=224):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    #img = crop_image_from_gray(img)\n    img = cv2.resize(img, (desired_size,desired_size))\n    img = cv2.addWeighted(img,4,cv2.GaussianBlur(img, (0,0), desired_size/40) ,-4 ,128)\n\n    return img\n\nclass MyDataset(Dataset):\n    def __init__(self,dataframe, root_dir,transform=None,train = False):\n        self.df =dataframe\n        self.transform = transform\n        self.train = train\n        self.root_dir = root_dir\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        p = self.df.id_code.values[idx]\n        if idx<10:\n            p = os.path.join(self.root_dir,str(p)+'.jpg')\n        else:\n            p = os.path.join(self.root_dir,str(p)+'.jpg')\n        if self.train:\n            image = preprocess_image_old(str(p))\n        else:\n            image = preprocess_image(str(p))\n        # image = transforms.ToPILImage()(image)\n\n        if self.transform:\n            image = self.transform(image)\n        return image\nfrom torchvision import transforms\ntest_transform = transforms.Compose([\n    # transforms.RandomHorizontalFlip(),\n    # transforms.RandomRotation((-120, 120)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\nroot_dir = '../input/xczdatasets/Original_Images/Testing_Set'\ntrain_df = pd.read_csv('../input/labels/test.csv')\n\ntestset        = MyDataset(train_df,root_dir,transform=test_transform)\ntest_loader    = torch.utils.data.DataLoader(testset, batch_size=test_batch_size, shuffle=False)\n#show image test\nimage = testset[random.randint(0, len(testset)-1)]\n\n\nclass ClassifierModule(nn.Sequential):\n    def __init__(self, n_features):\n        super().__init__(\n            nn.BatchNorm1d(n_features),\n            nn.Dropout(0.5),\n            nn.Linear(n_features, n_features),\n            nn.PReLU(),\n            nn.BatchNorm1d(n_features),\n            nn.Dropout(0.2),\n            nn.Linear(n_features, 1),\n        )\n\nclass CustomEfficientNet(nn.Module):\n    def __init__(self, model_name='efficientnet-b0', weights_path=None):\n        assert model_name in ('efficientnet-b0', 'efficientnet-b1', 'efficientnet-b2', 'efficientnet-b3', 'e fficientnet-b4', 'efficientnet-b5')\n        super().__init__()\n\n        self.net = EfficientNet.from_name(model_name)\n        self.net.load_state_dict(torch.load(weights_path))\n\n        n_features = self.net._fc.in_features\n\n        self.net._fc = ClassifierModule(n_features)\n\n    def forward(self, x):\n        return self.net(x)\n\nmodel = CustomEfficientNet(model_name='efficientnet-b0', weights_path=weights_path)\nmodel.load_state_dict(torch.load('../input/pretrained-model/efficientnet-b0_fold0_epoch12.pth'))\nmodel.cuda()\n# #\n\n\nmodel.eval()\nvalid_preds = np.zeros((len(testset)))\navg_val_loss = 0.\n\nfor i, images in enumerate(test_loader):\n    with torch.no_grad():\n        y_preds = model(images.to(device)).detach()\n\n    valid_preds[i * test_batch_size: (i+1) * test_batch_size] = y_preds[:, 0].to('cpu').numpy()\npred = predict(valid_preds)\nprint(np.squeeze(pred))\npred_label = pd.DataFrame({'id_code':pd.read_csv('../input/labels/test.csv').id_code.values,\n                           'labels':np.squeeze(pred).astype(int)})\nprint(pred_label.head())\npred_label.to_csv('train.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### train params","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_params = {\n    'n_splits': 5,\n    'n_epochs': 12,\n    'lr': 1e-3,\n    'base_lr': 1e-4,\n    'max_lr': 3e-3,\n    'step_factor': 6,\n    'train_batch_size': 32,\n    'train_image_size': 512,\n    'test_image_size': 512,\n    'test_batch_size': 32,\n    'accumulation_steps': 10,\n    \n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### packages","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# ! pip install torch==1.1.0","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-output":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# ! pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ../input/nvidiaapex/repository/NVIDIA-apex-39e153a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"import os\nos.listdir('../input/efficientnet-pytorch/')\n['efficientnet-b3-c8376fa2.pth',\n 'efficientnet-b0-08094119.pth',\n 'efficientnet-b5-586e6cc6.pth',\n 'efficientnet-b2-27687264.pth',\n 'efficientnet-b1-dbc7070a.pth',\n 'efficientnet-b7-dcc49843.pth',\n 'EfficientNet-PyTorch',\n 'efficientnet-b6-c76e70fd.pth',\n 'efficientnet-b4-e116e8b3.pth']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../input/efficientnet-my/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n# sys.path.append('../input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/')\n\n# \nsys.path.append('../input/efficientnet-my/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport random\nimport time\nfrom contextlib import contextmanager\nfrom pathlib import Path\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\nfrom fastprogress import master_bar, progress_bar\nfrom functools import partial\nfrom sklearn.metrics import cohen_kappa_score\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.utils.data import DataLoader, Dataset\n\n# import pretrainedmodels\nfrom efficientnet_pytorch import EfficientNet\n\nfrom albumentations import (\n    Compose, HorizontalFlip, IAAAdditiveGaussianNoise, Normalize, OneOf,\n    RandomBrightness, RandomContrast, Resize, VerticalFlip, Rotate, ShiftScaleRotate,\n    RandomBrightnessContrast, OpticalDistortion, GridDistortion, ElasticTransform, Cutout\n)\nfrom albumentations.pytorch import ToTensor\n\nfrom apex import amp\n\nfrom fastai.layers import Flatten, AdaptiveConcatPool2d","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### utils","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def init_logger(log_file='train.log'):\n    from logging import getLogger, DEBUG, FileHandler,  Formatter,  StreamHandler\n    \n    log_format = '%(asctime)s %(levelname)s %(message)s'\n    \n    stream_handler = StreamHandler()\n    stream_handler.setLevel(DEBUG)\n    stream_handler.setFormatter(Formatter(log_format))\n    \n    file_handler = FileHandler(log_file)\n    file_handler.setFormatter(Formatter(log_format))\n    \n    logger = getLogger('APTOS')\n    logger.setLevel(DEBUG)\n    logger.addHandler(stream_handler)\n    logger.addHandler(file_handler)\n    \n    return logger\n\nLOG_FILE = 'aptos-train.log'\nLOGGER = init_logger(LOG_FILE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_torch(seed=777):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nSEED = 777\nseed_torch(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def quadratic_weighted_kappa(y_hat, y):\n    return cohen_kappa_score(y_hat, y, weights='quadratic')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class OptimizedRounder():\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n\n        ll = quadratic_weighted_kappa(y, X_p)\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# NOTE: official CyclicLR implementation doesn't work now\n\nfrom torch.optim.lr_scheduler import _LRScheduler\n\nclass CyclicLR(_LRScheduler):\n    def __init__(self, optimizer, base_lr, max_lr, step_size, gamma=0.99, mode='triangular', last_epoch=-1):\n        self.optimizer = optimizer\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.gamma = gamma\n        self.mode = mode\n        assert mode in ['triangular', 'triangular2', 'exp_range']\n        super().__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        new_lr = []\n        # make sure that the length of base_lrs doesn't change. Dont care about the actual value\n        for base_lr in self.base_lrs:\n            cycle = np.floor(1 + self.last_epoch / (2 * self.step_size))\n            x = np.abs(float(self.last_epoch) / self.step_size - 2 * cycle + 1)\n            if self.mode == 'triangular':\n                lr = self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x))\n            elif self.mode == 'triangular2':\n                lr = self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x)) / float(2 ** (cycle - 1))\n            elif self.mode == 'exp_range':\n                lr = self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x)) * (self.gamma ** (self.last_epoch))\n            new_lr.append(lr)\n        return new_lr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# new_train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n# old_train = pd.read_csv('../input/retinopathy-train-2015/rescaled_train_896/trainLabels.csv')\n# print(new_train.shape)\n# print(old_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nold_train = old_train[['image','level']]\nold_train.columns = new_train.columns\nold_train.diagnosis.value_counts()\n\n# # path columns\nnew_train['id_code'] = '../input/aptos2019-blindness-detection/train_images/' + new_train['id_code'].astype(str) + '.png'\nold_train['id_code'] = '../input/retinopathy-train-2015/rescaled_train_896/rescaled_train_896/' + old_train['id_code'].astype(str) + '.png'\n\ntrain_df = old_train[:992].copy()\nval_df = new_train[:480].copy()\ntrain_df.head\nprint(train_df.shape)\nprint(val_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nimport matplotlib\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"root = '/kaggle/input/aptos2019-blindness-detection/'\ntrain_df = pd.read_csv(os.path.join(root, 'train.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nSEED = 125\nfig = plt.figure(figsize=(30,30))\nimg_list = []\nimg_size = []\nroot = '../input/aptos2019-blindness-detection/'\n# display 10 images from each class\n#for class_id in sorted(train_y.unique()):\nfor class_id in [0, 1, 2, 3, 4]:\n    for i, (idx, row) in enumerate(train_df.loc[train_df['diagnosis'] == class_id].sample(1, random_state=SEED).iterrows()):\n        ax = fig.add_subplot(1, 5, class_id+i+1 , xticks=[], yticks=[])\n        plt.subplots_adjust(left=None,bottom=None,right=None,top=None,wspace=0.05,hspace=0.05)\n        path = os.path.join(root, 'train_images', '{}.png'.format(row['id_code']))\n        image = cv2.imread(path)\n        image = cv2.resize(image,(256,256))\n        img_size.append(image.shape)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        #image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n        img_list.append(row['id_code'])\n        \n        plt.imshow(image)\n        ax.set_title('Grade: %d' % (class_id) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_df = train_df.sample(frac=1).reset_index(drop=True)\n# val_df = val_df.sample(frac=1).reset_index(drop=True)\n# print(train_df.shape)\n# print(val_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_image1(img,tol=7):\n    # img is image data\n    # tol  is tolerance\n        \n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]\n\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\ndef preprocess_image(image_path, desired_size=train_params['test_image_size']):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = crop_image_from_gray(img)\n    img = cv2.resize(img, (desired_size,desired_size))\n    img = cv2.addWeighted(img,4,cv2.GaussianBlur(img, (0,0), desired_size/30) ,-4 ,128)\n    \n    return img\n\ndef preprocess_image_old(image_path, desired_size=train_params['train_image_size']):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    #img = crop_image_from_gray(img)\n    img = cv2.resize(img, (desired_size,desired_size))\n    img = cv2.addWeighted(img,4,cv2.GaussianBlur(img, (0,0), desired_size/40) ,-4 ,128)\n    \n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyDataset(Dataset): \n    def __init__(self, dataframe, transform=None,train = True):\n        self.df = dataframe\n        self.transform = transform\n        self.train = train\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        label = self.df.diagnosis.values[idx]\n        label = np.expand_dims(label, -1)\n        label = torch.tensor(label).float()\n        \n        p = self.df.id_code.values[idx]\n        if self.train:\n            image = preprocess_image_old(str(p))\n        else:\n            image = preprocess_image(str(p))\n        image = transforms.ToPILImage()(image)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision import transforms\ntrain_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation((-120, 120)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\n\n# trainset     = MyDataset(train_df, transform =train_transform,train=True)\n# train_loader = torch.utils.data.DataLoader(trainset, batch_size=32, shuffle=True, num_workers=4)\n# valset     = MyDataset(val_df, transform=train_transform,train =False)\n# val_loader   = torch.utils.data.DataLoader(valset, batch_size=32, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# APTOS_DIR = Path('../input/aptos2019-blindness-detection')\n# APTOS_TRAIN_DIR = Path('../input/aptos-train-dataset')\n\n# APTOS_TRAIN_IMAGES = APTOS_TRAIN_DIR / 'aptos-train-images/aptos-train-images'\n\n# #APTOS_FOLDS = Path('../input/aptos-folds/folds.csv')\n# #APTOS_FOLDS = Path('../input/aptos-folds/jpeg_folds.csv')\n# #APTOS_TRAIN_FOLDS = Path('../input/aptos-folds/jpeg_folds_all.csv')\n# #APTOS_VALID_FOLDS = Path('../input/aptos-folds/png_folds_all.csv')\n# APTOS_TRAIN_FOLDS = Path('../input/aptos-folds/2015_5folds.csv')\n# APTOS_VALID_FOLDS = Path('../input/aptos-folds/2019_5folds.csv')\n\n# ID_COLUMN = 'id_code'\n# TARGET_COLUMN = 'diagnosis'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PRETRAINED_DIR = Path('../input/pytorch-pretrained-models')\nEFFICIENTNET_PRETRAINED_DIR = Path('../input/efficientnet-pytorch')\nos.listdir('../input/efficientnet-pytorch/')\nPRETRAINED_MAPPING = {\n    # ResNet\n    'resnet18': PRETRAINED_DIR / 'resnet18-5c106cde.pth', \n    'resnet34': PRETRAINED_DIR / 'resnet34-333f7ec4.pth',\n    'resnet50': PRETRAINED_DIR / 'resnet50-19c8e357.pth',\n    'resnet101': PRETRAINED_DIR / 'resnet101-5d3b4d8f.pth',\n    'resnet152': PRETRAINED_DIR / 'resnet152-b121ed2d.pth',\n\n    # ResNeXt\n    'resnext101_32x4d': PRETRAINED_DIR / 'resnext101_32x4d-29e315fa.pth',\n    'resnext101_64x4d': PRETRAINED_DIR / 'resnext101_64x4d-e77a0586.pth',\n\n    # WideResNet\n    #'wideresnet50'\n\n    # DenseNet\n    'densenet121': PRETRAINED_DIR / 'densenet121-fbdb23505.pth',\n    'densenet169': PRETRAINED_DIR / 'densenet169-f470b90a4.pth',\n    'densenet201': PRETRAINED_DIR / 'densenet201-5750cbb1e.pth',\n    'densenet161': PRETRAINED_DIR / 'densenet161-347e6b360.pth',\n\n    # SE-ResNet\n    'se_resnet50': PRETRAINED_DIR / 'se_resnet50-ce0d4300.pth',\n    'se_resnet101': PRETRAINED_DIR / 'se_resnet101-7e38fcc6.pth',\n    'se_resnet152': PRETRAINED_DIR / 'se_resnet152-d17c99b7.pth',\n\n    # SE-ResNeXt\n    'se_resnext50_32x4d': PRETRAINED_DIR / 'se_resnext50_32x4d-a260b3a4.pth',\n    'se_resnext101_32x4d': PRETRAINED_DIR / 'se_resnext101_32x4d-3b2fe3d8.pth',\n\n    # SE-Net\n    'senet154': PRETRAINED_DIR / 'senet154-c7b49a05.pth',\n\n    # InceptionV3\n    'inceptionv3': PRETRAINED_DIR / 'inception_v3_google-1a9a5a14.pth',\n\n    # InceptionV4\n    'inceptionv4': PRETRAINED_DIR / 'inceptionv4-8e4777a0.pth',\n\n    # BNInception\n    'bninception': PRETRAINED_DIR / 'bn_inception-52deb4733.pth',\n\n    # InceptionResNetV2\n    'inceptionresnetv2': PRETRAINED_DIR / 'inceptionresnetv2-520b38e4.pth',\n\n    # Xception\n    'xception': PRETRAINED_DIR / 'xception-43020ad28.pth',\n\n    # DualPathNet\n    'dpn68': PRETRAINED_DIR / 'dpn68-4af7d88d2.pth',\n    'dpn98': PRETRAINED_DIR / 'dpn98-722954780.pth',\n    'dpn131': PRETRAINED_DIR / 'dpn131-7af84be88.pth',\n    'dpn68b': PRETRAINED_DIR / 'dpn68b_extra-363ab9c19.pth',\n    'dpn92': PRETRAINED_DIR / 'dpn92_extra-fda993c95.pth',\n    'dpn107': PRETRAINED_DIR / 'dpn107_extra-b7f9f4cc9.pth',\n\n    # PolyNet\n    'polynet': PRETRAINED_DIR / 'polynet-f71d82a5.pth',\n\n    # NasNet-A-Large\n    'nasnetalarge': PRETRAINED_DIR / 'nasnetalarge-a1897284.pth',\n\n    # PNasNet-5-Large\n    'pnasnet5large': PRETRAINED_DIR / 'pnasnet5large-bf079911.pth',\n\n    # EfficientNet\n    'efficientnet-b0': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b0-08094119.pth',\n    'efficientnet-b1': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b1-dbc7070a.pth',\n    'efficientnet-b2': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b2-27687264.pth',\n    'efficientnet-b3': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b3-c8376fa2.pth',\n    'efficientnet-b4': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b4-e116e8b3.pth',\n    'efficientnet-b5': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b5-586e6cc6.pth',\n \n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class APTOSTrainDataset(Dataset):\n#     def __init__(self, image_dir, file_paths, labels, transform=None):\n#         self.image_dir = image_dir\n#         self.file_paths = file_paths\n#         self.labels = labels\n#         self.transform = transform\n        \n#     def __len__(self):\n#         return len(self.file_paths)\n\n#     def __getitem__(self, idx):\n#         file_path = f'{self.image_dir}/{self.file_paths[idx]}'\n#         label = torch.tensor(self.labels[idx]).float()\n        \n#         image = cv2.imread(file_path)\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n#         if self.transform:\n#             augmented = self.transform(image=image)\n#             image = augmented['image']\n        \n#         return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### transforms","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# from albumentations import ImageOnlyTransform\n\n# def crop_image_from_gray(img, tol=7):\n#     \"\"\"\n#     Crop out black borders\n#     https://www.kaggle.com/ratthachat/aptos-updated-preprocessing-ben-s-cropping\n#     \"\"\"  \n#     if img.ndim ==2:\n#         mask = img>tol\n#         return img[np.ix_(mask.any(1),mask.any(0))]\n#     elif img.ndim==3:\n#         gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n#         mask = gray_img>tol        \n#         check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n#         if (check_shape == 0):\n#             return img\n#         else:\n#             img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n#             img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n#             img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n#             img = np.stack([img1,img2,img3],axis=-1)\n#         return img\n\n\n# class CircleCrop(ImageOnlyTransform):\n#     def __init__(self, tol=7, always_apply=False, p=1.0):\n#         super().__init__(always_apply, p)\n#         self.tol = tol\n    \n#     def apply(self, img, **params):\n#         img = crop_image_from_gray(img)    \n    \n#         height, width, depth = img.shape    \n    \n#         x = int(width/2)\n#         y = int(height/2)\n#         r = np.amin((x,y))\n    \n#         circle_img = np.zeros((height, width), np.uint8)\n#         cv2.circle(circle_img, (x,y), int(r), 1, thickness=-1)\n#         img = cv2.bitwise_and(img, img, mask=circle_img)\n#         img = crop_image_from_gray(img)\n#         img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n#         return img \n    \n\n# class CircleCropV2(ImageOnlyTransform):\n#     def __init__(self, tol=7, always_apply=False, p=1.0):\n#         super().__init__(always_apply, p)\n#         self.tol = tol\n    \n#     def apply(self, img, **params):\n#         img = crop_image_from_gray(img)\n        \n#         height, width, depth = img.shape\n#         largest_side = np.max((height, width))\n#         img = cv2.resize(img, (largest_side, largest_side))\n    \n#         height, width, depth = img.shape    \n    \n#         x = int(width/2)\n#         y = int(height/2)\n#         r = np.amin((x,y))\n    \n#         circle_img = np.zeros((height, width), np.uint8)\n#         cv2.circle(circle_img, (x,y), int(r), 1, thickness=-1)\n#         img = cv2.bitwise_and(img, img, mask=circle_img)\n#         img = crop_image_from_gray(img)\n#         img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n#         return img ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def get_transforms(*, data):\n#     assert data in ('train', 'valid')\n    \n#     if data == 'train':\n#         return Compose([\n#             CircleCropV2(),\n#             Resize(256, 256),\n#             HorizontalFlip(p=0.5),\n#             VerticalFlip(p=0.5),\n#             Rotate(p=0.5), \n#             #ShiftScaleRotate(p=0.5),\n#             #RandomBrightnessContrast(brightness_limit=0.5, contrast_limit=0.5, p=0.5),\n#             #OpticalDistortion(distort_limit=(0.9,1.0), shift_limit=0.05, interpolation=1, border_mode=4, \n#             #                  value=None, always_apply=False, p=0.5),\n#             #GridDistortion(num_steps=5, distort_limit=0.3, interpolation=1, border_mode=4,\n#             #               value=None, always_apply=False, p=0.5),\n#             #ElasticTransform(alpha=1, sigma=50, alpha_affine=50, interpolation=1, border_mode=4,\n#             #                 value=None, always_apply=True, approximate=False, p=0.5),\n#             Cutout(p=0.25, max_h_size=25, max_w_size=25, num_holes=8),\n#             #OneOf([\n#             #    RandomBrightness(0.1, p=1),\n#             #    RandomContrast(0.1, p=1),\n#             #], p=0.25),\n#             RandomContrast(0.5, p=0.5),\n#             IAAAdditiveGaussianNoise(p=0.25),\n#             Normalize(\n#                 mean=[0.485, 0.456, 0.406],\n#                 std=[0.229, 0.224, 0.225],\n#             ),\n#             ToTensor(),\n#         ])\n    \n#     elif data == 'valid':\n#         return Compose([\n#             CircleCropV2(),\n#             Resize(256, 256),\n#             Normalize(\n#                 mean=[0.485, 0.456, 0.406],\n#                 std=[0.229, 0.224, 0.225],\n#             ),\n#             ToTensor(),\n#         ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def basic_conv(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=0, bn_momentum=0.1):\n    basicconv = nn.Sequential(\n        nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, dilation, bias=False),\n        nn.BatchNorm2d(out_channels, momentum=bn_momentum),\n        nn.ReLU()\n    )\n    return basicconv\nclass Basic_cell(nn.Module):\n    def __init__(self,in_channels=1280):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_channels,64,(1,1),padding=0)\n        self.relu = nn.ReLU()\n        self.conv2 = nn.Conv2d(64,16,(1,1),padding=0)\n        self.conv3 = nn.Conv2d(16,8,(1,1),padding=0)\n        self.conv4 = nn.Conv2d(8,1,(1,1),padding=0)\n        self.sig = nn.Sigmoid()\n    def forward(self, input):\n        x = self.relu(self.conv1(input))\n        x = self.relu(self.conv2(x))\n        x = self.relu(self.conv3(x))\n        x = self.sig(self.conv4(x))\n        return x\n\nclass Attention_Cell(nn.Module):\n    def __init__(self,in_channels):\n        super().__init__()\n        self.bn2 =  nn.BatchNorm2d(in_channels)\n        self.basic_cell = Basic_cell(in_channels)\n        self.GAP = nn.AdaptiveAvgPool2d((1,1))\n    def forward(self, input):\n        bn_x = self.bn2(input)\n        x = self.basic_cell(bn_x)\n      \n        multi_x = bn_x*x\n#         print('multi_x',multi_x.shape)\n        x = self.GAP(x)\n        multi_x = self.GAP(multi_x)\n#         print(\"GAP-multi-x\",multi_x.shape)\n#         print(\"GAP-x\",x.shape)\n        output  = torch.div(multi_x,x)\n#         print(\"output\",output)\n\n        return output\n\nclass RegressionDense(nn.Sequential):\n    def __init__(self, in_channels):\n        super().__init__(\n            nn.BatchNorm1d(in_channels),\n            nn.Dropout(0.5),\n            nn.Linear(in_channels, 1280),\n            nn.PReLU(),\n            nn.BatchNorm1d(1280),\n            nn.Dropout(0.2),\n            nn.Linear(1280, 1),\n        )\n\nclass ClassfierDense(nn.Sequential):\n    def __init__(self, in_channels):\n        super().__init__(\n            nn.BatchNorm1d(in_channels),\n            nn.Dropout(0.5),\n            nn.Linear(in_channels, 1280),\n            nn.PReLU(),\n            nn.BatchNorm1d(1280),\n            nn.Dropout(0.2),\n            nn.Linear(1280, 5),\n#             nn.Softmax(),\n        )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomEfficientNet_Att(nn.Module):\n    def __init__(self, model_name='efficientnet-b0', weights_path=None):\n        assert model_name in ('efficientnet-b0', 'efficientnet-b1', 'efficientnet-b2', 'efficientnet-b3', 'e fficientnet-b4', 'efficientnet-b5')\n        super().__init__()\n        in_channels = 1280\n        if model_name is 'efficientnet-b0':\n            in_channels = 1280\n        if model_name is 'efficientnet-b3':\n            in_channels = 1536\n        self.basic_net = EfficientNet.from_name(model_name)\n        pretrained_dict = torch.load(weights_path)\n#         n_features = self.basic_net._conv_head.in_features\n        model_dict = self.basic_net.state_dict()\n        state_dict = {}\n        for k, v in pretrained_dict.items():\n            if k in model_dict.keys():\n                # state_dict.setdefult(k, v)\n                state_dict[k] = v\n        model_dict.update(state_dict)  # 更新(合并)模型的参数\n        self.basic_net.load_state_dict(model_dict)\n        self.attention_cell= Attention_Cell(in_channels)\n        self.regress = RegressionDense(in_channels)\n        self.classfier = ClassfierDense(in_channels)\n    def forward(self, x):\n        x = self.basic_net(x)\n        x = self.attention_cell(x)\n#         print(\"注意力\",x.shape)\n        x = x.squeeze(-1).squeeze(-1)\n#         print(\"注意力\",x.shape)\n        pred_1 = self.regress(x)\n        pred_2 = self.classfier(x)\n        return pred_1,pred_2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomResnet_Att(nn.Module): \n    def __init__(self, model_name='resnet50', weights_path=None):\n        assert model_name in ('resnet50', 'resnet101', 'resnet152')\n        super().__init__()\n        \n        self.basic_net = pretrainedmodels.__dict__[model_name](pretrained=None)\n        pretrained_dict = self.basic_net.load_state_dict(torch.load(weights_path))\n#         self.basic_net = EfficientNet.from_name(model_name)\n#         pretrained_dict = torch.load(weights_path)\n#         n_features = self.basic_net._conv_head.in_features\n        model_dict = self.basic_net.state_dict()\n        state_dict = {}\n        for k, v in pretrained_dict.items():\n            if k in model_dict.keys():\n                # state_dict.setdefult(k, v)\n                state_dict[k] = v\n        model_dict.update(state_dict)  # 更新(合并)模型的参数\n        self.basic_net.load_state_dict(model_dict)\n        \n        self.attention_cell= Attention_Cell()\n        self.regress = RegressionDense(1280)\n        self.classfier = ClassfierDense(1280)\n    def forward(self, x):\n        x = self.basic_net(x)\n        x = self.attention_cell(x)\n#         print(\"注意力\",x.shape)\n        x = x.squeeze(-1).squeeze(-1)\n#         print(\"注意力\",x.shape)\n        pred_1 = self.regress(x)\n        pred_2 = self.classfier(x)\n        return pred_1,pred_2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ClassifierModule(nn.Sequential):\n    def __init__(self, n_features):\n        super().__init__(\n            nn.BatchNorm1d(n_features),\n            nn.Dropout(0.5),\n            nn.Linear(n_features, n_features),\n            nn.PReLU(),\n            nn.BatchNorm1d(n_features),\n            nn.Dropout(0.2),\n            nn.Linear(n_features, 1),\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class CustomResNet(nn.Module):\n    def __init__(self, model_name='resnet50', weights_path=None):\n        assert model_name in ('resnet50', 'resnet101', 'resnet152')\n        super().__init__()\n        \n        self.net = pretrainedmodels.__dict__[model_name](pretrained=None)\n        self.net.load_state_dict(torch.load(weights_path))\n        \n        n_features = self.net.last_linear.in_features\n        \n        self.net.avgpool = nn.AdaptiveAvgPool2d(1)\n        # self.net.avgpool = AdaptiveConcatPool2d(1)\n        self.net.last_linear = ClassifierModule(n_features)\n        \n    def forward(self, x):\n        return self.net(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class CustomResNeXt(nn.Module):\n    def __init__(self, model_name='resnext101_32x4d', weights_path=None):\n        assert model_name in ('resnext101_32x4d', 'resnext101_64x4d')\n        super().__init__()\n        \n        self.net = pretrainedmodels.__dict__[model_name](pretrained=None)\n        self.net.load_state_dict(torch.load(weights_path))\n        \n        n_features = self.net.last_linear.in_features\n        \n        self.net.avg_pool = nn.AdaptiveAvgPool2d(1)\n        # self.net.avg_pool = AdaptiveConcatPool2d(1)\n        self.net.last_linear = ClassifierModule(n_features)\n        \n    def forward(self, x):\n        return self.net(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class CustomSENet(nn.Module):\n    def __init__(self, model_name='se_resnet50', weights_path=None):\n        assert model_name in ('senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152', 'se_resnext50_32x4d', 'se_resnext101_32x4d')\n        super().__init__()\n        \n        self.net = pretrainedmodels.__dict__[model_name](pretrained=None)\n        self.net.load_state_dict(torch.load(weights_path))\n        \n        n_features = self.net.last_linear.in_features\n        \n        self.net.avg_pool = nn.AdaptiveAvgPool2d(1)\n        # self.net.avg_pool = AdaptiveConcatPool2d(1)\n        self.net.last_linear = ClassifierModule(n_features)\n        \n    def forward(self, x):\n        return self.net(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomEfficientNet(nn.Module):\n    def __init__(self, model_name='efficientnet-b0', weights_path=None):\n        assert model_name in ('efficientnet-b0', 'efficientnet-b1', 'efficientnet-b2', 'efficientnet-b3', 'e fficientnet-b4', 'efficientnet-b5')\n        super().__init__()\n        \n        self.net = EfficientNet.from_name(model_name)\n        self.net.load_state_dict(torch.load(weights_path))\n\n        n_features = self.net._fc.in_features\n        \n        self.net._fc = ClassifierModule(n_features)\n        \n    def forward(self, x):\n        return self.net(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = CustomEfficientNet_Att(model_name=MODEL, weights_path=PRETRAINED_MAPPING[MODEL])\n# model.to(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### entry point","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"LOGGER.debug(f'Fold: {FOLD}')\nLOGGER.debug(f'Model: {MODEL}')\nLOGGER.debug(f'Train params: {train_params}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer('Prepare train and valid sets'):\n#     with timer('  * load folds csv'):\n#         #folds = pd.read_csv(APTOS_FOLDS)\n#         #train_fold = folds[folds['fold'] != FOLD].reset_index(drop=True)\n#         #valid_fold = folds[folds['fold'] == FOLD].reset_index(drop=True)\n#         folds = pd.read_csv(APTOS_TRAIN_FOLDS)\n#         train_fold = folds[folds['fold'] != FOLD].reset_index(drop=True)\n#         #valid_fold2015 = folds[folds['fold'] == FOLD].reset_index(drop=True)\n#         #valid_fold2019 = pd.read_csv(APTOS_VALID_FOLDS)\n#         #valid_fold = pd.concat([valid_fold2015, valid_fold2019]).reset_index(drop=True)\n#         valid_fold = pd.read_csv(APTOS_VALID_FOLDS)\n    \n    with timer('  * define dataset'):\n        train_dataset     = MyDataset(train_df, transform =train_transform,train=True)\n       \n        valid_dataset     = MyDataset(val_df, transform=train_transform,train =False)\n\n#         APTOSTrainDataset = partial(APTOSTrainDataset, image_dir=APTOS_TRAIN_IMAGES)\n#         train_dataset = APTOSTrainDataset(file_paths=train_fold.id_code.values,\n#                                           labels=train_fold.diagnosis.values[:, np.newaxis],\n#                                           transform=get_transforms(data='train'))\n#         valid_dataset = APTOSTrainDataset(file_paths=valid_fold.id_code.values,\n#                                           labels=valid_fold.diagnosis.values[:, np.newaxis],\n#                                           transform=get_transforms(data='valid'))\n        \n    with timer('  * define dataloader'):\n        train_loader = DataLoader(train_dataset,\n                                  batch_size=train_params['train_batch_size'],\n                                  shuffle=True)\n        valid_loader = DataLoader(valid_dataset,\n                                  batch_size=train_params['test_batch_size'],\n                                  shuffle=False)\n        \nLOGGER.debug(f'train size: {len(train_dataset)}, valid size: {len(valid_dataset)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def multi_loss(pred_reg,pred_classfier,y_true,multi_method='mse_ce',alpha = 1,beta =1):\n    assert multi_method in ('mse_ce','')\n    loss = 0;c1=0;c2=0\n    if multi_method  =='mse_ce':\n        c1 = nn.MSELoss()\n        c2 = nn.CrossEntropyLoss()\n    loss_reg =  c1(pred_reg,y_true)\n    y_true = y_true.long()\n#     print(pred_classfier.shape,y_true.shape)\n    loss_class = c2(pred_classfier,y_true.squeeze())\n    loss = alpha*loss_reg + beta*loss_class\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# with timer('Train model'):\n#     n_epochs = train_params['n_epochs']\n#     lr = train_params['lr']\n#     base_lr = train_params['base_lr']\n#     max_lr = train_params['max_lr']\n#     step_factor = train_params['step_factor']\n#     test_batch_size = train_params['test_batch_size']\n#     accumulation_steps = train_params['accumulation_steps']\n    \n#     model = CustomEfficientNet_Att(model_name=MODEL, weights_path=PRETRAINED_MAPPING[MODEL])\n#     model.to(device)\n    \n#     optimizer = Adam(model.parameters(), lr=lr, amsgrad=False)\n#     #optimizer = SGD(model.parameters(), lr=lr, weight_decay=4e-5, momentum=0.9, nesterov=True)\n#     scheduler = CyclicLR(optimizer,\n#                          base_lr=base_lr,\n#                          max_lr=max_lr,\n#                          step_size=len(train_loader) * step_factor)\n\n#     model, optimizer = amp.initialize(model, optimizer, opt_level='O1', verbosity=0)\n    \n#     criterion = nn.MSELoss()\n#     #criterion = nn.SmoothL1Loss()\n    \n    \n#     optimized_rounder = OptimizedRounder()\n# #     optimized_rounder2 = OptimizedRounder()\n#     y_true = val_df.diagnosis.values\n    \n#     for epoch in range(n_epochs):\n#         start_time = time.time()\n\n#         model.train()\n#         avg_loss = 0.\n\n#         optimizer.zero_grad()\n\n#         for i, (images, labels) in enumerate(train_loader):\n#             if isinstance(scheduler, CyclicLR):\n#                 scheduler.step()\n#             images = images.to(device)\n# #             images = images.float()\n#             pred_reg,pred_cls = model(images)\n# #             labels = labels.float()\n# #             loss = criterion(y_preds, labels.to(device))\n#             loss  = multi_loss(pred_reg,pred_cls,labels.to(device))\n#             with amp.scale_loss(loss, optimizer) as scaled_loss:\n#                 scaled_loss.backward()\n\n#             if (i+1) % accumulation_steps == 0:\n#                 optimizer.step()\n#                 optimizer.zero_grad()\n\n#             avg_loss += loss.item() / accumulation_steps / len(train_loader)\n\n#         if not isinstance(scheduler, CyclicLR):\n#             scheduler.step()\n\n#         model.eval()\n#         valid_preds = np.zeros((len(valid_dataset)))\n#         avg_val_loss = 0.\n\n#         for i, (images, labels) in enumerate(valid_loader):\n#             with torch.no_grad():\n# #                 y_preds = model(images.to(device)).detach()\n#                   pred_reg,pred_cls = model(images.to(device))\n#                   pred_reg = pred_reg.detach()\n# #             loss = criterion(y_preds, labels.to(device))\n#             loss =  multi_loss(pred_reg,pred_cls,labels.to(device))\n#             valid_preds[i * test_batch_size: (i+1) * test_batch_size] = pred_reg[:, 0].to('cpu').numpy()\n#             avg_val_loss += loss.item() / len(valid_loader)\n\n# #         optimized_rounder.fit(valid_preds, y_true)\n#         optimized_rounder.fit(valid_preds, y_true)\n# #         optimized_rounder2.fit(pred_cls, y_true)\n#         coefficients = optimized_rounder.coefficients()\n# #         coefficients_cls = optimized_rounder2.coefficients()\n#         final_preds = optimized_rounder.predict(valid_preds, coefficients)\n#         qwk = quadratic_weighted_kappa(y_true, final_preds)\n\n#         elapsed = time.time() - start_time\n\n#         LOGGER.debug(f'  Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n#         LOGGER.debug(f'          - qwk: {qwk:.6f}  coefficients: {coefficients}')\n\n#         # FIXME: save all epochs for debug \n#         torch.save(model.state_dict(), f'{MODEL}_fold{FOLD}_epoch{epoch+1}.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":".softmax(input, self.dim, _stacklevel=5)\n2020-07-02 02:27:20,329 DEBUG   Epoch 1 - avg_train_loss: 0.3125  avg_val_loss: 4.7410  time: 112s\n2020-07-02 02:27:20,330 DEBUG           - qwk: 0.000000  coefficients: [0.5 1.5 2.5 3.5]\n2020-07-02 02:29:02,828 DEBUG   Epoch 2 - avg_train_loss: 0.3046  avg_val_loss: 5.4275  time: 102s\n2020-07-02 02:29:02,830 DEBUG           - qwk: 0.271479  coefficients: [-0.542408  2.047389  3.62589   5.926311]\n2020-07-02 02:30:45,871 DEBUG   Epoch 3 - avg_train_loss: 0.3203  avg_val_loss: 4.9421  time: 103s\n2020-07-02 02:30:45,872 DEBUG           - qwk: 0.060628  coefficients: [0.5 1.5 2.5 3.5]\n2020-07-02 02:32:29,153 DEBUG   Epoch 4 - avg_train_loss: 0.2932  avg_val_loss: 2.9958  time: 103s\n2020-07-02 02:32:29,155 DEBUG           - qwk: 0.602941  coefficients: [0.276694 1.188741 0.554224 7.733649]\n2020-07-02 02:34:12,737 DEBUG   Epoch 5 - avg_train_loss: 0.2489  avg_val_loss: 3.6941  time: 104s\n2020-07-02 02:34:12,741 DEBUG           - qwk: 0.364468  coefficients: [0.40734  0.359478 2.882061 4.309653]\n2020-07-02 02:35:55,522 DEBUG   Epoch 6 - avg_train_loss: 0.2240  avg_val_loss: 2.6687  time: 103s\n2020-07-02 02:35:55,523 DEBUG           - qwk: 0.607598  coefficients: [0.494824 0.815553 2.177532 4.449734]\n2020-07-02 02:37:39,348 DEBUG   Epoch 7 - avg_train_loss: 0.1965  avg_val_loss: 4.0795  time: 104s\n2020-07-02 02:37:39,349 DEBUG           - qwk: 0.653501  coefficients: [0.596621 1.498057 1.842185 4.625971]\n2020-07-02 02:39:22,939 DEBUG   Epoch 8 - avg_train_loss: 0.1642  avg_val_loss: 2.6657  time: 104s\n2020-07-02 02:39:22,942 DEBUG           - qwk: 0.713007  coefficients: [0.461684 1.598026 1.734047 4.394338]\n2020-07-02 02:41:06,369 DEBUG   Epoch 9 - avg_train_loss: 0.1579  avg_val_loss: 2.8600  time: 103s\n2020-07-02 02:41:06,371 DEBUG           - qwk: 0.635940  coefficients: [0.519184 1.490187 2.401873 3.643743]\n2020-07-02 02:42:49,061 DEBUG   Epoch 10 - avg_train_loss: 0.1606  avg_val_loss: 2.5784  time: 103s\n2020-07-02 02:42:49,063 DEBUG           - qwk: 0.563788  coefficients: [0.516132 0.831461 2.752982 3.96373 ]\n2020-07-02 02:44:32,402 DEBUG   Epoch 11 - avg_train_loss: 0.1453  avg_val_loss: 2.4899  time: 103s\n2020-07-02 02:44:32,404 DEBUG           - qwk: 0.559555  coefficients: [0.602128 0.988473 2.593635 3.840373]\n2020-07-02 02:46:15,584 DEBUG   Epoch 12 - avg_train_loss: 0.1299  avg_val_loss: 2.4178  time: 103s\n2020-07-02 02:46:15,586 DEBUG           - qwk: 0.618733  coefficients: [0.715129 1.184878 1.110287 4.461294]\n2020-07-02 02:46:15,646 INFO [Train model] done in 1247 s.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}