{"cells":[{"metadata":{},"cell_type":"markdown","source":"## General information\n\nIn this kernel I work with data from APTOS 2019 Blindness Detection competition\n\n![](https://nei.nih.gov/sites/default/files/health-images/macula_dr.gif)\n\nOur task is to detect blindeness. This problem feels quite important for me - I'm not blind or near it, but I have a bad eyesight, so I know that problems with eye are serious.\n\nIn this kernel I'll do a basic EDA and train a baseline pytorch model.\n\nthis kernel forked from: https://www.kaggle.com/artgor/basic-eda-and-baseline-pytorch-model"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# libraries\nimport numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport time \nimport tqdm\nfrom PIL import Image\ntrain_on_gpu = True\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n\nimport cv2\nimport albumentations\nfrom albumentations import torch as AT","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data overview"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nsample_submission = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['diagnosis'].value_counts().plot(kind='bar');\nplt.title('Class counts');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have a slight disbalance in data."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(25, 16))\n# display 10 images from each class\nfor class_id in sorted(train['diagnosis'].unique()):\n    for i, (idx, row) in enumerate(train.loc[train['diagnosis'] == class_id].sample(10).iterrows()):\n        ax = fig.add_subplot(5, 10, class_id * 10 + i + 1, xticks=[], yticks=[])\n        im = Image.open(f\"../input/aptos2019-blindness-detection/train_images/{row['id_code']}.png\")\n        plt.imshow(im)\n        ax.set_title(f'Label: {class_id}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Personally I see little differences between images"},{"metadata":{},"cell_type":"markdown","source":"## Data preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder, OneHotEncoder\ndef prepare_labels(y):\n    # From here: https://www.kaggle.com/pestipeti/keras-cnn-starter\n    values = np.array(y)\n    label_encoder = LabelEncoder()\n    integer_encoded = label_encoder.fit_transform(values)\n\n    onehot_encoder = OneHotEncoder(sparse=False)\n    integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n    onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n\n    y = onehot_encoded\n    return y, label_encoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y, le = prepare_labels(train['diagnosis'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GlassDataset(Dataset):\n    def __init__(self, df, datatype='train', transform = transforms.Compose([transforms.CenterCrop(32),transforms.ToTensor()]), y = None):\n        self.df = df\n        self.datatype = datatype\n        self.image_files_list = [f'../input/aptos2019-blindness-detection/{self.datatype}_images/{i}.png' for i in df['id_code'].values]\n        if self.datatype == 'train':\n            self.labels = y\n        else:\n            self.labels = np.zeros((df.shape[0], 5))\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_files_list)\n\n    def __getitem__(self, idx):\n        img_name = self.image_files_list[idx]\n        img = cv2.imread(img_name)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        image = self.transform(image=img)\n        image = image['image']\n\n        img_name_short = self.image_files_list[idx].split('.')[0]\n\n        label = self.labels[idx]\n        if self.datatype == 'test':\n            return image, label, img_name\n        else:\n            return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_transforms = albumentations.Compose([\n    albumentations.Resize(224, 224),\n    albumentations.RandomRotate90(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.Flip(p=0.5),\n    albumentations.OneOf([\n        albumentations.CLAHE(clip_limit=2), albumentations.RandomBrightness(), albumentations.RandomContrast(),\n        albumentations.JpegCompression(), albumentations.Blur(), albumentations.GaussNoise()], p=0.5), \n    albumentations.HueSaturationValue(p=0.5), \n    albumentations.ShiftScaleRotate(shift_limit=0.15, scale_limit=0.15, rotate_limit=45, p=0.5),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\n\ndata_transforms_test = albumentations.Compose([\n    albumentations.Resize(224, 224),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\n\ndataset = GlassDataset(df=train, datatype='train', transform=data_transforms, y=y)\ntest_set = GlassDataset(df=test, datatype='test', transform=data_transforms_test)\ntr, val = train_test_split(train.diagnosis, stratify=train.diagnosis, test_size=0.1)\ntrain_sampler = SubsetRandomSampler(list(tr.index))\nvalid_sampler = SubsetRandomSampler(list(val.index))\nbatch_size = 24\ntest_batch_size = 32\nnum_workers = 0\n# prepare data loaders (combine dataset and sampler)\ntrain_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)\nvalid_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers)\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=test_batch_size, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.models.resnet import ResNet, Bottleneck\n\ndef _resnext(arch, block, layers, pretrained, **kwargs):\n    model = ResNet(block, layers, **kwargs)\n    return model\n\ndef resnext101_32x8d(**kwargs):\n    kwargs['groups'] = 32\n    kwargs['width_per_group'] = 8\n    return _resnext('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], True, **kwargs)\n\nmodel_conv = resnext101_32x8d()\nmodel_conv.load_state_dict(torch.load('../input/resnext101wsl/ig_resnext101_32x8-c38310e5.pth'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_ftrs = model_conv.fc.in_features\nmodel_conv.fc = nn.Linear(num_ftrs, 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, gamma=2):\n        super().__init__()\n        self.gamma = gamma\n\n    def forward(self, logit, target):\n        target = target.float()\n        max_val = (-logit).clamp(min=0)\n        loss = logit - logit * target + max_val + \\\n               ((-max_val).exp() + (-logit - max_val).exp()).log()\n\n        invprobs = F.logsigmoid(-logit * (target * 2.0 - 1.0))\n        loss = (invprobs * self.gamma).exp() * loss\n        if len(loss.size())==2:\n            loss = loss.sum(dim=1)\n        return loss.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_conv.cuda()\ncriterion = nn.BCEWithLogitsLoss()\n\n#optimizer = optim.SGD(model_conv.fc.parameters(), lr=0.005, momentum=0.99)\n#scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, patience=2, )\n\nplist = [\n         {'params': model_conv.layer4.parameters(), 'lr': 1e-4, 'weight': 0.001},\n         {'params': model_conv.fc.parameters(), 'lr': 1e-3}\n         ]\n\noptimizer = optim.Adam(model_conv.fc.parameters(), lr=0.002)\nscheduler = lr_scheduler.StepLR(optimizer, step_size = 10, gamma = 0.5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model training"},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_loss_min = np.Inf\npatience = 6\n# current number of epochs, where validation loss didn't increase\np = 0\n# whether training should be stopped\nstop = False\n\n# number of epochs to train the model\nn_epochs = 64\nfor epoch in range(1, n_epochs+1):\n    print(time.ctime(), 'Epoch:', epoch)\n    model_conv.train()\n    \n    if epoch==4:\n        optimizer = optim.Adam(plist, lr=0.001)\n        scheduler = lr_scheduler.StepLR(optimizer, step_size = 2, gamma = 0.5)\n    if epoch==12:\n        optimizer = optim.SGD(model_conv.parameters(), lr=1e-6)\n        scheduler = lr_scheduler.StepLR(optimizer, step_size = 2, gamma = 0.5)\n\n    train_loss = []\n    train_auc = []\n    for batch_i, (data, target) in enumerate(train_loader):\n\n        data, target = data.cuda(), target.cuda()\n\n        optimizer.zero_grad()\n        output = model_conv(data)\n        loss = criterion(output, target.float())\n        train_loss.append(loss.item())\n        \n        a = target.data.cpu().numpy()\n        b = output[:,-1].detach().cpu().numpy()\n        # train_auc.append(roc_auc_score(a, b))\n        loss.backward()\n        optimizer.step()\n    \n    model_conv.eval()\n    val_loss = []\n    val_auc = []\n    for batch_i, (data, target) in enumerate(valid_loader):\n        data, target = data.cuda(), target.cuda()\n        output = model_conv(data)\n\n        loss = criterion(output, target.float())\n\n        val_loss.append(loss.item()) \n        a = target.data.cpu().numpy()\n        b = output[:,-1].detach().cpu().numpy()\n        # val_auc.append(roc_auc_score(a, b))\n\n    # print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}, valid loss: {np.mean(val_loss):.4f}, train auc: {np.mean(train_auc):.4f}, valid auc: {np.mean(val_auc):.4f}')\n    print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}, valid loss: {np.mean(val_loss):.4f}.')\n    \n    valid_loss = np.mean(val_loss)\n    scheduler.step(valid_loss)\n    if valid_loss <= valid_loss_min:\n        print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n        valid_loss_min,\n        valid_loss))\n        torch.save(model_conv.state_dict(), 'model.pt')\n        valid_loss_min = valid_loss\n        p = 0\n\n    # check if validation loss didn't improve\n    if valid_loss > valid_loss_min:\n        p += 1\n        print(f'{p} epochs of increasing val loss')\n        if p > patience:\n            print('Stopping training')\n            stop = True\n            break        \n            \n    if stop:\n        break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predicting"},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\n\nmodel_conv.load_state_dict(torch.load('./model.pt'))\nmodel_conv.eval()\n\nfor (data, target, name) in test_loader:\n    data = data.cuda()\n    output = model_conv(data)\n    output = output.cpu().detach().numpy()\n    for i, (e, n) in enumerate(list(zip(output, name))):\n        sub.loc[sub['id_code'] == n.split('/')[-1].split('.')[0], 'diagnosis'] = le.inverse_transform([np.argmax(e)])\n        \nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['diagnosis'].value_counts()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}