{"cells":[{"metadata":{},"cell_type":"markdown","source":"You can go a long way with just a single Resnet and no image pre-processing."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport cv2\nimport time\nimport torchvision\nimport torch.nn as nn\nfrom tqdm import tqdm_notebook as tqdm\nimport scipy as sp\n\nfrom sklearn import metrics\nfrom functools import partial\n\nfrom PIL import Image, ImageFile\nfrom torch.utils.data import Dataset\nimport torch\nimport torch.optim as optim\nfrom torchvision import transforms\nfrom torch.optim import lr_scheduler\nimport os\nimport numpy as np\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom scipy import stats\nimport random\n\nimport albumentations\nfrom albumentations import torch as AT","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\")\nImageFile.LOAD_TRUNCATED_IMAGES = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RetinopathyDataset_v2(Dataset):\n\n    def __init__(self, df, transform, train=True):\n\n        self.data = df.reset_index()\n        \n        if train:\n            self.prefix = \"train\"\n        else:\n            self.prefix = \"test\"\n            \n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n    \n\n    def __getitem__(self, idx):\n        img_name = os.path.join('../input/aptos2019-blindness-detection/{}_images'.format(self.prefix),\n                                self.data.loc[idx, 'id_code'] + '.png')\n        \n        img = cv2.imread(img_name)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        image = self.transform(image=img)\n        image = image['image']\n        \n        label = torch.tensor(self.data.loc[idx, 'diagnosis'])\n        \n        return {'image': image,\n                'labels': label\n                }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RetinopathyDataset(Dataset):\n\n    def __init__(self, csv_file, transform, train=True):\n\n        self.data = pd.read_csv(csv_file)\n        \n        if train:\n            self.prefix = \"train\"\n        else:\n            self.prefix = \"test\"\n            \n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join('../input/aptos2019-blindness-detection/{}_images'.format(self.prefix), self.data.loc[idx, 'id_code'] + '.png')\n       \n        img = cv2.imread(img_name)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        image = self.transform(image=img)\n        image = image['image']\n        \n        label = torch.tensor(self.data.loc[idx, 'diagnosis'])\n        return {'image': image,\n                'labels': label\n                }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class OptimizedRounder(object):\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n\n        ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torchvision.models.resnet101(pretrained=False)\nmodel.load_state_dict(torch.load(\"../input/fastai-pretrained-models/resnet101-5d3b4d8f.pth\"))\n\nfor param in model.parameters():\n    param.requires_grad = False\n\nmodel.fc =  nn.Sequential(\n                          nn.Linear(in_features=2048, out_features=1, bias=True),\n                         )\n\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = albumentations.Compose([\n    albumentations.Resize(224, 224),\n    albumentations.HorizontalFlip(),\n    albumentations.VerticalFlip(),\n    albumentations.RandomBrightness(),\n    albumentations.ShiftScaleRotate(rotate_limit=20, scale_limit=0.10),\n    albumentations.Normalize(),\n    AT.ToTensor()\n])\n\ntransforms_validation = albumentations.Compose([\n    albumentations.Resize(224, 224),\n    albumentations.Normalize(),\n    AT.ToTensor()\n])\n\n\ntransforms_test = albumentations.Compose([\n    albumentations.Resize(224, 224),\n    albumentations.Rotate(limit=10),\n    albumentations.Normalize(),\n    AT.ToTensor()\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\n\nskf = StratifiedKFold(n_splits=10, random_state=2052)\n\nfor i, (train_index, val_index) in enumerate(skf.split(train_df['diagnosis'], train_df['diagnosis'])):\n    # we want to have a 90/10 split, so we use a Stratified 10-fold split and just take the first split\n    if i == 0:\n        validation_df = train_df.iloc[val_index]\n        train_df = train_df.iloc[train_index]\n\ntrain_dataset = RetinopathyDataset_v2(df=train_df, train=True, transform=transforms_train)\nvalidation_dataset = RetinopathyDataset_v2(df=validation_df, train=True, transform=transforms_validation)\ntest_dataset = RetinopathyDataset(csv_file='../input/aptos2019-blindness-detection/sample_submission.csv',\n                                  transform=transforms_test, train=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# we have a class imbalance, so we will weight the classes differently, this computes a weighted MSE loss\n\ndef weighted_mse_loss(preds, target, weight):\n    return torch.sum(weight * (preds - target) ** 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=4)\ndata_loader_val = torch.utils.data.DataLoader(validation_dataset, batch_size=32, shuffle=True, num_workers=4)\n\nnum_epochs = 25\n\noptimizer = optim.Adam(model.parameters(), lr=0.0002)\n\n# going to be doing MixUp, this controls the beta distribution of the mixing ratio\nalpha_mixup = 0.3\n\n# train top FC layer for n_freeze epochs, then un-freeze all other layers and train the full Resnet\nn_freeze = 2\n\n# we want class weights to be inversely proportional to the frequency of the classes\n# weight(class) ~ (proportion of class in dataset)^(-weight_pow)\nweight_pow = 0.3\nlabel_counts = train_df['diagnosis'].value_counts().values\nlabel_weights = (label_counts[:] / label_counts.sum())**(-weight_pow)\n\n# normalize so that 1st class has a weight of 1\nlabel_weights /= label_weights[0]\nprint(label_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"since = time.time()\n\ncriterion = weighted_mse_loss\n\nbest = 0\nnep = 0\nbest_coeff = []\n\nmonth_day = time.gmtime()[1:3]\n\nfor epoch in range(num_epochs):\n    print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n    print('-' * 10)\n    model.train()\n    running_loss = 0.0\n    tk0 = tqdm(data_loader, total=int(len(data_loader)))\n    counter = 0\n    \n    if epoch == n_freeze:                \n        for param in model.parameters():\n            param.requires_grad = True \n    \n    for bi, d in enumerate(tk0):\n        inputs = d[\"image\"]\n        labels = d[\"labels\"].view(-1, 1)\n        weights = labels[:, 0].clone()\n                \n        for i in range(len(label_weights)):\n            weights[labels[:, 0] == i] = label_weights[i]\n\n        # we won't load a different batch for the mixup, just shuffle the current one\n        # and mix it with the unshuffled version\n        shuffled_index = list(range(inputs.shape[0]))\n        random.shuffle(shuffled_index)\n\n        mixed_up_inputs = inputs[shuffled_index, :, :, :]\n        mixed_up_labels = labels[shuffled_index, :]\n\n        mixed_up_weights = mixed_up_labels[:, 0].clone()\n\n        for i in range(len(label_weights)):\n            mixed_up_weights[mixed_up_labels[:, 0] == i] = label_weights[i]\n\n        l_mixup = stats.beta.rvs(a=alpha_mixup, b=alpha_mixup, size=inputs.shape[0])\n\n        inputs = torch.as_tensor(l_mixup, dtype=torch.float).view(-1, 1, 1, 1) * inputs\n        inputs += torch.as_tensor(1. - l_mixup, dtype=torch.float).view(-1, 1, 1, 1) * mixed_up_inputs\n\n        labels = labels.float()\n        mixed_up_labels = mixed_up_labels.float()\n\n        labels = torch.as_tensor(l_mixup, dtype=torch.float).view(-1, 1) * labels\n        labels += torch.as_tensor(1. - l_mixup, dtype=torch.float).view(-1, 1) * mixed_up_labels\n\n        weights = weights.float()\n        mixed_up_weights = mixed_up_weights.float()\n        weights = torch.as_tensor(l_mixup, dtype=torch.float).view(-1, 1) * weights\n        weights += torch.as_tensor(1. - l_mixup, dtype=torch.float).view(-1, 1) * mixed_up_weights\n\n        inputs = inputs.to(device, dtype=torch.float)\n        labels = labels.to(device, dtype=torch.float)\n        weights = weights.to(device, dtype=torch.float)\n        optimizer.zero_grad()\n        \n        with torch.set_grad_enabled(True):\n            outputs = model(inputs)\n                \n            loss = criterion(outputs, labels, weights)\n            loss.backward()\n            optimizer.step()\n            \n        running_loss += loss.item() * inputs.size(0)\n        counter += 1\n        tk0.set_postfix(loss=(running_loss / (counter * data_loader.batch_size)))\n    \n    epoch_loss = running_loss / len(data_loader)\n    print('Training Loss: {:.4f}'.format(epoch_loss))\n    \n    model.eval()\n    counter = 0\n    val_loss = 0.0\n    \n    preds = []\n    full_labels = []\n    \n    for d in data_loader_val:\n        inputs = d[\"image\"]\n        labels = d[\"labels\"].view(-1, 1)\n        inputs = inputs.to(device, dtype=torch.float)\n        with torch.set_grad_enabled(False):\n            outputs = model(inputs)\n            preds.append(outputs.cpu().numpy())\n            full_labels.append(labels.numpy())\n            \n    preds = np.vstack(preds)[:, 0]\n    full_labels = np.vstack(full_labels)[:, 0]\n    \n    optR = OptimizedRounder()\n    optR.fit(preds, full_labels)\n    coefficients = optR.coefficients()\n    valid_predictions = optR.predict(preds, coefficients)\n    \n    val_loss = metrics.cohen_kappa_score(valid_predictions, full_labels, weights='quadratic')\n    print('Validation Loss: {:.4f}'.format(val_loss))\n    if val_loss > best:\n        best = val_loss\n        nep = epoch\n        best_coeff = coefficients[:]\n        print('Current best', best, nep, best_coeff)\n        torch.save(model.state_dict(), \"model_best.pth\")\n\n    \ntime_elapsed = time.time() - since\nprint('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sometimes the last threshold would very close to the next-to-last threshold, so I manually shift it a bit\nif best_coeff[-1] < 3.0:\n    best_coeff[-1] = 3.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del inputs, labels, weights, mixed_up_inputs, mixed_up_labels, mixed_up_weights, outputs\ndel train_dataset, train_df, validation_dataset, validation_df\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_state_dict(torch.load(\"model_best.pth\"))\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# number of TTA runs we'll be doing\nn_tta = 10\n\n# sometimes got submission errors when running with larger batch size, so probably OOM issue\nbsize = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=bsize, shuffle=False, num_workers=4)\ntest_preds = np.zeros((len(test_dataset), 1))\n\nfor j in range(n_tta):\n    tk0 = tqdm(test_data_loader)\n    for i, x_batch in enumerate(tk0):\n        x_batch = x_batch[\"image\"]\n        pred = model(x_batch.to(device))\n        test_preds[i * bsize:(i + 1) * bsize] += pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\n        \ntest_preds /= n_tta","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, pred in enumerate(test_preds):\n    if pred < best_coeff[0]:\n        test_preds[i] = 0\n    elif pred >= best_coeff[0] and pred < best_coeff[1]:\n        test_preds[i] = 1\n    elif pred >= best_coeff[1] and pred < best_coeff[2]:\n        test_preds[i] = 2\n    elif pred >= best_coeff[2] and pred < best_coeff[3]:\n        test_preds[i] = 3\n    else:\n        test_preds[i] = 4\n        \nsample = pd.read_csv(\"../input/aptos2019-blindness-detection/sample_submission.csv\")\nsample.diagnosis = test_preds.astype(int)\nsample.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}