{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"## My first kernel on Kaggle :)\n## Credits \n## https://www.kaggle.com/abhishek/very-simple-pytorch-training-0-59\nimport pandas as pd\nimport time\nimport torchvision\nimport torch.nn as nn\nfrom tqdm import tqdm_notebook as tqdm\nfrom PIL import Image, ImageFile\nfrom torch.utils.data import Dataset\nimport torch\nimport torch.optim as optim\nfrom torchvision import transforms\nimport torch.optim.lr_scheduler as lr_scheduler\ndevice = torch.device(\"cuda:0\")\nImageFile.LOAD_TRUNCATED_IMAGES = True\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nprint(os.listdir(\"../input\"))\n#!pip install pretrainedmodels\n#import  pretrainedmodels\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Objective - Classify into any of these classes\n1. 0 - No DR\n2. 1 - Mild\n3. 2 - Moderate\n4. 3 - Severe\n5. 4 - Proliferative DR"},{"metadata":{"trusted":true},"cell_type":"code","source":"## read the files\ntrain = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nsub = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## split train data into train and validation data \nX_train, X_val, y_train, y_val = train_test_split(train['id_code'], train['diagnosis'], test_size=0.33, random_state=42)\ndf = pd.DataFrame({'id_code': X_train,'diagnosis': y_train})\ndf.to_csv( 'train.csv' , index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame({'id_code': X_val,'diagnosis': y_val})\ndf.to_csv( 'val.csv' , index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('train.csv')\nval = pd.read_csv('val.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train class "},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"\nfrom PIL import Image\nfrom torch.utils.data import Dataset\n\nclass AptosDataset(Dataset):\n    def __init__(self, \n                 csv_file, \n                 root_dir, \n                 transform=None):\n        self.data = pd.read_csv(csv_file)\n        self.root_dir = root_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.root_dir, \n                                self.data.loc[idx, 'id_code'] + '.png')\n        image = Image.open(img_name)\n        #image = image.resize((256, 256), resample=Image.BILINEAR)\n        label_tensor = torch.tensor(self.data.loc[idx, 'diagnosis'])\n\n        if self.transform:\n            image = self.transform(image)\n\n        return {'image': image,\n                'labels': label_tensor\n                }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setting up the model "},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torchvision.models as models\n# https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py\nmodel = models.resnet101(pretrained=False)\nmodel.load_state_dict(torch.load(\"../input/resnet101/resnet101-5d3b4d8f.pth\"))\nnum_features = model.fc.in_features\nprint(num_features)\nmodel.fc = nn.Linear(2048, 1)\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, data_loader, dataset_size, optimizer, scheduler, num_epochs):  \n    since = time.time()\n    criterion =  nn.MSELoss()\n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n        scheduler.step()\n        model.train()\n        running_loss = 0.0\n        tk0 = tqdm(data_loader, total=int(len(data_loader)))\n        counter = 0\n        for bi, d in enumerate(tk0):\n            inputs = d[\"image\"]\n            labels = d[\"labels\"].view(-1, 1)\n            inputs = inputs.to(device, dtype=torch.float)\n            labels = labels.to(device, dtype=torch.float)\n            optimizer.zero_grad()\n            with torch.set_grad_enabled(True):\n                outputs = model(inputs)\n                loss = criterion(outputs, labels)\n                loss.backward()\n                optimizer.step()\n            running_loss += loss.item() * inputs.size(0)\n            counter += 1\n            tk0.set_postfix(loss=(running_loss / (counter * data_loader.batch_size)))\n        epoch_loss = running_loss / len(data_loader)\n        print('Training Loss: {:.4f}'.format(epoch_loss))\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n    return model ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define the optimizer and scheduler."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = transforms.Compose([\n        # Data augmentation is a good practice for the train set\n        # Here, we randomly crop the image to 224x224 and\n        # randomly flip it horizontally. \n        transforms.Resize((224,224)),\n        transforms.RandomHorizontalFlip(),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                                 [0.229, 0.224, 0.225])\n    ])\n\n\ntrain_dataset = AptosDataset(csv_file='train.csv' , \n                             root_dir='../input/aptos2019-blindness-detection/train_images',\n                             transform=train_transform)\ntrain_dataset_loader = torch.utils.data.DataLoader(train_dataset, batch_size=16, shuffle=True, num_workers=4)\nlr_min = 1e-4\nlr_max = 1e-3\n\nplist = [\n         {'params': model.layer4.parameters(), 'lr': 1e-4, 'weight': 0.001},\n         {'params': model.fc.parameters(), 'lr': 1e-3}\n     ]\n\noptimizer_ft = optim.Adam(plist, lr=0.001)\nscheduler = lr_scheduler.StepLR(optimizer_ft, step_size=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"since = time.time()\nmodel=train_model(model,train_dataset_loader,len(train_dataset),optimizer_ft,scheduler,num_epochs=10)\n#model=train_model_patience(model,train_dataset_loader,len(train_dataset),optimizer_ft,scheduler,num_epochs=1)\ntime_elapsed = time.time() - since\nprint('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### persist the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(model.state_dict(), \"aptos_model.bin\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\".\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### https://www.learnopencv.com/pytorch-for-beginners-image-classification-using-pre-trained-models/\n### Picking transforms from above"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torchvision import transforms\n\n# define some re-usable stuff\nIMAGE_SIZE = 224\nNUM_CLASSES = 5\nTEST_BATCH_SIZE = 1\ndevice = torch.device(\"cuda:0\")\n\n\n# make some augmentations on training data\ntest_transform = transforms.Compose([\n    transforms.Resize((IMAGE_SIZE,IMAGE_SIZE)),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406],\n                                 [0.229, 0.224, 0.225])\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading pre-trained model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import pretrainedmodels\n# import sys\n# package_dir = \"../input/resnet101/\"\n# sys.path.insert(0, package_dir)\n# model_pt = pretrainedmodels.__dict__['resnet101'](pretrained=None)\n# model_pt.avg_pool = nn.AdaptiveAvgPool2d(1)\n# model_pt.last_linear = nn.Sequential(\n#                       nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n#                       nn.Dropout(p=0.25),\n#                       nn.Linear(in_features=2048, out_features=2048, bias=True),\n#                       nn.ReLU(),\n#                       nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n#                       nn.Dropout(p=0.5),\n#                       nn.Linear(in_features=2048, out_features=1, bias=True),\n#                      )\n# # setting strict=False to get around the weight loading problem \n# # model_pt.load_state_dict(torch.load(\"../input/20epoch/aptos_model.bin\"), strict=False)\n# model_pt.load_state_dict(torch.load(\"aptos_model.bin\") , strict=False)\n# # model_pt = model_pt.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for param in model_pt.parameters():\n#     param.requires_grad = False\nmodel_pt=model \nmodel_pt.eval()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Setting test data set loader"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AptosTestDataset(Dataset):\n\n    def __init__(self, \n                 csv_file, \n                 root_dir, \n                 transform=None):\n        self.data = pd.read_csv(csv_file)\n        self.root_dir = root_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.root_dir, \n                                self.data.loc[idx, 'id_code'] + '.png')\n        image = Image.open(img_name)\n        if self.transform:\n            image = self.transform(image)\n\n        return {'image': image}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Try to predict for val "},{"metadata":{"trusted":true},"cell_type":"code","source":"val_dataset = AptosTestDataset(csv_file='val.csv',\n                                      transform=test_transform, root_dir='../input/aptos2019-blindness-detection/train_images')\n\nval_data_loader = torch.utils.data.DataLoader(val_dataset, batch_size=32, shuffle=False, num_workers=4)\nval_preds = np.zeros((len(val_dataset), 1))\ntk0 = tqdm(val_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model_pt(x_batch.to(device))\n    val_preds[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val = pd.read_csv('val.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport scipy as sp\nfrom functools import partial\nfrom sklearn import metrics\nfrom collections import Counter\nimport json\n\nclass OptimizedRounder(object):\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n\n        ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optR = OptimizedRounder()\noptR.fit(val_preds.astype(int), val['diagnosis'])\nprint(optR.coefficients)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"coefficients = optR.coefficients()\nvalid_predictions = optR.predict(val_preds.astype(int), coefficients)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.unique(valid_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn\nacc = sklearn.metrics.accuracy_score(val['diagnosis'], valid_predictions)\nprint(' accuracy on validation set : {}'.format(acc)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# i was never able to get this working correctly for classification \n# kagglers plz let me know where i am botching this up\n# sub = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\n# test_preds = np.zeros((len(test_dataset), NUM_CLASSES))\n# tk0 = tqdm(test_data_loader , total=int(len(test_data_loader)))\n# for i, x_batch in enumerate(tk0):\n#     x_batch = x_batch[\"image\"]\n#     pred = model_pt(x_batch.to(device))\n#     test_preds[i * TEST_BATCH_SIZE:(i + 1) * TEST_BATCH_SIZE, :] = pred.detach().cpu().squeeze().numpy()\n    \n# test_preds = torch.from_numpy(test_preds).float().to(device).sigmoid()\n# test_preds = test_preds.detach().cpu().squeeze().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = AptosTestDataset(csv_file='../input/aptos2019-blindness-detection/test.csv',\n                                      transform=test_transform, root_dir='../input/aptos2019-blindness-detection/test_images')\ntest_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds1 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model_pt(x_batch.to(device))\n    test_preds1[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds2 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model_pt(x_batch.to(device))\n    test_preds2[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds3 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model_pt(x_batch.to(device))\n    test_preds3[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds4 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model_pt(x_batch.to(device))\n    test_preds4[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds5 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model_pt(x_batch.to(device))\n    test_preds5[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds6 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model_pt(x_batch.to(device))\n    test_preds6[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds7 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model_pt(x_batch.to(device))\n    test_preds7[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds8 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model_pt(x_batch.to(device))\n    test_preds8[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds9 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model_pt(x_batch.to(device))\n    test_preds9[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds10 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model_pt(x_batch.to(device))\n    test_preds10[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = (test_preds1 + test_preds2 + test_preds3 + test_preds4 + test_preds5+test_preds6 + test_preds7 + test_preds8 + test_preds9 + test_preds10)/10.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.unique(test_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predictions = optR.predict(test_preds.astype(int), coefficients)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = pd.read_csv(\"../input/aptos2019-blindness-detection/sample_submission.csv\")\nsample.diagnosis = test_predictions\nsample.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample=pd.read_csv(\"submission.csv\").head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.unique(sample['diagnosis'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}