{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        break\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Trained efficientnet b2 on fully connected layer for 2 epochs with 2019 data, then trained rest of layers for 5 epochs"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\npackage_path = '../input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/'\nsys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torch.nn.functional as F\nimport numpy as np\nimport torchvision\nfrom torchvision import datasets, models, transforms\nimport matplotlib.pyplot as plt\nimport time\nimport os\nimport copy\nfrom sklearn.metrics import confusion_matrix\nfrom torch.utils.data import random_split, Dataset\nfrom torch.utils.data.sampler import SubsetRandomSampler\nimport os\nfrom PIL import Image\nimport glob\nimport cv2\nimport gc #garbage collector for gpu memory \nfrom tqdm import tqdm\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom tqdm import tqdm\n\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transform = transforms.Compose(\n        [transforms.Resize((320,320)),\n        #transforms.RandomCrop(),\n        #transforms.RandomHorizontalFlip(.33),\n        #transforms.RandomVerticalFlip(.33),\n        transforms.RandomApply([\n        #transforms.ColorJitter(brightness=(1,3), contrast=(1,3), saturation=(1,2)),\n        transforms.RandomAffine(degrees = (-360,360), shear = (-45,45))],\n        p=.5),\n        transforms.ToTensor(),\n        transforms.Normalize([0.460, 0.247, 0.080], [0.249, 0.138, 0.081])\n        ])\n\n\ntrain_transform = transforms.Compose(\n        [transforms.Resize((320,320)),\n        #transforms.RandomCrop(),\n        #transforms.RandomHorizontalFlip(.33),\n        #transforms.RandomVerticalFlip(.33),\n        transforms.RandomApply([\n        #transforms.ColorJitter(brightness=(1,3), contrast=(1,3), saturation=(1,2)),\n        transforms.RandomAffine(degrees = (-360,360), shear = (-45,45))],\n        p=.5),\n        transforms.ToTensor(),\n        transforms.Normalize([0.460, 0.247, 0.080], [0.249, 0.138, 0.081])\n        ])\nvalid_transform = transforms.Compose(\n        [transforms.Resize((320,320)),\n        #transforms.RandomCrop(),\n        #transforms.RandomHorizontalFlip(.33),\n        #transforms.RandomVerticalFlip(.33),\n        #transforms.ColorJitter(brightness=(1,3), contrast=(1,3), saturation=(1,2)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.460, 0.247, 0.080], [0.249, 0.138, 0.081])\n        ])     \n\ntest_transform = transforms.Compose(\n        [transforms.Resize((320,320)),\n        #transforms.RandomCrop(),\n        #transforms.RandomHorizontalFlip(.33),\n        #transforms.RandomVerticalFlip(.33),\n        #transforms.ColorJitter(brightness=(1,3), contrast=(1,3), saturation=(1,2)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.460, 0.247, 0.080], [0.249, 0.138, 0.081])\n        ])  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class APTOSDataset(Dataset):\n    \"\"\"Eye images dataset.\"\"\"\n    def __init__(self, csv_file, filetype, transform=None):\n        self.eye_frame = pd.read_csv(csv_file)\n        self.filetype = filetype\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.eye_frame)\n    \n    def __getitem__(self, idx):\n        if self.filetype == 'train':\n            img_name = os.path.join('../input/aptos2019-blindness-detection/train_images',\n                                    self.eye_frame.loc[idx,'id_code'] + '.png')\n\n            image = Image.open(img_name)\n            if self.transform:\n                image = self.transform(image)\n            else:\n                image = transforms.ToTensor()(image)\n\n            return image,self.eye_frame.diagnosis[idx]\n        \n        else:\n            img_name = os.path.join('../input/aptos2019-blindness-detection/test_images',\n                                    self.eye_frame.loc[idx,'id_code'] + '.png')\n            image = Image.open(img_name)\n            if self.transform:\n                image = self.transform(image)\n            else:\n                image = transforms.ToTensor()(image)\n            return image, self.eye_frame.loc[idx,'id_code']\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class KFoldDataset(Dataset):\n    \"\"\"Eye images dataset.\"\"\"\n    def __init__(self, csv_file, indices, filetype, transform=None):\n        self.eye_frame = pd.read_csv(csv_file).iloc[indices].reset_index(drop=True)\n        self.filetype = filetype\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.eye_frame)\n    \n    def __getitem__(self, idx):\n        if self.filetype == 'train':\n            img_name = os.path.join('../input/aptos2019-blindness-detection/train_images',\n                                    self.eye_frame.loc[idx,'id_code'] + '.png')\n\n            image = Image.open(img_name)\n            if self.transform:\n                image = self.transform(image)\n            else:\n                image = transforms.ToTensor()(image)\n\n            return image,self.eye_frame.diagnosis[idx]\n        \n        else:\n            img_name = os.path.join('../input/aptos2019-blindness-detection/test_images',\n                                    self.eye_frame.loc[idx,'id_code'] + '.png')\n            image = Image.open(img_name)\n            if self.transform:\n                image = self.transform(image)\n            else:\n                image = transforms.ToTensor()(image)\n            return image, self.eye_frame.loc[idx,'id_code']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\ntrain_dataset = APTOSDataset(csv_file='../input/aptos2019-blindness-detection/train.csv',filetype='train',transform=train_transform)\ntest_dataset = APTOSDataset(csv_file='../input/aptos2019-blindness-detection/test.csv', filetype='test',transform=test_transform)\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=16, shuffle=True, num_workers=4)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=16, shuffle=False, num_workers=4)\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\nfrom efficientnet_pytorch import EfficientNet\nmodel = EfficientNet.from_name('efficientnet-b2')\nmodel.load_state_dict(torch.load(\"/kaggle/input/efficientnet-pytorch/efficientnet-b2-27687264.pth\"))\n\nnum_ftrs = model._fc.in_features\n\nmodel._fc = nn.Linear(num_ftrs, 5)\n\n\n#model.load_state_dict(torch.load(\"/kaggle/input/efficientnet-pytorch/efficientnet-b2-27687264.pth\"))\nmodel = model.to(device)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nct = 0\nfor child in model.children():\n    ct += 1\n    if ct < 3:\n        for param in child.parameters():\n            param.requires_grad = False\n    else:\n        for param in child.parameters():\n            param.requires_grad = True\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def class_proportional_weights(train_labels):\n    '''\n    helper function to scale weights of classes in loss function based on their sampled proportions\n    # This custom loss function is defined to reduce the effect of class imbalance.\n    # Since there are so many samples labeled as \"O\", this allows the RNN to not \n    # be weighted too heavily in that area.\n    '''\n    weights = []\n    for lab in range(0,5):\n        weights.append(1-(train_labels.count(lab)/(len(train_labels)-train_labels.count(lab)))) #proportional to number without tags\n    return weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nweights = class_proportional_weights(train_pd.iloc[train_idx, 'diagnosis']) \nclass_weights = torch.FloatTensor(weights).cuda()\ncriterion = nn.CrossEntropyLoss(weights=class_weights)\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nfrom efficientnet_pytorch import EfficientNet\n\nBATCH_SIZE = 28\ntrain_pd = pd.read_csv(r\"/kaggle/input/aptos2019-blindness-detection/train.csv\")\n\ndef compute_accuracy(model, data_loader, device):\n    correct_pred, num_examples = 0, 0\n    tqdm()\n    all_labels = []\n    all_preds = []\n    for i, (inputs, labels) in enumerate(tqdm(data_loader)):\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n        \n        outputs = model(inputs)\n        _, preds = torch.max(outputs, 1)\n        num_examples += labels.size(0)\n        correct_pred += (preds == labels).sum()\n        all_labels.extend([label.item() for label in labels])\n        all_preds.extend([pred.item() for pred in preds])\n    return np.array(all_labels), np.array(all_preds), correct_pred.float()/num_examples * 100\n\n\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nNUM_EPOCHS = 5\nN_SPLITS = 2\nstart_time = time.time()\n\nsplits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=1234).split(train_pd['id_code'], train_pd['diagnosis']))\n\n\n# using a numpy array because it's faster than a list\ntest_pd = pd.read_csv(r\"/kaggle/input/aptos2019-blindness-detection/test.csv\")\n\npredictionsfinal = torch.zeros((len(test_pd),5), dtype=torch.float32)\ntest_dataset = APTOSDataset(csv_file='../input/aptos2019-blindness-detection/test.csv', filetype='test',transform=test_transform)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\n\n\n\nfor i, (train_idx, valid_idx) in enumerate(splits):\n    print(\"\\n\")\n    print(\"--- Fold Number: {} ---\".format(i+1))\n    train_dataset = KFoldDataset(csv_file='../input/aptos2019-blindness-detection/train.csv', indices=train_idx,filetype='train',transform=train_transform)\n    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)\n    valid_dataset = KFoldDataset(csv_file='../input/aptos2019-blindness-detection/train.csv', indices=valid_idx,filetype='train',transform=valid_transform)\n    val_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\n\n    \n\n    model = EfficientNet.from_name('efficientnet-b3')\n    model.load_state_dict(torch.load(\"/kaggle/input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth\"))\n\n    num_ftrs = model._fc.in_features\n\n    model._fc = nn.Linear(num_ftrs, 5)\n    #model.load_state_dict(torch.load(\"/kaggle/input/efficientnet-pytorch/efficientnet-b2-27687264.pth\"))\n    model = model.to(device)\n    \n\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=3e-4)\n    losses = []\n    valid_acc_list = []\n    best_acc = 0.0\n\n    \n    for epoch in range(NUM_EPOCHS):\n        print('Seconds elapsed: ', round((time.time() - start_time),2))\n        print('Running Epoch: ', epoch+1)\n        print('-' * 10)\n        running_loss = 0.0\n        model.train()\n        i = 0\n        for iteration, (inputs, labels) in enumerate(train_loader):\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            optimizer.zero_grad()\n        \n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            running_loss += float(loss.item())\n            del inputs, labels\n            i+=1\n            # printing the results every 20 iterations\n            if not iteration%20:\n                print('Epoch {:03d}/{:03d} | Batch: {:03d}/{:03d} |'\n                      ' Cost: {:.4f} | Avg Loss: {:.4f}'.format(epoch+1, NUM_EPOCHS, iteration+1, len(train_loader),loss, running_loss/i))\n                running_loss = 0.0\n                i = 0\n            del loss\n            gc.collect()\n            torch.cuda.empty_cache()\n          \n        with torch.set_grad_enabled(False):\n            model.eval()\n            train_labels, train_preds, train_accuracy = compute_accuracy(model, train_loader, device)\n            valid_labels, valid_preds, valid_accuracy = compute_accuracy(model, val_loader, device)\n            valid_acc_list.append(valid_accuracy)\n            print('Epoch: {:03d}/{:03d} Train Acc.: {:.2f} | Validation Acc.: {:.2f}'.format(epoch+1,NUM_EPOCHS, train_accuracy,\n                                                                                    valid_accuracy))\n            print('Confusion Matrix for Train and Validation')\n            train_cnf_matrix = confusion_matrix(train_labels, train_preds)\n            valid_cnf_matrix = confusion_matrix(valid_labels, valid_preds)\n            train_cnf_matrix_norm = train_cnf_matrix.astype('float') / train_cnf_matrix.sum(axis=1)[:, np.newaxis]\n            validation_cnf_matrix_norm = valid_cnf_matrix.astype('float') / valid_cnf_matrix.sum(axis=1)[:, np.newaxis]\n            graph_labels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\n            train_df_cm = pd.DataFrame(train_cnf_matrix_norm, index=graph_labels, columns=graph_labels)\n            validation_df_cm = pd.DataFrame(validation_cnf_matrix_norm, index=graph_labels, columns=graph_labels)\n            fig, (ax1, ax2) = plt.subplots(1,2, sharex='col', figsize=(10,4))\n            sns.heatmap(train_df_cm, annot=True, fmt='.2f', cmap=\"Blues\", ax=ax1).set_title('Train')\n            sns.heatmap(validation_df_cm, annot=True, fmt='.2f', cmap=sns.cubehelix_palette(8), ax=ax2).set_title('Validation')\n            plt.show()\n            print('-'*20)\n            print('\\n')\n\n    kfold_test_predictions = torch.zeros((len(test_pd)),5)\n    model.eval()\n    with torch.no_grad():\n        for i, (inputs, img_id) in enumerate(tqdm(test_loader)):\n            inputs = inputs.to(device)\n            outputs = model(inputs)\n            kfold_test_predictions[i*BATCH_SIZE:(i+1)*BATCH_SIZE] = outputs\n\n        predictionsfinal += (kfold_test_predictions/N_SPLITS)\n        \n_, preds = torch.max(predictionsfinal, 1)     \nfinal_predictions = pd.DataFrame(([test_pd['id_code'].tolist(), preds.tolist()])).transpose()\nfinal_predictions.columns = ['id_code', 'diagnosis']\nfinal_predictions_df = final_predictions.copy()\nfinal_predictions.to_csv(\"submission.csv\",index=False)\nprint(\"Training Complete --- {} seconds ---\".format(round((time.time() - start_time),2)))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nfrom tqdm._tqdm_notebook import tqdm_notebook\ndef compute_predictions(model, model_type, data_loader, device):\n    if model_type == 'train':\n        predictions = []\n        correct_pred, num_examples = 0, 0\n        tqdm_notebook()\n        for i, (inputs, labels) in enumerate(tqdm_notebook(data_loader)):\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n            predictions.append(preds)\n            num_examples += labels.size(0)\n            correct_pred += (preds==labels).sum()\n        return predictions, correct_pred.item()/num_examples*100\n    \n    else:\n        predictions = []\n        img_ids = []\n        tqdm_notebook()\n        for i, (inputs, img_id) in enumerate(tqdm_notebook(data_loader)):\n            inputs = inputs.to(device)\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n            predictions.extend(preds)\n            img_ids.extend(img_id)\n            \n        predictions = [pred.item() for pred in predictions]\n        final_predictions = pd.DataFrame(np.array([img_ids, predictions])).transpose()\n        final_predictions.columns = ['id_code', 'diagnosis']\n        final_predictions_df = final_predictions.copy()\n        final_predictions.to_csv(\"submission.csv\",index=False)\n        return final_predictions_df\n            \n\"\"\"            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nwith torch.set_grad_enabled(False):\n    model.eval()\n    #train_predictions, train_accuracy = compute_predictions(model, 'train', train_loader, device)\n    #print('Train Accuracy: ', train_accuracy)\n    print('Computing Test Predictions')\n    test_predictions = compute_predictions(model, 'test', test_loader, device)\n        \n\"\"\"    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\n\nBATCH_SIZE = 28\ntrain_pd = pd.read_csv(r\"/kaggle/input/aptos2019-blindness-detection/train.csv\")\n\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nNUM_EPOCHS = 5\nN_SPLITS = 5\nstart_time = time.time()\n\nsplits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=1234).split(train_pd['id_code'], train_pd['diagnosis']))\n\n\n# using a numpy array because it's faster than a list\ntest_pd = pd.read_csv(r\"/kaggle/input/aptos2019-blindness-detection/test.csv\")\n\npredictionsfinal = torch.zeros((len(test_pd),5), dtype=torch.float32)\ntest_dataset = APTOSDataset(csv_file='../input/aptos2019-blindness-detection/test.csv', filetype='test',transform=test_transform)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\n\n\n\nfor fold_num, (train_idx, valid_idx) in enumerate(splits):\n    print(\"\\n\")\n    print(\"--- Fold Number: {} ---\".format(fold_num+1))\n\n    model = EfficientNet.from_name('efficientnet-b2')\n    num_ftrs = model._fc.in_features\n    model._fc = nn.Linear(num_ftrs, 5)\n    model.load_state_dict(torch.load(\"/kaggle/input/pretrained-cv-weights/valid_fold_\"+str(fold_num+1)+\".pth\"))\n    model = model.to(device)\n\n\n    kfold_test_predictions = torch.zeros((len(test_pd)),5)\n    model.eval()\n    with torch.no_grad():\n        for i, (inputs, img_id) in enumerate(tqdm(test_loader)):\n            inputs = inputs.to(device)\n            outputs = model(inputs)\n            kfold_test_predictions[i*BATCH_SIZE:(i+1)*BATCH_SIZE] = outputs\n\n        predictionsfinal += (kfold_test_predictions/N_SPLITS)\n        \n_, preds = torch.max(predictionsfinal, 1)     \nfinal_predictions = pd.DataFrame(([test_pd['id_code'].tolist(), preds.tolist()])).transpose()\nfinal_predictions.columns = ['id_code', 'diagnosis']\nfinal_predictions_df = final_predictions.copy()\nfinal_predictions.to_csv(\"submission.csv\",index=False)\nprint(\"Training Complete --- {} seconds ---\".format(round((time.time() - start_time),2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}