{"cells":[{"metadata":{},"cell_type":"markdown","source":"Simple example on training an inception v3 model starting from a pretrained model.\n\nTraining time on GPU approx. 6 min per epoch => 2 hours for 20 epochs"},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport torch\nimport torchvision\nimport numpy as np\nfrom PIL import Image\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import LabelBinarizer\nfrom sklearn.model_selection import ShuffleSplit\nimport torchvision.models.inception\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# hyperparameter\n\n# training\nnum_epochs = 20\nbatch_size = 32\nnum_workers = 6\nlr = 0.001\n\n# data sources\nsample_submission = '../input/aptos2019-blindness-detection/sample_submission.csv'\nroot = '../input/aptos2019-blindness-detection/test_images/'\ntraining_file = '../input/aptos2019-blindness-detection/train.csv'\ntrainroot = '../input/aptos2019-blindness-detection/train_images/'\npretrained = '../input/torchvision-inception-v3-imagenet-pretrained/inception_v3_google-1a9a5a14.pth'\ntest_size = 0.2\n\n# data preprocessing from imagenet\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\n\n# device checker, use GPU if available\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(\"Using device\", device)\n\n# fixing random seed (for reproducibility)\nseed = 555\ntorch.manual_seed(seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\nnp.random.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading the pretrained inception v3\nnet = torchvision.models.inception_v3()\nckpt = torch.load(pretrained, map_location='cpu')\nnet.load_state_dict(ckpt)\n\n# as we only have 5 output classes (and want to use pretrained models)\n# we need to replace the final layers by new layers which have only 5\n# ourput channels. Inception v3 uses AuxLogits, a learning helper, during\n# training, so we need to adjust this layer, too.\nnet.fc = torch.nn.Linear(in_features=2048, out_features=5)\nnet.AuxLogits = torchvision.models.inception.InceptionAux(in_channels=768, num_classes=5)\n_ = net.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Adam and Binary Cross Entropy are pretty standard for multi-class classification\noptim = torch.optim.Adam(lr=lr, params=net.parameters())\ncrit = torch.nn.BCEWithLogitsLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# simple preprocessing with resizing and cropping to 299x299\n# followed by normalization (formally correct actually standardization)\n# given mean and std above\ntransform = torchvision.transforms.Compose([\n    torchvision.transforms.Resize(299),\n    torchvision.transforms.CenterCrop(299),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(mean=mean, std=std)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# simple dataset class which takes the csv filename, the root dir of the images, and the\n# transformation above and returns the transformed image and binarized label as tensors\nclass SimpleDataset():\n    def __init__(self, data, root, transform):\n        self.files = list(root + data['id_code'] + '.png')\n        \n        # LabelBinarizer takes numerical labels and returns a one-hot label\n        binarizer = LabelBinarizer()\n        self.targets = binarizer.fit_transform(data['diagnosis'].values)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.files)\n\n    def __getitem__(self, idx):\n        img = Image.open(self.files[idx])\n        x = self.transform(img)\n        y = torch.tensor(self.targets[idx,:]).float()\n        return x, y","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"data = pd.read_csv(training_file)\nssplit = ShuffleSplit(n_splits=1, test_size=test_size, random_state=seed)\n\ntrain_index, test_index = next(ssplit.split(data['id_code']))\n\ndataset = SimpleDataset(data.iloc[train_index], trainroot, transform)\nvalidationset = SimpleDataset(data.iloc[test_index], trainroot, transform)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"train_loss = []\nvalidation_loss = []\nfor ep in tqdm(range(num_epochs), position=0):\n    \n    # Training\n    net.train()\n    loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n    total_loss = 0\n    for x, y in loader:\n        x = x.to(device)\n        y = y.to(device)\n        pred = net(x)\n        # as we use auxLogits (default True for inception) we get 2 outputs and need to calculat\n        # the loss of both outputs\n        loss = crit(pred[0], y) + crit(pred[1], y)\n        total_loss += loss\n        optim.zero_grad()\n        loss.backward()\n        optim.step()\n\n    total_loss /= len(dataset) # average loss per image\n    total_loss /= 2 # adjustment for summing aux loss and normal loss\n    train_loss.append(total_loss)\n    \n    # Validation\n    net.eval()\n    loader = torch.utils.data.DataLoader(validationset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n    total_loss = 0\n    with torch.no_grad():\n        for x, y in loader:\n            x = x.to(device)\n            y = y.to(device)\n            pred = net(x)\n            loss = crit(pred, y)\n            total_loss += loss\n        \n        # this gives us the average loss per image\n        total_loss /= len(validationset)\n        validation_loss.append(total_loss)\n        tqdm.write('Loss after epoch {:d}: train {:.4f}, test {:.4f}'.format(ep, train_loss[-1], validation_loss[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(train_loss, label='train loss')\nplt.plot(validation_loss, label='validation loss')\nplt.xlabel('epochs')\nplt.ylabel('loss/image [logits]')\nplt.title('Training and validation loss')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Evaluation\nsubmit = pd.read_csv(sample_submission)\nnet.eval()\n\nwith torch.no_grad():\n    for name in tqdm(submit['id_code']):\n        img = Image.open(root+name+'.png')\n        x = transform(img).to(device).unsqueeze(0)\n        y = net(x).cpu().numpy()\n        diag = int(np.argmax(y[:5]))\n        submit.loc[submit['id_code']==name, 'diagnosis'] = diag","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}