{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Here's my adaptation to this problem of the [PyTorch transfer learning tutorial](https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html)\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport os\n\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport time\nimport copy\nfrom torchvision.datasets.folder import default_loader, DatasetFolder","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/aptos2019-blindness-detection/train.csv\")\ntest = pd.read_csv(\"../input/aptos2019-blindness-detection/test.csv\")\n\ntrain = train[\n    train.id_code.isin([i.split(\".\")[0] for i in os.listdir(\"../input/aptos2019-blindness-detection/train_images\")])\n]\ntest = test[\n    test.id_code.isin([i.split(\".\")[0] for i in os.listdir(\"../input/aptos2019-blindness-detection/test_images\")])\n]\n\ntest[\"diagnosis\"] = test[\"id_code\"].copy()  # copy this here for ease of retrieval later\n\ntrain.id_code = train.id_code.apply(lambda x: f\"../input/aptos2019-blindness-detection/train_images/{x}.png\")\ntest.id_code = test.id_code.apply(lambda x: f\"../input/aptos2019-blindness-detection/test_images/{x}.png\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Adapt datasets.ImageFolder to our current structure\ndatasets.ImageFolder requires a folder structure of the type\n```\nroot/dog/xxx.png\nroot/dog/xxy.png\nroot/dog/xxz.png\n\nroot/cat/123.png\nroot/cat/nsdf3.png\nroot/cat/asd932_.png\n```\n. We, instead, have all images in a folder together and get the labels from the csv files, so we have to adapt their DatasetFolder\nobject to our needs"},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyDatasetFolder(DatasetFolder):\n    def __init__(\n        self,\n        samples,\n        loader=default_loader,\n        extensions=None,\n        transform=None,\n        target_transform=None,\n        is_valid_file=None,\n    ):\n        self.transform = transform\n        self.target_transform = target_transform\n        classes, class_to_idx = self._find_classes()\n        self.samples = samples\n        if len(samples) == 0:\n            raise (RuntimeError(\"Empty list of samples passed\"))\n\n        self.loader = loader\n        self.extensions = extensions\n\n        self.classes = classes\n        self.class_to_idx = class_to_idx\n        self.samples = samples\n        self.targets = [s[1] for s in samples]\n\n    def _find_classes(self):\n        \"\"\" Hardcoded, as the folders aren't organised by class. \"\"\"\n        classes = [\"No DR\", \"Mild\", \"Moderate\", \"Severe\", \"Proliferative DR\"]\n        class_to_idx = {\n            \"No DR\": 0,\n            \"Mild\": 1,\n            \"Moderate\": 2,\n            \"Severe\": 3,\n            \"Proliferative DR\": 4,\n        }\n        return classes, class_to_idx","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define a function to train the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(\n    model,\n    dataloaders,\n    criterion,\n    optimizer,\n    scheduler,\n    device,\n    dataset_sizes,\n    num_epochs=25,\n    patience=3,\n):\n    since = time.time()\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    no_improvement = 0\n\n    for epoch in range(num_epochs):\n        print(\"Epoch {}/{}\".format(epoch, num_epochs - 1))\n        print(\"-\" * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in [\"train\", \"val\"]:\n            if phase == \"train\":\n                scheduler.step()  # dafuq is scheduler?\n                model.train()  # Set model to training mode\n            else:\n                model.eval()  # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            for inputs, labels in dataloaders[phase]:\n                inputs = inputs.to(\n                    device\n                )  # device? what's that? I think it's just 'cpu'\n                # I think it's more crucial when you've got a gpu\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == \"train\"):\n                    outputs = model(inputs)\n                    _, preds = torch.max(\n                        outputs, 1\n                    )  # just gives you the maximum in each row\n                    loss = criterion(outputs, labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == \"train\":\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)  # loss.item()?\n                running_corrects += torch.sum(preds == labels.data)\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n\n            print(\"{} Loss: {:.4f} Acc: {:.4f}\".format(phase, epoch_loss, epoch_acc))\n\n            # deep copy the model. ok, so there is a kind of early stopping going on here...\n            if phase == \"val\":\n                if epoch_acc > best_acc:\n                    best_acc = epoch_acc\n                    best_model_wts = copy.deepcopy(model.state_dict())\n                    no_improvement = 0\n                else:\n                    no_improvement += 1\n                    print(\n                        f\"No improvement for {no_improvement} round{'s'*int(no_improvement>1)}\"\n                    )\n        if no_improvement > patience:\n            break\n\n        print()\n\n    time_elapsed = time.time() - since\n    print(\n        \"Training complete in {:.0f}m {:.0f}s\".format(\n            time_elapsed // 60, time_elapsed % 60\n        )\n    )\n    print(\"Best val Acc: {:4f}\".format(best_acc))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train, and submit!"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_transforms = {\n    \"train\": transforms.Compose(\n        [\n            transforms.RandomResizedCrop(224),\n            transforms.RandomHorizontalFlip(),\n            transforms.ToTensor(),\n            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n        ]\n    ),\n    \"val\": transforms.Compose(\n        [\n            transforms.Resize(256),\n            transforms.CenterCrop(224),\n            transforms.ToTensor(),\n            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n        ]\n    ),\n}\n\ntrain_data, val_data = train_test_split(\n    train, stratify=train.diagnosis, test_size=0.1\n)\n\ntrain_dataset = MyDatasetFolder(\n    train_data.values.tolist(), transform=data_transforms[\"train\"]\n)\nval_dataset = MyDatasetFolder(\n    val_data.values.tolist(), transform=data_transforms[\"val\"]\n)\ntest_dataset = MyDatasetFolder(\n    test.values.tolist(), transform=data_transforms[\"val\"]\n)\nimage_datasets = {\"train\": train_dataset, \"val\": val_dataset, \"test\": test_dataset}\ndataloaders = {\n    x: torch.utils.data.DataLoader(\n        image_datasets[x], batch_size=4, shuffle=True, num_workers=4\n    )\n    for x in [\"train\", \"val\", \"test\"]\n}\ndataset_sizes = {x: len(image_datasets[x]) for x in [\"train\", \"val\", \"test\"]}\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\nmodel_conv = torchvision.models.resnet50()\nmodel_conv.load_state_dict(torch.load('../input/pytorch-pretrained-models/resnet50-19c8e357.pth'))\nfor param in model_conv.parameters():\n    param.requires_grad = (\n        False\n    )  # right. settings this to false so these parameters don't get retrained.\n\n# Parameters of newly constructed modules have requires_grad=True by default\nnum_ftrs = model_conv.fc.in_features\nmodel_conv.fc = nn.Linear(num_ftrs, 5)  # again, we're replacing the last layer.\n\nmodel_conv = model_conv.to(device)\n\ncriterion = nn.CrossEntropyLoss()\n\n# Observe that only parameters of final layer are being optimized as\n# opposed to before.\noptimizer_conv = optim.SGD(model_conv.fc.parameters(), lr=0.001, momentum=0.9)\n# so...the only difference is that this time, all the other parameters are blocked? cool!\n\n# Decay LR by a factor of 0.1 every 7 epochs\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer_conv, step_size=7, gamma=0.1)\n\nmodel_conv = train_model(\n    model_conv,\n    dataloaders,\n    criterion,\n    optimizer_conv,\n    exp_lr_scheduler,\n    device,\n    dataset_sizes,\n    num_epochs=25,\n    patience=10\n)\n\nsub = pd.read_csv(\"../input/aptos2019-blindness-detection/sample_submission.csv\").set_index(\"id_code\")\n\nfor (inputs, _spam) in dataloaders[\"test\"]:\n    inputs = inputs.to(device)\n    outputs = model_conv(inputs)\n    _, preds = torch.max(outputs, 1)\n\n    preds = preds.to('cpu')\n    sub.loc[list(_spam), \"diagnosis\"] = preds.numpy()\n\nsub.reset_index().to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}