{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install efficientnet-pytorch\n!pip uninstall dataclasses -y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd \nimport tqdm as tqdm \nfrom functools import partial \nimport numpy as np \nimport os \nimport torch \nimport torch.nn as nn \nimport torch.optim as optim \nfrom torch.optim import lr_scheduler\nimport torch.nn.functional as F \nfrom torch.utils.data import Dataset, DataLoader \nfrom torch.utils.data import random_split \nimport torchvision \nfrom torchvision import datasets, models, transforms \nimport time \nimport copy\nimport matplotlib.pyplot as plt \nfrom PIL import Image\n\nfrom ray import tune \nfrom ray.tune import CLIReporter \nfrom ray.tune.schedulers import ASHAScheduler \nfrom sklearn.model_selection import train_test_split \n\n#from efficientnet_pytorch import EfficientNet\n\n\n#from ipynb.fs.full.train_baseline import MyDataset  # to access functions and class from jupyter notebook\n# import nbimporter # to import class and methods of other jupyter notebook \n# from train_baseline import MyDataset\n# from train_baseline import set_transform_mode\n# from train_baseline import train_model\n\nfrom torch.optim import Adam, SGD\nimport albumentations as A","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_transform_mode(mode=None): \n        if mode == \"train\":\n            transform = transforms.Compose([\n                        transforms.RandomResizedCrop(299),  # changed from 224 to 299 \n                        transforms.RandomHorizontalFlip(),\n                        transforms.ToTensor(),\n                        transforms.Normalize((0.485,0.456,0.406), (0.229, 0.224,0.225))]) \n        elif mode == \"val\":\n            transform = transforms.Compose([\n                        transforms.Resize(299), # changed from 256 to 299\n                        transforms.CenterCrop(254), # changed from 224 to 254\n                        transforms.ToTensor(),\n                        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])\n        elif mode == \"test\": \n            pass \n\n        return transform","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create custom dataset with tensor for each image and label for each image\n# source: https://stackoverflow.com/questions/61391919/loading-image-data-from-pandas-to-pytorch \n\nclass MyDataset(Dataset): \n    def __init__(self, dataframe, transform = None): \n        self.dataframe = dataframe \n        self.transform = transform #self.transform_mode(transform, mode)\n      \n    def __len__(self): \n        return len(self.dataframe) \n\n    def __getitem__(self, index): \n        row = self.dataframe.iloc[index] \n        img = Image.open(row[\"path\"])  \n        #tensor = torchvision.transforms.functional.to_tensor(img)\n        label = row[\"label\"]\n        \n        if self.transform: \n            img = self.transform(img) \n        \n        return (img, label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load and transform data, that we have \n\ndef load_data(data=None, mode=None): \n\n    if mode==\"train\":\n        train_transform = set_transform_mode(mode)\n        dataset = MyDataset(data, train_transform) \n    elif mode==\"val\":\n        val_transform = set_transform_mode(mode)\n        dataset = MyDataset(data, val_transform) \n    elif mode==\"test\": \n        test_transform = set_transform_mode(mode)\n        dataset = MyDataset(data, val_transform)       \n\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(config, model, traindata, testdata, criterion, num_epochs=25, checkpoint_dir=None):\n    since = time.time()\n    \n    # set optimizer and tune lr with help of ray tune\n    #optimizer = optim.SGD(resnext.parameters(), lr=config[\"lr\"], momentum=0.9)\n    optimizer = Adam(resnext.parameters(), lr=config[\"lr\"], weight_decay=config[\"weight_decay\"], amsgrad=False)\n \n    device = \"cpu\"\n    if torch.cuda.is_available():\n        device = \"cuda:0\"\n        if torch.cuda.device_count() > 1:\n            model = nn.DataParallel(model)\n    model.to(device)\n    #device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') \n\n    \n    # Create Dataloader for train and validation set\n    trainloader = DataLoader(traindata, batch_size=int(config[\"batch_size\"]), shuffle = True, num_workers=4) \n    valloader = DataLoader(testdata, batch_size=int(config[\"batch_size\"]), shuffle=True, num_workers=4) \n\n    # Train and validation loader in a dict\n    dataloaders = {\"train\": trainloader, \"val\": valloader}\n    dataset_sizes = {j: len(dataloaders[j].dataset) for j in [\"train\", \"val\"]} \n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n\n    # The checkpoints_dir\" parameter gets passed by Ray Tune when a checkpoint should be restored \n    # Reference: https://docs.ray.io/en/master/tune/tutorials/tune-pytorch-cifar.html \n    if checkpoint_dir: \n        checkpoint = os.path.join(checkpoint_dir, \"checkpoint\") \n        model_state, optimizer_state = torch.load(checkpoint)#, map_location=\"cpu\") \n        model.load_state_dict(optimizer_state)\n        optimizer.load_state_dict(optimizer_state)\n    \n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in [\"train\", \"val\"]:      \n            if phase == \"train\":\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Validation loss for ray tune, referenced from here: https://docs.ray.io/en/master/tune/tutorials/tune-pytorch-cifar.html\n            if phase == \"val\": \n                val_loss = 0.0\n                val_steps = 0\n                total = 0\n                correct = 0\n\n            # Iterate over data.\n            for inputs, labels in dataloaders[phase]:\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                    # for ray tune, referenced from here: https://docs.ray.io/en/master/tune/tutorials/tune-pytorch-cifar.html\n                    if phase == \"val\": \n                        total += labels.size(0) \n                        correct += (preds == labels).sum().item()\n                        val_loss += loss.cpu().numpy() \n                        val_steps += 1\n\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n            \n            # if phase == 'train':\n            #     scheduler.step()\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n\n            # deep copy the model\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict()) \n\n            # Here we save a checkpoint. It is automatically registered with\n            # Ray Tune and will potentially be passed as the `checkpoint_dir`\n            # parameter in future iterations.\n            if phase == \"val\": \n                with tune.checkpoint_dir(step=epoch) as checkpoint_dir:\n                    path = os.path.join(checkpoint_dir, \"checkpoint\")\n                    torch.save(\n                        (model.state_dict(), optimizer.state_dict()), path)\n\n                tune.report(loss=(val_loss / val_steps), accuracy=correct / total)\n\n        print()\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    \n    print(\"Finished Training\")\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading data \n# in Kaggle:\ndataset_dir = \"/kaggle/input/cassava-leaf-disease-classification/\"\n# Outside of Kaggle\n#dataset_dir = \"/home/data2/yan/cassava/\"\n\ndata_df = pd.read_csv(dataset_dir + \"train.csv\")   \n\n# Add to column Image_ID the image path in dataframe \ndata_df[\"path\"] = dataset_dir + \"train_images/\" + data_df[\"image_id\"] \n\n# Rearrange column order\ndata_df = data_df[[\"image_id\", \"path\", \"label\"]] \n# data_df = data_df[:500] # for test purpose ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split data in tran and validation data \ntrain, val = train_test_split(data_df, test_size=0.15) \n\n# create dataset with tensors and their belonging labels for each image\ntrain_data = load_data(data=train, mode=\"train\")\nval_data = load_data(data=val, mode=\"val\") ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load pretrained model and finetune covnet\n# load first time online and save offline\nresnext = models.resnext50_32x4d(pretrained=\"True\")\n\nnum_feature = resnext.fc.in_features  # of the final layer\n\nresnext.fc = nn.Linear(num_feature, 5) # change output to 5 classes, expl: https://discuss.pytorch.org/t/how-to-modify-the-final-fc-layer-based-on-the-torch-model/766\n\n# from ray tune: https://docs.ray.io/en/master/tune/tutorials/tune-pytorch-cifar.html\n# device = \"cpu\" \n# if torch.cuda.is_available(): \n#     device = \"cuda:2\" \n    # if torch.cuda.device_count() > 1:\n    #     efficient_net = nn.DataParallel(efficient_net) \n#efficient_net\n\ncriterion = nn.CrossEntropyLoss() \n\n# Decay LR by a factor of 0.1 every 7 epochs\n#exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)  # not with ray tune used\n\n\n# load pretrained model and finetune covnet\n# load first time online and save offline\n\n#efficient_net = EfficientNet.from_pretrained(\"efficientnet-b3\").to(device)\n\n# Observe that all parameters are being optimized\noptimizer_feature = optim.SGD(resnext.parameters(), lr=0.001, momentum=0.9) \n\n# Decay LR by a factor of 0.1 every 7 epochs\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer_feature, step_size=7, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check Fully Connected Layer of Efficient Net \n#efficient_net._fc","execution_count":null,"outputs":[]},{"metadata":{"tags":[],"trusted":true},"cell_type":"code","source":"# Config ray tune parameters \nnum_samples = 5 \nnum_epochs = 5\ngpus_per_trial = 1\ncheckpoint_dir = \"results\"\n\nconfig = {\n    \"lr\": tune.loguniform(1e-4, 1e-1),\n    \"batch_size\": tune.choice([4, 8, 16, 32]),\n    \"weight_decay\": tune.loguniform(1e-8, 1e-3),\n    }\n\nscheduler = ASHAScheduler( \n    metric=\"loss\",\n    mode=\"min\",\n    max_t=num_epochs, \n    grace_period=1,\n    reduction_factor=2\n    )\n\nreporter = CLIReporter( \n    # parameter_columns=[\"lr\", \"batch_size\"] \n    metric_columns=[\"loss\", \"accuracy\", \"training_iteration\"]\n    )\n\nresult = tune.run( \n    partial(train_model, model=resnext, traindata=train_data, testdata=val_data, criterion=criterion, num_epochs=num_epochs),\n    resources_per_trial={\"gpu\": gpus_per_trial},\n    config=config, \n    num_samples=num_samples,\n    scheduler=scheduler, \n    progress_reporter=reporter\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_trial = result.get_best_trial(\"loss\", \"min\", \"last\")\nprint(\"Best trial config: {}\".format(best_trial.config))\nprint(\"Best trial final validation loss: {}\".format(\n    best_trial.last_result[\"loss\"]))\nprint(\"Best trial final validation accuracy: {}\".format(\n    best_trial.last_result[\"accuracy\"]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # train pre-trained model with my parameters \n# efficient_net = train_model(efficient_net, train_data, val_data, criterion, num_epochs=25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save trained model in output/kaggle/working\npath = \"./resnext.pt\"\n\n# Save outside of kaggle \n#path = \"./train/resnext.pt\"\n\n# save uni server\n#path = \"../../../data2/yan/cassava/save/efficient_net_b7_finetune.pt\"\n\n# Save model (parameters: weights and biases of each layer) \ntorch.save(resnext.state_dict(), path) ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}