{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import wandb\nimport cassava_utils as utils\n\nimport numpy as np\nimport pandas as pd\nimport os\nfrom PIL import Image\nimport json\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision import transforms, models\nfrom torch.utils.data import DataLoader\nfrom torch.optim import lr_scheduler\n\nfrom sklearn.model_selection import StratifiedKFold, train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# login for weights and biases for logging the training process\nwandb.login()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folderpath = \"../input/cassava-leaf-disease-classification\"\nlabel2name_json = \"../input/cassava-leaf-disease-classification/label_num_to_disease_map.json\"\nseed = 42","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train/Val Split"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(os.path.join(folderpath, \"train.csv\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# only use some samples (e.g. for code testing, simple baseline or overfitting part)\ndf = df.iloc[:2500]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Version 1: using just one train/val split with sklearn.train_test_split\n\ntrain_df, val_df = train_test_split(df, test_size=0.2, \n                                    random_state=seed)\n#save dataframes as csv\ntrain_df.to_csv(\"train_df.csv\", index=False)\nval_df.to_csv(\"val_df.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Version 2: using k-Fold cross validation with stratified k-folds\n# from \"Approaching almost all machine learning problems\" by Abhishek Thaku\n\n# we create a new column called kfold and fill it with -1\ndf[\"kfold\"] = -1\n# the next step is to randomize the rows of the data\ndf = df.sample(frac=1, random_state=seed).reset_index(drop=True)\n # fetch targets\ny = df.label.values\n# initiate the kfold class from model_selection module\nkf = StratifiedKFold(n_splits=5)\n# fill the new kfold column\nfor f, (t_, v_) in enumerate(kf.split(X=df, y=y)):\n    df.loc[v_, 'kfold'] = f\n# save the new csv with kfold column\ndf.to_csv(\"train_folds.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Sanity Check Dataloaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"# define simple transforms\ndef get_transforms(img_size):\n    data_transforms = {\n        'train': transforms.Compose([\n            transforms.RandomResizedCrop(img_size),\n            transforms.ToTensor()\n        ]),\n        'val': transforms.Compose([\n            transforms.CenterCrop(img_size),\n            transforms.ToTensor()\n        ])\n    }\n    return data_transforms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# only look at fold 0\nKFOLD = 0\n# split into train and validation dataframes according to KFOLD\ndf = pd.read_csv(\"train_folds.csv\")\ndfs = {\n    \"train\": df[df.kfold != KFOLD].reset_index(drop=True),\n    \"val\": df[df.kfold == KFOLD].reset_index(drop=True)\n}\n\ndatasets = {x: utils.CassavaDataset(dfs[x],\n                                    os.path.join(folderpath,\"train_images\"),\n                                    transforms=get_transforms(224)[x])\n            for x in [\"train\", \"val\"]}\n\ndataloaders = {x: DataLoader(datasets[x], batch_size=32,\n                             num_workers=4, shuffle=x==\"train\")\n               for x in [\"train\", \"val\"]}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"utils.check_dataloaders(dataloaders)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Simple Model Baseline"},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = \"cuda\"\n# just use one fold\nFOLD = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_training_simplenet(fold, save_model=False):\n    # wandb preferences (configurations and initialization)\n    config_defaults = {\n        \"lr\": 1e-3,\n        \"epochs\": 5,\n        \"img_size\": 224\n    }\n    wandb.init(project=\"cassava\", config=config_defaults)\n    config = wandb.config\n    \n    # split into train and validation dataframes according to fold parameter\n    df = pd.read_csv(\"train_folds.csv\")\n    dfs = {\n        \"train\": df[df.kfold != fold].reset_index(drop=True),\n        \"val\": df[df.kfold == fold].reset_index(drop=True)\n    }\n    \n    datasets = {x: utils.CassavaDataset(dfs[x],\n                                        os.path.join(folderpath,\"train_images\"),\n                                        transforms=get_transforms(config.img_size)[x])\n                for x in [\"train\", \"val\"]}\n    \n    dataloaders = {x: DataLoader(datasets[x], batch_size=16,\n                                 num_workers=8, shuffle=x==\"train\")\n                   for x in [\"train\", \"val\"]}\n    \n    model = utils.SimpleNet()\n    model.to(DEVICE)\n    optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)\n    eng = utils.Engine(model, optimizer, device=DEVICE)\n    \n    best_loss = np.inf\n    \n    for epoch in range(config.epochs):\n        train_loss = eng.train(dataloaders[\"train\"])\n        val_loss, val_acc = eng.evaluate(dataloaders[\"val\"])\n        print('Fold {} epoch [{}/{}] train loss: {:.4f} ' \n                  'valid loss: {:.4f} acc: {:.4f}'.format(\n                      fold, epoch+1, config.epochs, \n                      train_loss, val_loss, val_acc))\n        if val_loss < best_loss:\n            best_loss = val_loss\n            if save_model:\n                torch.save(model, f\"model_{fold}.pth\")\n    return best_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### uncomment line below to run training\n# run_training_simplenet(FOLD)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Insights:\nWe ran different experiments on the simple baseline model and check if we have any bugs in our code and to learn more about the data (inspired by Karpathy's blog post \"A recipe for training neural networks\") \n* Input-independent baseline: set the input to all zero to check that our model extracts any information out of the input during training --> check!\n* Overfit one batch: we tried to reach zero training loss by training only on one batch --> check!\n* Verify decreasing training loss: increased the model capacity (bigger hidden layers) and saw that training loss goes down --> check!"},{"metadata":{},"cell_type":"markdown","source":"# Overfit"},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = \"cuda\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_training_overfit(fold, watch_model=False, save_model=False):\n    #wandb preferences (configurations and initialization)\n    config_defaults = {\n        \"lr\": 2.85e-4,\n        \"epochs\": 15,\n        \"img_size\": 400\n    }\n    wandb.init(project=\"cassava\", config=config_defaults)\n    config = wandb.config\n    \n    # split into train and validation dataframes according to fold parameter\n    df = pd.read_csv(\"train_folds.csv\")\n    dfs = {\n        \"train\": df[df.kfold != fold].reset_index(drop=True),\n        \"val\": df[df.kfold == fold].reset_index(drop=True)\n    }\n    \n    datasets = {x: utils.CassavaDataset(dfs[x],\n                                        os.path.join(folderpath,\"train_images\"),\n                                        transforms=get_transforms(config.img_size)[x])\n                for x in [\"train\", \"val\"]}\n\n    dataloaders = {x: DataLoader(datasets[x], batch_size=16,\n                                 num_workers=8, shuffle=x==\"train\")\n                   for x in [\"train\", \"val\"]}\n    \n    model = utils.create_model_layer4(DEVICE, pretrained=True)\n    optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)\n    eng = utils.Engine(model, optimizer, device=DEVICE)\n    \n    best_loss = np.inf\n    \n    if watch_model:\n        wandb.watch(model, log=\"all\")\n    \n    for epoch in range(config.epochs):\n        train_loss = eng.train(dataloaders[\"train\"])\n        val_loss, val_acc = eng.evaluate(dataloaders[\"val\"])\n        print('Epoch [{}/{}] train loss: {:.4f} ' \n              'valid loss: {:.4f} acc: {:.4f}'.format(\n                epoch+1, config.epochs, train_loss, \n                val_loss, val_acc))\n        if val_loss < best_loss:\n            best_loss = val_loss\n            if save_model:\n                torch.save(model, f\"model_{fold}.pth\")\n        wandb.log({\n            \"Training loss\": train_loss,\n            \"Validation loss\": val_loss,\n            \"Accuracy\": val_acc \n        })\n    return best_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### uncomment line below to run training\n# run training on 5 folds and output the average loss\n# losses = 0\n# for fold in range(5):\n#     loss_temp = run_training_overfit(fold)\n#     losses += loss_temp\n# print (\"Final loss over all 5 folds is:\", str(losses / 5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Regularization\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_albums = {\n    'train': A.Compose([\n        A.RandomResizedCrop(height=400, width=400),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ShiftScaleRotate(p=0.5),\n        A.RandomBrightnessContrast(p=0.5),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2()]),\n    'val': A. Compose([\n        A.Resize(height=400, width=400),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2()])}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check dataloaders\nFOLD = 0\n\ndf = pd.read_csv(\"train_folds.csv\")\ndfs = {\n    \"train\": df[df.kfold != FOLD].reset_index(drop=True),\n    \"val\": df[df.kfold == FOLD].reset_index(drop=True)\n}\n\ndatasets = {x: utils.CassavaDataset(dfs[x],\n                                    os.path.join(folderpath,\"train_images\"),\n                                    albums=data_albums[x])\n            for x in [\"train\", \"val\"]}\n\ndataloaders = {x: DataLoader(datasets[x], batch_size=16,\n                             num_workers=8, shuffle=x==\"train\")\n               for x in [\"train\", \"val\"]}\n\nutils.check_dataloaders(dataloaders, index=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_training_regularization(fold, watch_model=False, save_model=False):\n    # wandb preferences (configurations and initialization)\n    config_defaults = {\n        \"lr\": 1e-4,\n        \"epochs\": 15,\n        \"step_size\": 7,\n        \"gamma\": 0.2\n    }\n    wandb.init(project=\"cassava\", config=config_defaults)\n    config = wandb.config\n    \n    # load csv data and split in train/val according to the fold parameter\n    df = pd.read_csv(\"train_folds.csv\")\n    dfs = {\n        \"train\": df[df.kfold != fold].reset_index(drop=True),\n        \"val\": df[df.kfold == fold].reset_index(drop=True)\n    }\n    \n    # define datasets and dataloaders with helper functions of utils.py \n    datasets = {x: utils.CassavaDataset(dfs[x],\n                                        os.path.join(folderpath,\"train_images\"),\n                                        albums=data_albums[x])\n                for x in [\"train\", \"val\"]}\n\n    dataloaders = {x: DataLoader(datasets[x], batch_size=16,\n                                 num_workers=8, shuffle=x==\"train\")\n                   for x in [\"train\", \"val\"]}\n    \n    # initialize model, optimizer, engine and lr scheduler\n    model = utils.create_model_layer4(DEVICE, pretrained=True)\n    optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)\n    eng = utils.Engine(model, optimizer, device=DEVICE)\n    scheduler = lr_scheduler.StepLR(optimizer, step_size=config.step_size, gamma=config.gamma)\n    \n    if watch_model:\n        wandb.watch(model, log=\"all\")\n    \n    # training loop\n    best_loss = np.inf    \n    \n    for epoch in range(config.epochs):\n        train_loss = eng.train(dataloaders[\"train\"])\n        val_loss, val_acc = eng.evaluate(dataloaders[\"val\"])\n        print('Epoch [{}/{}] train loss: {:.4f} ' \n              'valid loss: {:.4f} acc: {:.4f}'.format(\n                epoch+1, config.epochs, train_loss, \n                val_loss, val_acc))\n        if val_loss < best_loss:\n            best_loss = val_loss\n            if save_model:\n                torch.save(model, f\"model_{fold}.pth\")\n                wandb.save(f\"model_{fold}.pth\")\n\n        wandb.log({\n            \"Training loss\": train_loss,\n            \"Validation loss\": val_loss,\n            \"Accuracy\": val_acc,\n            \"Learning_rate\": optimizer.state_dict()[\"param_groups\"][0][\"lr\"]\n        })\n        scheduler.step()\n    return best_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### uncomment line below to run training\n# run training on 5 folds and output the average loss\n# losses = 0\n# for fold in range(5):\n#     loss_temp = run_training_regularization(fold, save_model=True)\n#     losses += loss_temp\n# print (\"Final loss over all 5 folds is:\", str(losses / 5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LR Finder"},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install torch-lr-finder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch_lr_finder import LRFinder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def use_lr_finder():\n    # used infos on https://pypi.org/project/torch-lr-finder/\n    wandb.init(project=\"cassava\")\n    # only use one fold\n    FOLD = 0\n    \n    # load csv data and split in train/val according to the fold parameter\n    df = pd.read_csv(\"train_folds.csv\")\n    dfs = {\n        \"train\": df[df.kfold != FOLD].reset_index(drop=True),\n        \"val\": df[df.kfold == FOLD].reset_index(drop=True)\n    }\n\n    datasets = {x: utils.CassavaDataset(dfs[x],\n                                        os.path.join(folderpath,\"train_images\"),\n                                        albums=data_albums[x])\n                for x in [\"train\", \"val\"]}\n\n    dataloaders = {x: DataLoader(datasets[x], batch_size=16,\n                                 num_workers=8, shuffle=x==\"train\")\n                   for x in [\"train\", \"val\"]}\n    \n    model = utils.create_model_layer4(DEVICE, pretrained=True)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-7)\n    lr_finder = LRFinder(model, optimizer, criterion, device=DEVICE)\n    lr_finder.range_test(dataloaders[\"train\"], end_lr=100, num_iter=100)\n    lr_finder.plot() # to inspect the loss-learning rate graph\n    lr_finder.reset() # to reset the model and optimizer to their initial state","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### uncomment to run learning rate finder\n# use_lr_finder()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluate predictions with confusion matrix"},{"metadata":{},"cell_type":"markdown","source":"Evaluates the predictions of one model, which was trained above on training data, on unseen validation data."},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = 'cuda'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = \"../input/resnet50/model (1).pth\"\nlabel2name_json = \"../input/cassava-leaf-disease-classification/label_num_to_disease_map.json\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_albums = {\n    'test': A.Compose([\n        A.Resize(height=400, width=400),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2()])}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_test_dataloader(dataframe, root_dir, data_albums):\n    test_files = np.asarray(dataframe.iloc[:,0])\n    test_dataset = utils.CassavaTestDataset(test_files, root_dir, albums=data_albums[\"test\"])\n    test_dataloader = {\"test\": DataLoader(test_dataset, batch_size=16, num_workers=8)}\n    return test_dataloader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"root_dir = os.path.join(folderpath, \"train_images\")\nval_df = pd.read_csv(\"./val_df.csv\")\n\nval_dataloader = create_test_dataloader(val_df, root_dir, data_albums)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torch.load(model_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_inference_df(dataloader, model, device):\n    test_eng = utils.Engine(model, optimizer=None, device=device)\n    inference_df = test_eng.predict(dataloader[\"test\"])\n    return inference_df ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get predictions for validation data\nval_preds_df = get_inference_df(val_dataloader, model, DEVICE)\nval_preds = val_preds_df.iloc[:,1]\n\n# get labels for validation data\nval_labels = val_df.iloc[:,1]\n\n# get confusion matrix\nconf_mat = confusion_matrix(val_labels, val_preds, normalize=\"true\")\n\n# display confusion matrix\ndisp = ConfusionMatrixDisplay(conf_mat)\ndisp.plot();","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}