{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install git+https://github.com/FlorianMuellerklein/PyTorchTrainer\n!pip install cloud-tpu-client==0.10 torch==1.11.0 https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-1.11-cp37-cp37m-linux_x86_64.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-10T02:35:12.083564Z","iopub.execute_input":"2022-06-10T02:35:12.083952Z","iopub.status.idle":"2022-06-10T02:37:21.273827Z","shell.execute_reply.started":"2022-06-10T02:35:12.083907Z","shell.execute_reply":"2022-06-10T02:37:21.272920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **PyTroch Trainer**\n\nThis notebook finetunes the official PyTorch implementation of ConvNext by using the [PyTorchTrainer](https://github.com/FlorianMuellerklein/PyTorchTrainer) package to keep the code simple and readable. \n","metadata":{}},{"cell_type":"code","source":"import os\n#assert os.environ['COLAB_TPU_ADDR']#, 'Make sure to select TPU from Edit > Notebook settings > Hardware accelerator'","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:37:21.276338Z","iopub.execute_input":"2022-06-10T02:37:21.276837Z","iopub.status.idle":"2022-06-10T02:37:21.280564Z","shell.execute_reply.started":"2022-06-10T02:37:21.276797Z","shell.execute_reply":"2022-06-10T02:37:21.279734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport glob\n\nimport numpy as np\nimport pandas as pd\n\nimport tensorflow as tf\n\nfrom typing import List, Optional, Iterable\n\nfrom PIL import Image\n\nimport io\nimport IPython.display as display\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nimport torch.optim as optim\n\n# imports the torch_xla package\n#import torch_xla\n#import torch_xla.core.xla_model as xm\n\nimport torchvision\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import Dataset, DataLoader, ConcatDataset\nimport torchvision.transforms as transforms\nfrom torchvision.utils import make_grid\n\nimport matplotlib.pyplot as plt\n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\");\n\nfrom pytorchtrainer.trainers import SingleOutputTrainer","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:37:21.282611Z","iopub.execute_input":"2022-06-10T02:37:21.283264Z","iopub.status.idle":"2022-06-10T02:37:26.226838Z","shell.execute_reply.started":"2022-06-10T02:37:21.283210Z","shell.execute_reply":"2022-06-10T02:37:26.225929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **1. Data Loading**","metadata":{}},{"cell_type":"markdown","source":"This section contains code that is used in most PyTorch notebooks for this challenge as the data comes in TFRecord format. Why reinvent the wheel herer?","metadata":{}},{"cell_type":"markdown","source":"**Paths**","metadata":{}},{"cell_type":"code","source":"train_files = glob.glob('../input/tpu-getting-started/*/train/*.tfrec')\nval_files = glob.glob('../input/tpu-getting-started/*/val/*.tfrec')\ntest_files = glob.glob('../input/tpu-getting-started/*/test/*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:37:26.228698Z","iopub.execute_input":"2022-06-10T02:37:26.229204Z","iopub.status.idle":"2022-06-10T02:37:26.421345Z","shell.execute_reply.started":"2022-06-10T02:37:26.229165Z","shell.execute_reply":"2022-06-10T02:37:26.420416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Here we read tfrecords files in PyTorch. I recommend** https://medium.com/analytics-vidhya/how-to-read-tfrecords-files-in-pytorch-72763786743f","metadata":{}},{"cell_type":"code","source":"train_feature_description = {\n    'class': tf.io.FixedLenFeature([], tf.int64),\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\ndef _parse_image_function(example_proto):\n  # Parse the input tf.Example proto using the dictionary above.\n  return tf.io.parse_single_example(example_proto, train_feature_description)\n\ntrain_ids = []\ntrain_class = []\ntrain_images = []\n\nfor i in train_files:\n    train_image_dataset = tf.data.TFRecordDataset(i)\n\n    train_image_dataset = train_image_dataset.map(_parse_image_function)\n\n    ids = [str(id_features['id'].numpy())[2:-1] for id_features in train_image_dataset] # [2:-1] is done to remove b' from 1st and 'from last in train id names\n    train_ids = train_ids + ids\n\n    classes = [int(class_features['class'].numpy()) for class_features in train_image_dataset]\n    train_class = train_class + classes\n\n    images = [image_features['image'].numpy() for image_features in train_image_dataset]\n    train_images = train_images + images","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:37:26.422614Z","iopub.execute_input":"2022-06-10T02:37:26.423118Z","iopub.status.idle":"2022-06-10T02:38:47.171222Z","shell.execute_reply.started":"2022-06-10T02:37:26.423072Z","shell.execute_reply":"2022-06-10T02:38:47.170339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_feature_description = {\n    'class': tf.io.FixedLenFeature([], tf.int64),\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\ndef _parse_image_function(example_proto):\n  # Parse the input tf.Example proto using the dictionary above.\n  return tf.io.parse_single_example(example_proto, val_feature_description)\n\nval_ids = []\nval_class = []\nval_images = []\n\nfor i in val_files:\n    val_image_dataset = tf.data.TFRecordDataset(i)\n\n    val_image_dataset = val_image_dataset.map(_parse_image_function)\n\n    ids = [str(image_features['id'].numpy())[2:-1] for image_features in val_image_dataset]\n    val_ids += ids\n\n    classes = [int(image_features['class'].numpy()) for image_features in val_image_dataset]\n    val_class += classes \n\n    images = [image_features['image'].numpy() for image_features in val_image_dataset]\n    val_images += images","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:38:47.172672Z","iopub.execute_input":"2022-06-10T02:38:47.173035Z","iopub.status.idle":"2022-06-10T02:39:12.720947Z","shell.execute_reply.started":"2022-06-10T02:38:47.172998Z","shell.execute_reply":"2022-06-10T02:39:12.720049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_feature_description = {\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\ndef _parse_image_function_test(example_proto):\n    return tf.io.parse_single_example(example_proto, test_feature_description)\n\ntest_ids = []\ntest_images = []\nfor i in test_files:\n    test_image_dataset = tf.data.TFRecordDataset(i)\n    \n    test_image_dataset = test_image_dataset.map(_parse_image_function_test)\n\n    ids = [str(id_features['id'].numpy())[2:-1] for id_features in test_image_dataset]\n    test_ids = test_ids + ids\n\n    images = [image_features['image'].numpy() for image_features in test_image_dataset]\n    test_images = test_images + images","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:12.722380Z","iopub.execute_input":"2022-06-10T02:39:12.722723Z","iopub.status.idle":"2022-06-10T02:39:47.786616Z","shell.execute_reply.started":"2022-06-10T02:39:12.722687Z","shell.execute_reply":"2022-06-10T02:39:47.785702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display.display(display.Image(data=val_images[1]))","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:47.788003Z","iopub.execute_input":"2022-06-10T02:39:47.788361Z","iopub.status.idle":"2022-06-10T02:39:47.797904Z","shell.execute_reply.started":"2022-06-10T02:39:47.788317Z","shell.execute_reply":"2022-06-10T02:39:47.797049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_images), len(val_images), np.unique(train_class), len(np.unique(train_class)), np.unique(val_class), len(np.unique(val_class))","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:47.801413Z","iopub.execute_input":"2022-06-10T02:39:47.802140Z","iopub.status.idle":"2022-06-10T02:39:47.840585Z","shell.execute_reply.started":"2022-06-10T02:39:47.802083Z","shell.execute_reply":"2022-06-10T02:39:47.839661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **2. Data preparation**\n\n**Using the PyTorchTrainer package requires us to use most PyTorch functionality as-is. For this task we'll use the PyTorch Dataset class and the standard dataloaders.**","metadata":{}},{"cell_type":"code","source":"class FlowerDataset(Dataset):\n    \n    def __init__(\n        self,\n        imgs: Iterable = None,\n        targets: Iterable = None,\n        valid: bool = False,\n        tforms: dict = None,\n    ):\n        self.imgs = imgs\n        self.targets = targets\n        self.mode = 'valid' if valid else 'train'\n        self.tforms = tforms\n\n    def __getitem__(self, idx: int) -> dict:\n        # load an augment the image\n        img = Image.open(io.BytesIO(self.imgs[idx]))\n        targ = self.targets[idx]\n        img = tforms[self.mode](img)\n\n        return img, torch.tensor(targ)\n\n    def __len__(self):\n        return len(self.imgs)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:47.842205Z","iopub.execute_input":"2022-06-10T02:39:47.842475Z","iopub.status.idle":"2022-06-10T02:39:47.849649Z","shell.execute_reply.started":"2022-06-10T02:39:47.842449Z","shell.execute_reply":"2022-06-10T02:39:47.848384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Set up the transforms for training and validation/testing.**","metadata":{}},{"cell_type":"code","source":"# transforms for training and validation\ntforms = {\n    'train': transforms.Compose([\n        transforms.RandomResizedCrop(size=224),\n        transforms.RandomHorizontalFlip(p=0.5),\n        transforms.RandomVerticalFlip(p=0.5),\n        transforms.ToTensor(),\n        transforms.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225]\n        )\n    ]),\n    'valid': transforms.Compose([\n        transforms.CenterCrop(224),\n        transforms.Resize(224),\n        transforms.ToTensor(),\n        transforms.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225]\n        )\n    ])\n}","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:47.851250Z","iopub.execute_input":"2022-06-10T02:39:47.851841Z","iopub.status.idle":"2022-06-10T02:39:47.868393Z","shell.execute_reply.started":"2022-06-10T02:39:47.851802Z","shell.execute_reply":"2022-06-10T02:39:47.867413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = FlowerDataset(\n    imgs = train_images,\n    targets = train_class,\n    valid = False,\n    tforms = tforms\n)\n\nvalid_dataset = FlowerDataset(\n    imgs = val_images, \n    targets = val_class,\n    valid = True,\n    tforms = tforms\n)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:47.869573Z","iopub.execute_input":"2022-06-10T02:39:47.869836Z","iopub.status.idle":"2022-06-10T02:39:47.878088Z","shell.execute_reply.started":"2022-06-10T02:39:47.869808Z","shell.execute_reply":"2022-06-10T02:39:47.877204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Use either GPU or TPU**","metadata":{}},{"cell_type":"code","source":"#device = xm.xla_device()\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:47.880925Z","iopub.execute_input":"2022-06-10T02:39:47.881206Z","iopub.status.idle":"2022-06-10T02:39:47.891166Z","shell.execute_reply.started":"2022-06-10T02:39:47.881178Z","shell.execute_reply":"2022-06-10T02:39:47.890277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(\n    train_dataset, \n    64, \n    num_workers=4, \n    pin_memory=True\n)\n\nvalid_loader = DataLoader(\n    valid_dataset, \n    64, \n    num_workers=4, \n    pin_memory=True\n)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:47.894286Z","iopub.execute_input":"2022-06-10T02:39:47.894637Z","iopub.status.idle":"2022-06-10T02:39:47.901126Z","shell.execute_reply.started":"2022-06-10T02:39:47.894596Z","shell.execute_reply":"2022-06-10T02:39:47.900035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **ConvNext**\n\n**Initialize the official PyTorch ConvNext and replace the final layers**","metadata":{}},{"cell_type":"code","source":"# set up network\nnet = torchvision.models.convnext_base(weights='ConvNeXt_Base_Weights.DEFAULT')\nnet.classifier = nn.Sequential(\n    nn.LayerNorm(1024),\n    nn.Flatten(start_dim=1, end_dim=-1),\n    nn.Linear(in_features=1024, out_features=len(np.unique(train_class)), bias=True)\n\n)\nnet = net.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:47.902610Z","iopub.execute_input":"2022-06-10T02:39:47.903088Z","iopub.status.idle":"2022-06-10T02:39:49.650346Z","shell.execute_reply.started":"2022-06-10T02:39:47.903050Z","shell.execute_reply":"2022-06-10T02:39:49.648849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs = 50\n\n# set up training loss and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.AdamW(\n    net.parameters(),\n    lr = 0.0001,\n)\n\n# drop learning rate by factor of 10 after 80% of epochs and again after 90%\nscheduler = torch.optim.lr_scheduler.MultiStepLR(\n    optimizer,\n    milestones = [int(num_epochs * 0.8), int(num_epochs * 0.9)],\n    gamma = 0.1,\n    verbose = True\n)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:49.651572Z","iopub.status.idle":"2022-06-10T02:39:49.652353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Train with PyTorchTrainer**","metadata":{}},{"cell_type":"code","source":"# make a custom accuracy metric\ndef accuracy(targets, preds):\n    _, pred_class = preds.max(-1)\n    total_correct = (pred_class == targets).sum()\n    total = targets.size(0)\n    return total_correct / total\n\n# set up our trainer\ntrainer = SingleOutputTrainer(\n    train_loader = train_loader,\n    valid_loader = valid_loader,\n    net = net,\n    crit = criterion,\n    device = device,\n    optimizer = optimizer,\n    epochs = num_epochs,\n    scheduler = scheduler,\n    metrics = [accuracy],\n    checkpoint_every = 1,\n    model_name = 'flower_convnext'\n)\n\n# train the network\ntrainer.train_network()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:49.653607Z","iopub.status.idle":"2022-06-10T02:39:49.654401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **4. Submit Preparing**","metadata":{}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    \n    def __init__(\n        self,\n        imgs: Iterable = None,\n        img_ids: Iterable = None,\n        valid: bool = False,\n        tforms: dict = None,\n    ):\n        self.imgs = imgs\n        self.img_ids = img_ids\n        self.mode = 'valid' if valid else 'train'\n        self.tforms = tforms\n\n    def __getitem__(self, idx: int) -> dict:\n        # load an augment the image\n        img = Image.open(io.BytesIO(self.imgs[idx]))\n        img = tforms[self.mode](img)\n        \n        img_id = self.img_ids[idx]\n\n        return img, img_id\n\n    def __len__(self):\n        return len(self.imgs)\n\ntest_dataset = TestDataset(\n    imgs = test_images, \n    img_ids = test_ids,\n    tforms = tforms,\n    valid = True\n)\n\ntestloader = DataLoader(\n    test_dataset, \n    128, \n    num_workers=4, p\n    in_memory=True, \n    shuffle=False\n)\n\n\nensemble_df = submit_df.copy()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:49.655831Z","iopub.status.idle":"2022-06-10T02:39:49.656596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\nprediction_ids = []\n\ntrainer.net.eval()\nfor i, (inputs, img_id) in enumerate(testloader):\n    inputs = inputs.to(self.device)\n    \n    preds = trainer.net(inputs)\n    \n    _, pred_class = preds.max(-1)\n    \n    predictions.extend(pred_class)\n    prediction_ids.extend(img_id)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:49.657732Z","iopub.status.idle":"2022-06-10T02:39:49.658493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Final prediction\nsubmit_df = pd.DataFrame({'label': predictions, 'id': prediction_ids})\nsubmit_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:49.659639Z","iopub.status.idle":"2022-06-10T02:39:49.660395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a submission file\nsubmit_df.to_csv('submission11062021.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T02:39:49.661574Z","iopub.status.idle":"2022-06-10T02:39:49.662328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **5. Learning Visualization**","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}