{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import io\nimport glob\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport seaborn as sns; sns.set()\nfrom sklearn.metrics import f1_score\n\nimport tensorflow as tf\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torchvision.transforms import Compose, Lambda, ToTensor, Normalize, Resize, RandomCrop, TenCrop, RandomHorizontalFlip","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":11.542076,"end_time":"2023-03-06T18:32:47.402019","exception":false,"start_time":"2023-03-06T18:32:35.859943","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-15T17:08:49.323749Z","iopub.execute_input":"2023-03-15T17:08:49.324693Z","iopub.status.idle":"2023-03-15T17:09:00.978414Z","shell.execute_reply.started":"2023-03-15T17:08:49.324638Z","shell.execute_reply":"2023-03-15T17:09:00.977234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Some settings:\n# --------------\ntrain_files = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/train/*.tfrec'\nvalid_files = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/val/*.tfrec'\ntest_files  = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/test/*.tfrec'\ndevice      = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # hardware\nn_epochs    = 10                                                            # number of training epochs\nbatch_size  = 20                                                           # training batch size\nnum_prints  = 10                                                            # number of losses to print per epoch\ntrain_size  = 12753                                                        # number of training data samples\nprint_freq  = train_size // (batch_size * num_prints) + 1                  # print if iteration is a multiple of this\ncheck_freq  = 1                                                            # save model if epoch is a multiple of this","metadata":{"execution":{"iopub.status.busy":"2023-03-15T17:09:00.980823Z","iopub.execute_input":"2023-03-15T17:09:00.981814Z","iopub.status.idle":"2023-03-15T17:09:01.097397Z","shell.execute_reply.started":"2023-03-15T17:09:00.98177Z","shell.execute_reply":"2023-03-15T17:09:01.09614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Utility functions:\n# ------------------\n\ndef tfrecords_to_dataframe(fp, test = False):\n    '''\n    Parse data files into rows of a dataframe.\n    \n    arguments\n    ---------\n    fp : str\n        Data files pattern.\n        \n    test : bool\n        If true, data files correspond to testing data.\n    '''\n    def parse(pb, test = False):\n        d = {'id': tf.io.FixedLenFeature([], tf.string), 'image': tf.io.FixedLenFeature([], tf.string)}\n        if not test:\n            d['class'] = tf.io.FixedLenFeature([], tf.int64)\n        return tf.io.parse_single_example(pb, d)\n\n    df = {'id': [], 'img': []} \n    if not test:\n        df['lab'] = []\n    for sample in tf.data.TFRecordDataset(glob.glob(fp)).map(lambda pb: parse(pb, test)):\n        df['id'].append(sample['id'].numpy().decode('utf-8'))\n        df['img'].append(sample['image'].numpy())\n        if not test:\n            df['lab'].append(sample['class'].numpy())\n    return pd.DataFrame(df)\n\n# ------------------------------------------------------------------------------------------------------------------------\n\ndef display_images(dataset, n, cols):\n    '''\n    Display a grid of labelled images of flowers.\n    \n    arguments\n    ---------\n    dataset : Dataset\n        Dataset containing the flower images and labels.\n        \n    n : int\n        Number of images to display.\n        \n    cols : int\n        Number of columns in the grid.\n    '''\n    rows = n // cols if n % cols == 0 else n // cols + 1\n    plt.figure(figsize = (2 * cols, 2 * rows))\n    for i in range(n):\n        plt.subplot(rows, cols, i + 1)\n        img, lab = dataset[i]\n        plt.imshow(img.permute(1, 2, 0).numpy())\n        plt.title(str(lab))\n        plt.axis('off')\n    plt.show()","metadata":{"papermill":{"duration":0.01665,"end_time":"2023-03-06T18:32:47.421964","exception":false,"start_time":"2023-03-06T18:32:47.405314","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-15T17:09:01.099333Z","iopub.execute_input":"2023-03-15T17:09:01.100093Z","iopub.status.idle":"2023-03-15T17:09:01.112494Z","shell.execute_reply.started":"2023-03-15T17:09:01.100051Z","shell.execute_reply":"2023-03-15T17:09:01.111443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let $X \\sim \\text{Uniform}[300, 640]$. \n\nAs can be seen in the ``Trainset`` class below, each training sample is randomly scaled by sampling $X$, is randomly cropped to size $300$, and is flipped horizontally with probability $0.50$.\n\nAs can be seen in the ``Validset`` and ``Testset`` classes below, each validation and testing sample is scaled to the fixed sizes $372 \\approx \\mathbb{E}[X] - \\sqrt{\\mathbb{V}[X]}$ and $568 \\approx \\mathbb{E}[X] + \\sqrt{\\mathbb{V}[X]}$, each of which is then transformed into 10 new images (i.e., 5 crops of size 300 and the horizontal flip of each crop). In other words, each validation and testing image becomes 20. \n\nTraining batches are set to size 20 whereas validation and testing batches are set to size 1. However, since each validation and testing image becomes 20 (as mentioned above), training, validation, and testing batch sizes are the same. \n\nAs can be seen in the validation and testing loops below, the 20 model outputs that result from a validation or testing input are averaged. Intuitively, the randomness introduced during training is \"averaged out\" during validation and testing.\n\nSee section \"3.4. Implementation\" in https://arxiv.org/pdf/1512.03385.pdf. ","metadata":{}},{"cell_type":"code","source":"# Classes:\n# --------\n\nclass Trainset(Dataset):\n    '''\n    Representation of the training dataset.\n    '''\n    def __init__(self, frac = 1):\n        '''\n        arguments\n        ---------\n        frac : float\n            Fraction of data samples to keep.\n            \n            For example, if frac = 0.5, then a random sample of 50% \n            of the data is kept and the remaining 50% is discarded.\n        '''\n        super().__init__()\n        self.df = tfrecords_to_dataframe(train_files).sample(frac = frac).reset_index(drop = True)\n        self.t1 = Lambda(lambda b: Image.open(io.BytesIO(b)))\n        self.t2 = Compose([RandomCrop(300), \n                           RandomHorizontalFlip(), \n                           ToTensor(), \n                           Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, i):\n        transform = Compose([self.t1, Resize(np.random.randint(300, 641)), self.t2])\n        sample = self.df.iloc[i]\n        return transform(sample['img']), sample['lab']\n\n# -------------------------------------------------------------------------------------------------------------------------    \n\nclass Evalset(Dataset):\n    '''\n    Representation of the evaluation datasets.\n    '''\n    def __init__(self, frac = 1, test = False):\n        '''\n        arguments\n        ---------\n        frac : float\n            Fraction of data samples to keep.\n            \n            For example, if frac = 0.5, then a random sample of 50% \n            of the data is kept and the remaining 50% is discarded.\n            \n        test : bool\n            If true, this dataset contains the testing data. \n            Otherwise, this dataset contains the validation data. \n        '''\n        super().__init__()\n        files = valid_files if not test else test_files\n        self.df = tfrecords_to_dataframe(files, test).sample(frac = frac).reset_index(drop = True)\n        self.transforms = [Compose([Lambda(lambda b: Image.open(io.BytesIO(b))), \n                                    Resize(scale), \n                                    TenCrop(300), \n                                    Lambda(lambda xs: torch.stack([ToTensor()(x) for x in xs])), \n                                    Lambda(lambda xs: torch.stack([Normalize([0.485, 0.456, 0.406], \n                                                                             [0.229, 0.224, 0.225])(x) for x in xs]))])\n                           for scale in [372, 568]]\n        self.test = test\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, i):\n        sample = self.df.iloc[i]\n        imgs = torch.stack([t(sample['img']) for t in self.transforms])\n        return imgs, sample['lab'] if not self.test else sample['id']\n\n# -------------------------------------------------------------------------------------------------------------------------    \n\nclass EfficientNetB0(nn.Module):\n    '''\n    EfficientNet B0 fine-tune.\n    '''\n    def __init__(self, n_classes, learnable_modules = ('classifier.1',)):\n        '''\n        arguments\n        ---------\n        n_classes : int\n            Number of classification categories.\n            \n        learnable_modules : tuple\n            Names of the modules to fine-tune.\n        '''\n        super().__init__()\n        self.efficientnet_b0 = models.efficientnet_b0(weights = 'DEFAULT')\n        self.efficientnet_b0.classifier[1] = nn.Linear(self.efficientnet_b0.classifier[1].in_features, n_classes)\n        self.efficientnet_b0.requires_grad_(False)\n        modules = dict(self.efficientnet_b0.named_modules())\n        for name in learnable_modules:\n            modules[name].requires_grad_(True)\n        \n    def forward(self, x):\n        return F.log_softmax(self.efficientnet_b0(x), dim = 1)","metadata":{"nteract":{"transient":{"deleting":false}},"execution":{"iopub.status.busy":"2023-03-15T17:09:01.116287Z","iopub.execute_input":"2023-03-15T17:09:01.117322Z","iopub.status.idle":"2023-03-15T17:09:01.134902Z","shell.execute_reply.started":"2023-03-15T17:09:01.117287Z","shell.execute_reply":"2023-03-15T17:09:01.134159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training, validation, and testing data:\n# ---------------------------------------\ntrain_set    = Trainset()\ntrain_loader = DataLoader(train_set, batch_size = batch_size, shuffle = True, num_workers = 2)\nvalid_loader = DataLoader(Evalset(frac = 0.20), batch_size = 1, num_workers = 2)\ntest_loader  = DataLoader(Evalset(test = True), batch_size = 1, num_workers = 2)","metadata":{"papermill":{"duration":25.462451,"end_time":"2023-03-06T18:33:12.910134","exception":false,"start_time":"2023-03-06T18:32:47.447683","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-15T17:09:01.136415Z","iopub.execute_input":"2023-03-15T17:09:01.137078Z","iopub.status.idle":"2023-03-15T17:09:34.294669Z","shell.execute_reply.started":"2023-03-15T17:09:01.137042Z","shell.execute_reply":"2023-03-15T17:09:34.293507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display some training images and labels:\n# ----------------------------------------\ndisplay_images(train_set, n = 40, cols = 10)","metadata":{"papermill":{"duration":3.649506,"end_time":"2023-03-06T18:33:16.566804","exception":false,"start_time":"2023-03-06T18:33:12.917298","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-15T17:09:34.29631Z","iopub.execute_input":"2023-03-15T17:09:34.296956Z","iopub.status.idle":"2023-03-15T17:09:38.051258Z","shell.execute_reply.started":"2023-03-15T17:09:34.296897Z","shell.execute_reply":"2023-03-15T17:09:38.050012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the optimizer below, we define five parameter groups (really two as the first four can be combined). \n\nThe first four parameter groups consist of parameters pre-trained on ImageNet and the last parameter group consists of the final affine layer parameters which are trained from scratch. \n\nWe set the pre-trained parameters' learning rate to 0.0001 and the \"from scratch\" parameters' learning rate to 0.001 -- that is, pre-trained parameters are fine-tuned using a learning rate which is an order of magnitude less than the learning rate of the from scratch parameters. \n\nThe learning rate scheduler decays the learning rates of all groups to zero over the course of training.","metadata":{}},{"cell_type":"code","source":"# Modelling components:\n# ---------------------\nmodel = nn.DataParallel(EfficientNetB0(n_classes = 104, learnable_modules = ('features.5.2', \n                                                                             'features.6', \n                                                                             'features.7', \n                                                                             'features.8', \n                                                                             'classifier')))\nmodel.to(device)\n\noptimizer = torch.optim.Adam(params = [{'params': model.module.efficientnet_b0.features[5][2].parameters()}, \n                                       {'params': model.module.efficientnet_b0.features[6].parameters()}, \n                                       {'params': model.module.efficientnet_b0.features[7].parameters()},\n                                       {'params': model.module.efficientnet_b0.features[8].parameters()},\n                                       {'params': model.module.efficientnet_b0.classifier.parameters(), 'lr': 1e-3}], \n                             lr = 1e-4, \n                             weight_decay = 1e-4)\n\nscheduler = CosineAnnealingLR(optimizer, T_max = n_epochs)\n\nloss_fn = F.nll_loss","metadata":{"papermill":{"duration":0.722297,"end_time":"2023-03-06T18:33:17.344","exception":false,"start_time":"2023-03-06T18:33:16.621703","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-15T17:09:38.05245Z","iopub.execute_input":"2023-03-15T17:09:38.052819Z","iopub.status.idle":"2023-03-15T17:09:38.647349Z","shell.execute_reply.started":"2023-03-15T17:09:38.052786Z","shell.execute_reply":"2023-03-15T17:09:38.646196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training loop:\n# --------------\nlosses = []                                                               \nvalid_f1s = []                                                            \nfor epoch in range(n_epochs):\n    print()\n    print(f'Epoch {epoch}:')\n    print('-' * len(f'Epoch {epoch}:'))\n    model.train() \n    for i, (x, y) in enumerate(train_loader):\n        loss = loss_fn(model(x.to(device)), y.to(device))\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        if i % print_freq == 0:\n            print('Loss {}: {:.3f}'.format(i, loss.item()))\n            losses.append(loss.item())\n    if epoch % check_freq == 0:\n        model.eval()\n        valid_true_labs = []\n        valid_pred_labs = []\n        with torch.no_grad():\n            for x, y in valid_loader:\n                valid_true_labs.append(y.item())\n                mean_logp = model(x.view(-1, 3, 300, 300).to(device)).mean(dim = 0)\n                valid_pred_labs.append(torch.argmax(mean_logp).item())\n        valid_f1 = f1_score(valid_true_labs, valid_pred_labs, average = 'weighted')\n        valid_f1s.append(valid_f1)\n        print()\n        print('Validation F1: {:.2f}%'.format(valid_f1 * 100))\n        torch.save(model.state_dict(), f'./epoch{epoch // check_freq}.pth')\n    scheduler.step()","metadata":{"papermill":{"duration":3470.218849,"end_time":"2023-03-06T19:31:07.577597","exception":false,"start_time":"2023-03-06T18:33:17.358748","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-15T17:09:38.652155Z","iopub.execute_input":"2023-03-15T17:09:38.652572Z","iopub.status.idle":"2023-03-15T17:24:15.397269Z","shell.execute_reply.started":"2023-03-15T17:09:38.652531Z","shell.execute_reply":"2023-03-15T17:24:15.395981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimal_epoch = np.argmax(np.array(valid_f1s)) # highest validation F1 epoch / checkpoint frequency","metadata":{"papermill":{"duration":0.036756,"end_time":"2023-03-06T19:31:07.641305","exception":false,"start_time":"2023-03-06T19:31:07.604549","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-15T17:24:15.399309Z","iopub.execute_input":"2023-03-15T17:24:15.400063Z","iopub.status.idle":"2023-03-15T17:24:15.405767Z","shell.execute_reply.started":"2023-03-15T17:24:15.40002Z","shell.execute_reply":"2023-03-15T17:24:15.404699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot training loss and validation F1:\n# -------------------------------------\nplt.figure(figsize = (10, 2.5))\nplt.subplot(1, 2, 1)\nplt.plot(np.arange(len(losses)) / n_epochs, losses, linewidth = 2)\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training')\nplt.subplot(1, 2, 2)\nplt.plot(np.arange(len(valid_f1s)) * check_freq, valid_f1s, linewidth = 2)\nplt.vlines(optimal_epoch * check_freq, 0, valid_f1s[optimal_epoch], colors = 'black', linestyles = 'dashed', label = f'Optimal epoch ({optimal_epoch * check_freq})')\nplt.xlabel('Epoch')\nplt.ylabel('Weighted F1')\nplt.ylim(0, 1)\nplt.title('Validation')\nplt.legend(loc = 'lower left')\nplt.savefig('plot.png')\nplt.show()","metadata":{"papermill":{"duration":0.69134,"end_time":"2023-03-06T19:31:08.358922","exception":false,"start_time":"2023-03-06T19:31:07.667582","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-15T17:24:15.409531Z","iopub.execute_input":"2023-03-15T17:24:15.409948Z","iopub.status.idle":"2023-03-15T17:24:15.949908Z","shell.execute_reply.started":"2023-03-15T17:24:15.409896Z","shell.execute_reply":"2023-03-15T17:24:15.948947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the model which achieved the largest validation F1:\n# --------------------------------------------------------\nmodel = nn.DataParallel(EfficientNetB0(n_classes = 104, learnable_modules = ())).to(device)\nmodel.load_state_dict(torch.load(f'./epoch{optimal_epoch}.pth'))","metadata":{"papermill":{"duration":0.29888,"end_time":"2023-03-06T19:31:08.682134","exception":false,"start_time":"2023-03-06T19:31:08.383254","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-15T17:24:15.951574Z","iopub.execute_input":"2023-03-15T17:24:15.952289Z","iopub.status.idle":"2023-03-15T17:24:16.207038Z","shell.execute_reply.started":"2023-03-15T17:24:15.952246Z","shell.execute_reply":"2023-03-15T17:24:16.205949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission:\n# -----------\nids = []\npreds = []\nmodel.eval()\nwith torch.no_grad():\n    for x, y in test_loader:\n        ids.append(y[0])\n        mean_logp = model(x.view(-1, 3, 300, 300).to(device)).mean(dim = 0)\n        preds.append(torch.argmax(mean_logp).item())\nsubmission = pd.DataFrame({'id': ids, 'label': preds})\nsubmission.to_csv('submission.csv', index = False)\nsubmission.head()","metadata":{"papermill":{"duration":825.091225,"end_time":"2023-03-06T19:44:53.796998","exception":false,"start_time":"2023-03-06T19:31:08.705773","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-15T17:24:16.20844Z","iopub.execute_input":"2023-03-15T17:24:16.208894Z","iopub.status.idle":"2023-03-15T17:37:43.168188Z","shell.execute_reply.started":"2023-03-15T17:24:16.208855Z","shell.execute_reply":"2023-03-15T17:37:43.166888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **If you found this notebook helpful, please consider upvoting.**","metadata":{}}]}