{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-30T03:33:05.254187Z","iopub.execute_input":"2024-03-30T03:33:05.254609Z","iopub.status.idle":"2024-03-30T03:33:05.291176Z","shell.execute_reply.started":"2024-03-30T03:33:05.254578Z","shell.execute_reply":"2024-03-30T03:33:05.290336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import io\nimport glob\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport seaborn as sns; sns.set()\nfrom sklearn.metrics import f1_score\n\nimport tensorflow as tf\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torchvision.transforms import Compose, Lambda, ToTensor, Normalize, Resize, RandomCrop, TenCrop, RandomHorizontalFlip","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:33:05.294875Z","iopub.execute_input":"2024-03-30T03:33:05.295510Z","iopub.status.idle":"2024-03-30T03:33:05.302656Z","shell.execute_reply.started":"2024-03-30T03:33:05.295479Z","shell.execute_reply":"2024-03-30T03:33:05.301887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# path folder for train, validation, and testing files\ntrain_files = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/train/*.tfrec'\nvalid_files = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/val/*.tfrec'\ntest_files  = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/test/*.tfrec'","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:33:05.303814Z","iopub.execute_input":"2024-03-30T03:33:05.304393Z","iopub.status.idle":"2024-03-30T03:33:05.315314Z","shell.execute_reply.started":"2024-03-30T03:33:05.304344Z","shell.execute_reply":"2024-03-30T03:33:05.314368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device      = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # hardware\nepochs    = 10                                                             # number of training epochs\nbatch_size  = 20                                                           # training batch size\nnum_prints  = 10                                                           # number of losses to print per epoch\ntrain_size  = 12753                                                        # number of training data samples\nprint_freq  = train_size // (batch_size * num_prints) + 1                  # print if iteration is a multiple of this\ncheck_freq  = 1                                                            # save model if epoch is a multiple of this","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:33:05.318049Z","iopub.execute_input":"2024-03-30T03:33:05.318627Z","iopub.status.idle":"2024-03-30T03:33:05.325122Z","shell.execute_reply.started":"2024-03-30T03:33:05.318589Z","shell.execute_reply":"2024-03-30T03:33:05.324372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Parse data files into rows of a dataframe.\ndef tfrecords_to_dataframe(fp, test = False):\n    '''\n    arguments\n    ---------\n    fp : str\n        Data files pattern.\n        \n    test : bool\n        If true, data files correspond to testing data.\n    '''\n    def parse(pb, test = False):\n        d = {'id': tf.io.FixedLenFeature([], tf.string), 'image': tf.io.FixedLenFeature([], tf.string)}\n        if not test:\n            d['class'] = tf.io.FixedLenFeature([], tf.int64)\n        return tf.io.parse_single_example(pb, d)\n\n    df = {'id': [], 'img': []} \n    if not test:\n        df['lab'] = []\n    for sample in tf.data.TFRecordDataset(glob.glob(fp)).map(lambda pb: parse(pb, test)):\n        df['id'].append(sample['id'].numpy().decode('utf-8'))\n        df['img'].append(sample['image'].numpy())\n        if not test:\n            df['lab'].append(sample['class'].numpy())\n    return pd.DataFrame(df)\n\n# Display a grid of labelled images of flowers.\ndef display_images(dataset, n, cols):\n    '''\n    arguments\n    ---------\n    dataset : Dataset\n        Dataset containing the flower images and labels.\n        \n    n : int\n        Number of images to display.\n        \n    cols : int\n        Number of columns in the grid.\n    '''\n    rows = n // cols if n % cols == 0 else n // cols + 1\n    plt.figure(figsize = (2 * cols, 2 * rows))\n    for i in range(n):\n        plt.subplot(rows, cols, i + 1)\n        img, lab = dataset[i]\n        plt.imshow(img.permute(1, 2, 0).numpy())\n        plt.title(str(lab))\n        plt.axis('off')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:33:05.326398Z","iopub.execute_input":"2024-03-30T03:33:05.327121Z","iopub.status.idle":"2024-03-30T03:33:05.341263Z","shell.execute_reply.started":"2024-03-30T03:33:05.327092Z","shell.execute_reply":"2024-03-30T03:33:05.340390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainset(Dataset):\n    def __init__(self, frac = 1):\n        '''\n        arguments\n        ---------\n        frac : float\n            Fraction of data samples to keep.\n            \n            For example, if frac = 0.5, then a random sample of 50% \n            of the data is kept and the remaining 50% is discarded.\n        '''\n        super().__init__()\n        self.df = tfrecords_to_dataframe(train_files).sample(frac = frac).reset_index(drop = True)\n        self.t1 = Lambda(lambda b: Image.open(io.BytesIO(b)))\n        self.t2 = Compose([RandomCrop(300), \n                           RandomHorizontalFlip(), \n                           ToTensor(), \n                           Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, i):\n        transform = Compose([self.t1, Resize(np.random.randint(300, 641)), self.t2])\n        sample = self.df.iloc[i]\n        return transform(sample['img']), sample['lab']\n    \nclass Evalset(Dataset):\n    def __init__(self, frac = 1, test = False):\n        '''\n        arguments\n        ---------\n        frac : float\n            Fraction of data samples to keep.\n            \n            For example, if frac = 0.5, then a random sample of 50% \n            of the data is kept and the remaining 50% is discarded.\n            \n        test : bool\n            If true, this dataset contains the testing data. \n            Otherwise, this dataset contains the validation data. \n            '''\n        super().__init__()\n        files = valid_files if not test else test_files\n        self.df = tfrecords_to_dataframe(files, test).sample(frac = frac).reset_index(drop = True)\n        self.transforms = [Compose([Lambda(lambda b: Image.open(io.BytesIO(b))), \n                                    Resize(scale), \n                                    TenCrop(300), \n                                    Lambda(lambda xs: torch.stack([ToTensor()(x) for x in xs])), \n                                    Lambda(lambda xs: torch.stack([Normalize([0.485, 0.456, 0.406], \n                                                                             [0.229, 0.224, 0.225])(x) for x in xs]))])\n                           for scale in [372, 568]]\n        self.test = test\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, i):\n        sample = self.df.iloc[i]\n        imgs = torch.stack([t(sample['img']) for t in self.transforms])\n        return imgs, sample['lab'] if not self.test else sample['id']\n    \nclass EfficientNetB0(nn.Module):\n    def __init__(self, n_classes, learnable_modules = ('classifier.1',)):\n        '''\n        arguments\n        ---------\n        n_classes : int\n            Number of classification categories.\n            \n        learnable_modules : tuple\n            Names of the modules to fine-tune.\n        '''\n        super().__init__()\n        self.efficientnet_b0 = models.efficientnet_b0(weights = 'DEFAULT')\n        self.efficientnet_b0.classifier[1] = nn.Linear(self.efficientnet_b0.classifier[1].in_features, n_classes)\n        self.efficientnet_b0.requires_grad_(False)\n        modules = dict(self.efficientnet_b0.named_modules())\n        for name in learnable_modules:\n            modules[name].requires_grad_(True)\n        \n    def forward(self, x):\n        return F.log_softmax(self.efficientnet_b0(x), dim = 1)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:33:05.457678Z","iopub.execute_input":"2024-03-30T03:33:05.458061Z","iopub.status.idle":"2024-03-30T03:33:05.474290Z","shell.execute_reply.started":"2024-03-30T03:33:05.458033Z","shell.execute_reply":"2024-03-30T03:33:05.473203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training, validation, and testing data\ntrain_set    = Trainset()\ntrain_loader = DataLoader(train_set, batch_size = batch_size, shuffle = True, num_workers = 2)\nvalid_loader = DataLoader(Evalset(frac = 0.20), batch_size = 1, num_workers = 2)\ntest_loader  = DataLoader(Evalset(test = True), batch_size = 1, num_workers = 2)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:33:05.476573Z","iopub.execute_input":"2024-03-30T03:33:05.476953Z","iopub.status.idle":"2024-03-30T03:33:14.041179Z","shell.execute_reply.started":"2024-03-30T03:33:05.476921Z","shell.execute_reply":"2024-03-30T03:33:14.040193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display some training images and labels\ndisplay_images(train_set, n = 50, cols = 10)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:33:14.043009Z","iopub.execute_input":"2024-03-30T03:33:14.043373Z","iopub.status.idle":"2024-03-30T03:33:20.945892Z","shell.execute_reply.started":"2024-03-30T03:33:14.043341Z","shell.execute_reply":"2024-03-30T03:33:20.944575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model\nmodel = nn.DataParallel(EfficientNetB0(n_classes = 104, learnable_modules = ('features.5.2', \n                                                                             'features.6', \n                                                                             'features.7', \n                                                                             'features.8', \n                                                                             'classifier')))\nmodel.to(device)\n\noptimizer = torch.optim.Adam(params = [{'params': model.module.efficientnet_b0.features[5][2].parameters()}, \n                                       {'params': model.module.efficientnet_b0.features[6].parameters()}, \n                                       {'params': model.module.efficientnet_b0.features[7].parameters()},\n                                       {'params': model.module.efficientnet_b0.features[8].parameters()},\n                                       {'params': model.module.efficientnet_b0.classifier.parameters(), 'lr': 1e-3}], \n                             lr = 1e-4, \n                             weight_decay = 1e-4)\n\nscheduler = CosineAnnealingLR(optimizer, T_max = epochs)\n\nloss_fn = F.nll_loss","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:33:20.947628Z","iopub.execute_input":"2024-03-30T03:33:20.947968Z","iopub.status.idle":"2024-03-30T03:33:21.142414Z","shell.execute_reply.started":"2024-03-30T03:33:20.947938Z","shell.execute_reply":"2024-03-30T03:33:21.141387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(1, epochs + 1):  \n    print()\n    print(f'Epoch {epoch}/{epochs}:')\n    print('-' * len(f'Epoch {epoch}:'))\n    model.train() \n    for i, (x, y) in enumerate(train_loader):\n        loss = loss_fn(model(x.to(device)), y.to(device))\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        if i % print_freq == 0:\n            print('Loss {}: {:.3f}'.format(i, loss.item()))\n            losses.append(loss.item())\n    if epoch % check_freq == 0:\n        model.eval()\n        valid_true_labs = []\n        valid_pred_labs = []\n        with torch.no_grad():\n            for x, y in valid_loader:\n                valid_true_labs.append(y.item())\n                mean_logp = model(x.view(-1, 3, 300, 300).to(device)).mean(dim = 0)\n                valid_pred_labs.append(torch.argmax(mean_logp).item())\n        valid_f1 = f1_score(valid_true_labs, valid_pred_labs, average = 'weighted')\n        valid_f1s.append(valid_f1)\n        print()\n        print('Validation F1: {:.2f}%'.format(valid_f1 * 100))\n        torch.save(model.state_dict(), f'./epoch{epoch // check_freq}.pth')\n    scheduler.step()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T03:33:21.145042Z","iopub.execute_input":"2024-03-30T03:33:21.145464Z","iopub.status.idle":"2024-03-30T13:35:54.141062Z","shell.execute_reply.started":"2024-03-30T03:33:21.145433Z","shell.execute_reply":"2024-03-30T13:35:54.138662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimal_epoch = np.argmax(np.array(valid_f1s))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T13:37:25.987740Z","iopub.execute_input":"2024-03-30T13:37:25.988679Z","iopub.status.idle":"2024-03-30T13:37:25.994734Z","shell.execute_reply.started":"2024-03-30T13:37:25.988639Z","shell.execute_reply":"2024-03-30T13:37:25.993212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 2.5))\nplt.subplot(1, 2, 1)\nplt.plot(np.arange(len(losses)) / epochs, losses, linewidth = 2)\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training')\nplt.subplot(1, 2, 2)\nplt.plot(np.arange(len(valid_f1s)) * check_freq, valid_f1s, linewidth = 2)\nplt.vlines(optimal_epoch * check_freq, 0, valid_f1s[optimal_epoch], colors = 'black', linestyles = 'dashed', label = f'Optimal epoch ({optimal_epoch * check_freq})')\nplt.xlabel('Epoch')\nplt.ylabel('Weighted F1')\nplt.ylim(0, 1)\nplt.title('Validation')\nplt.legend(loc = 'lower left')\nplt.savefig('plot.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T13:37:29.971185Z","iopub.execute_input":"2024-03-30T13:37:29.971643Z","iopub.status.idle":"2024-03-30T13:37:30.762104Z","shell.execute_reply.started":"2024-03-30T13:37:29.971609Z","shell.execute_reply":"2024-03-30T13:37:30.760862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the model with highest validation F1\nmodel = nn.DataParallel(EfficientNetB0(n_classes = 104, learnable_modules = ())).to(device)\nmodel.load_state_dict(torch.load(f'./epoch{optimal_epoch}.pth'))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T13:37:34.995629Z","iopub.execute_input":"2024-03-30T13:37:34.996724Z","iopub.status.idle":"2024-03-30T13:37:35.230888Z","shell.execute_reply.started":"2024-03-30T13:37:34.996686Z","shell.execute_reply":"2024-03-30T13:37:35.229947Z"},"trusted":true},"execution_count":null,"outputs":[]}]}