{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":30734,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import io\nimport glob\n\n#data manupulation\nimport numpy as np\nimport pandas as pd\n\n#image processiong\nfrom PIL import Image\n\n#visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns; sns.set()\n\n#machinelearning matrix\nfrom sklearn.metrics import f1_score\n\n#deep learning\nimport tensorflow as tf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torchvision.transforms import Compose, Lambda, ToTensor, Normalize, Resize, RandomCrop, TenCrop, RandomHorizontalFlip","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#traing, test and validation file paths\ntrain_files = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/train/*.tfrec'\nvalid_files = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/val/*.tfrec'\ntest_files  = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/test/*.tfrec'\n#check for the GPU and set\ndevice      = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n#number of epochs to train the model\nn_epochs    = 7\n#define batch size\nbatch_size  = 20  \n#specify loss/efficiency print\nnum_prints  = 10    \n#traing sample size\ntrain_size  = 12753\n#frequency calculation\nprint_freq  = train_size // (batch_size * num_prints) + 1 \ncheck_freq  = 1                                                           ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#convert tfrrecords to dataframe with file pattern and boolean flag\ndef tfrecords_to_dataframe(fp, test = False):\n    '''\n    Parse data files into rows of a dataframe.\n    \n    arguments\n    ---------\n    fp : str\n        Data files pattern.\n        \n    test : bool\n        If true, data files correspond to testing data.\n    '''\n    def parse(pb, test = False):\n        d = {'id': tf.io.FixedLenFeature([], tf.string), 'image': tf.io.FixedLenFeature([], tf.string)}\n        if not test:\n            d['class'] = tf.io.FixedLenFeature([], tf.int64)\n        return tf.io.parse_single_example(pb, d)\n\n    df = {'id': [], 'img': []} \n    if not test:\n        df['lab'] = []\n    #apply on sample dataset\n    for sample in tf.data.TFRecordDataset(glob.glob(fp)).map(lambda pb: parse(pb, test)):\n        df['id'].append(sample['id'].numpy().decode('utf-8'))\n        df['img'].append(sample['image'].numpy())\n        if not test:\n            df['lab'].append(sample['class'].numpy())\n    return pd.DataFrame(df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define a function to display images \ndef display_images(dataset, n, cols):\n    '''\n    Display a grid of labelled images of flowers.\n    \n    arguments\n    ---------\n    dataset : Dataset\n        Dataset containing the flower images and labels.\n        \n    n : int\n        Number of images to display.\n        \n    cols : int\n        Number of columns in the grid.\n    '''\n    rows = n // cols if n % cols == 0 else n // cols + 1\n    plt.figure(figsize = (2 * cols, 2 * rows))\n    for i in range(n):\n        plt.subplot(rows, cols, i + 1)\n        img, lab = dataset[i]\n        plt.imshow(img.permute(1, 2, 0).numpy())\n        plt.title(str(lab))\n        plt.axis('off')\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainset(Dataset):\n    '''\n    Representation of the training dataset.\n    '''\n    def __init__(self, frac = 1):\n        '''\n        arguments\n        ---------\n        frac : float\n            Fraction of data samples to keep.\n            \n            For example, if frac = 0.5, then a random sample of 50% \n            of the data is kept and the remaining 50% is discarded.\n        '''\n        super().__init__()   # Calls the initializer of the parent class (Dataset)\n        \n        # Converts the TFRecord files to a DataFrame and samples a fraction of the data\n        self.df = tfrecords_to_dataframe(train_files).sample(frac = frac).reset_index(drop = True)\n        \n        # Transformation to convert binary image data to a PIL image\n        self.t1 = Lambda(lambda b: Image.open(io.BytesIO(b)))\n        self.t2 = Compose([RandomCrop(300), #Randomly crop the image to 300x300 pixels\n                           RandomHorizontalFlip(), #Randomly flip the image horizontally\n                           ToTensor(), #Convert the image to a tensor\n                           Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])   #Normalize the image using ImageNet mean and std\n        \n    def __len__(self):\n        return self.df.shape[0]   #The number of rows in the DataFrame\n    \n    #Create a transformation pipeline\n    def __getitem__(self, i):\n        transform = Compose([self.t1, Resize(np.random.randint(300, 641)), self.t2])\n        sample = self.df.iloc[i]\n        return transform(sample['img']), sample['lab']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Evalset(Dataset):\n    '''\n    Representation of the evaluation datasets.\n    '''\n    def __init__(self, frac = 1, test = False):\n        '''\n        arguments\n        ---------\n        frac : float\n            Fraction of data samples to keep.\n            \n            For example, if frac = 0.5, then a random sample of 50% \n            of the data is kept and the remaining 50% is discarded.\n            \n        test : bool\n            If true, this dataset contains the testing data. \n            Otherwise, this dataset contains the validation data. \n        '''\n        super().__init__()\n        files = valid_files if not test else test_files\n        self.df = tfrecords_to_dataframe(files, test).sample(frac = frac).reset_index(drop = True)\n        self.transforms = [Compose([Lambda(lambda b: Image.open(io.BytesIO(b))), \n                                    Resize(scale), \n                                    TenCrop(300), \n                                    Lambda(lambda xs: torch.stack([ToTensor()(x) for x in xs])), \n                                    Lambda(lambda xs: torch.stack([Normalize([0.485, 0.456, 0.406], \n                                                                             [0.229, 0.224, 0.225])(x) for x in xs]))])\n                           for scale in [372, 568]]\n        self.test = test\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, i):\n        sample = self.df.iloc[i]\n        imgs = torch.stack([t(sample['img']) for t in self.transforms])\n        return imgs, sample['lab'] if not self.test else sample['id']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EfficientNetB0(nn.Module):\n    '''\n    EfficientNet B0 fine-tune.\n    '''\n    def __init__(self, n_classes, learnable_modules = ('classifier.1',)):\n        '''\n        arguments\n        ---------\n        n_classes : int\n            Number of classification categories.\n            \n        learnable_modules : tuple\n            Names of the modules to fine-tune.\n        '''\n        super().__init__()\n        self.efficientnet_b0 = models.efficientnet_b0(weights = 'DEFAULT')\n        self.efficientnet_b0.classifier[1] = nn.Linear(self.efficientnet_b0.classifier[1].in_features, n_classes)\n        self.efficientnet_b0.requires_grad_(False)\n        modules = dict(self.efficientnet_b0.named_modules())\n        for name in learnable_modules:\n            modules[name].requires_grad_(True)\n        \n    def forward(self, x):\n        return F.log_softmax(self.efficientnet_b0(x), dim = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set    = Trainset()\ntrain_loader = DataLoader(train_set, batch_size = batch_size, shuffle = True, num_workers = 2)\nvalid_loader = DataLoader(Evalset(frac = 0.20), batch_size = 1, num_workers = 2)\ntest_loader  = DataLoader(Evalset(test = True), batch_size = 1, num_workers = 2)\n# Display some training images and labels\ndisplay_images(train_set, n = 40, cols = 10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = nn.DataParallel(EfficientNetB0(n_classes = 104, learnable_modules = ('features.5.2', \n                                                                             'features.6', \n                                                                             'features.7', \n                                                                             'features.8', \n                                                                             'classifier')))\nmodel.to(device)\n\noptimizer = torch.optim.Adam(params = [{'params': model.module.efficientnet_b0.features[5][2].parameters()}, \n                                       {'params': model.module.efficientnet_b0.features[6].parameters()}, \n                                       {'params': model.module.efficientnet_b0.features[7].parameters()},\n                                       {'params': model.module.efficientnet_b0.features[8].parameters()},\n                                       {'params': model.module.efficientnet_b0.classifier.parameters(), 'lr': 1e-3}], \n                             lr = 1e-4, \n                             weight_decay = 1e-4)\n\nscheduler = CosineAnnealingLR(optimizer, T_max = n_epochs)\n\nloss_fn = F.nll_loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"losses = []                                                               \nvalid_f1s = []                                                            \nfor epoch in range(n_epochs):\n    print()\n    print(f'Epoch {epoch}:')\n    print('-' * len(f'Epoch {epoch}:'))\n    model.train() \n    for i, (x, y) in enumerate(train_loader):\n        loss = loss_fn(model(x.to(device)), y.to(device))\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        if i % print_freq == 0:\n            print('Loss {}: {:.3f}'.format(i, loss.item()))\n            losses.append(loss.item())\n    if epoch % check_freq == 0:\n        model.eval()\n        valid_true_labs = []\n        valid_pred_labs = []\n        with torch.no_grad():\n            for x, y in valid_loader:\n                valid_true_labs.append(y.item())\n                mean_logp = model(x.view(-1, 3, 300, 300).to(device)).mean(dim = 0)\n                valid_pred_labs.append(torch.argmax(mean_logp).item())\n        valid_f1 = f1_score(valid_true_labs, valid_pred_labs, average = 'weighted')\n        valid_f1s.append(valid_f1)\n        print()\n        print('Validation F1: {:.2f}%'.format(valid_f1 * 100))\n        torch.save(model.state_dict(), f'./epoch{epoch // check_freq}.pth')\n    scheduler.step()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimal_epoch = np.argmax(np.array(valid_f1s))\nplt.figure(figsize = (10, 2.5))\nplt.subplot(1, 2, 1)\nplt.plot(np.arange(len(losses)) / n_epochs, losses, linewidth = 2)\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training')\nplt.subplot(1, 2, 2)\nplt.plot(np.arange(len(valid_f1s)) * check_freq, valid_f1s, linewidth = 2)\nplt.vlines(optimal_epoch * check_freq, 0, valid_f1s[optimal_epoch], colors = 'black', linestyles = 'dashed', label = f'Optimal epoch ({optimal_epoch * check_freq})')\nplt.xlabel('Epoch')\nplt.ylabel('Weighted F1')\nplt.ylim(0, 1)\nplt.title('Validation')\nplt.legend(loc = 'lower left')\nplt.savefig('plot.png')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = nn.DataParallel(EfficientNetB0(n_classes = 104, learnable_modules = ())).to(device)\nmodel.load_state_dict(torch.load(f'./epoch{optimal_epoch}.pth'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = []\npreds = []\nmodel.eval()\nwith torch.no_grad():\n    for x, y in test_loader:\n        ids.append(y[0])\n        mean_logp = model(x.view(-1, 3, 300, 300).to(device)).mean(dim = 0)\n        preds.append(torch.argmax(mean_logp).item())\nsubmission = pd.DataFrame({'id': ids, 'label': preds})\nsubmission.to_csv('submission.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}