{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-19T13:26:59.713955Z","iopub.execute_input":"2024-03-19T13:26:59.714927Z","iopub.status.idle":"2024-03-19T13:26:59.821031Z","shell.execute_reply.started":"2024-03-19T13:26:59.714885Z","shell.execute_reply":"2024-03-19T13:26:59.820149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport io\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport seaborn as sns; sns.set()\nfrom sklearn.metrics import f1_score\nimport tensorflow as tf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torchvision.transforms import Compose, Lambda, ToTensor, Normalize, Resize, RandomCrop, TenCrop, RandomHorizontalFlip\nIMAGE_SIZE = [512, 512]\ntrain_files = \"/kaggle/input/tpu-getting-started/tfrecords-jpeg-512x512/train/*.tfrec\"\ntest_files = \"/kaggle/input/tpu-getting-started/tfrecords-jpeg-512x512/test/*.tfrec\"\nval_files = \"/kaggle/input/tpu-getting-started/tfrecords-jpeg-512x512/val/*.tfrec\"\ndevice = (\"cuda\" if torch.cuda.is_available() else \"cpu\")\nepochs = 10\nbatch_size = 24\nnum_prints  = 10\ntrain_size  = 12753\nprint_freq  = train_size // (batch_size * num_prints) + 1                 \ncheck_freq  = 1\nfolds = 3","metadata":{"execution":{"iopub.status.busy":"2024-03-19T13:27:03.104380Z","iopub.execute_input":"2024-03-19T13:27:03.104751Z","iopub.status.idle":"2024-03-19T13:27:03.115200Z","shell.execute_reply.started":"2024-03-19T13:27:03.104720Z","shell.execute_reply":"2024-03-19T13:27:03.114377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0 \n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) \n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"class\": tf.io.FixedLenFeature([], tf.int64),  \n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label \n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.string), \n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2024-03-19T13:27:04.674632Z","iopub.execute_input":"2024-03-19T13:27:04.675390Z","iopub.status.idle":"2024-03-19T13:27:04.685433Z","shell.execute_reply.started":"2024-03-19T13:27:04.675354Z","shell.execute_reply":"2024-03-19T13:27:04.684438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tfrecords_to_dataframe(fp, test = False):\n    def parse(pb, test = False):\n        d = {'id': tf.io.FixedLenFeature([], tf.string), 'image': tf.io.FixedLenFeature([], tf.string)}\n        if not test:\n            d['class'] = tf.io.FixedLenFeature([], tf.int64)\n        return tf.io.parse_single_example(pb, d)\n\n    df = {'id': [], 'img': []} \n    if not test:\n        df['lab'] = []\n    for sample in tf.data.TFRecordDataset(glob.glob(fp)).map(lambda pb: parse(pb, test)):\n        df['id'].append(sample['id'].numpy().decode('utf-8'))\n        df['img'].append(sample['image'].numpy())\n        if not test:\n            df['lab'].append(sample['class'].numpy())\n    return pd.DataFrame(df)\ndef display_images(dataset, n, cols):\n    rows = n // cols if n % cols == 0 else n // cols + 1\n    plt.figure(figsize = (2 * cols, 2 * rows))\n    for i in range(n):\n        plt.subplot(rows, cols, i + 1)\n        img, lab = dataset[i]\n        plt.imshow(img.permute(1, 2, 0).numpy())\n        plt.title(str(lab))\n        plt.axis('off')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-19T13:27:05.619506Z","iopub.execute_input":"2024-03-19T13:27:05.620276Z","iopub.status.idle":"2024-03-19T13:27:05.630927Z","shell.execute_reply.started":"2024-03-19T13:27:05.620243Z","shell.execute_reply":"2024-03-19T13:27:05.629964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainset(Dataset):\n    def __init__(self, frac = 1):\n        super().__init__()\n        self.df = tfrecords_to_dataframe(train_files).sample(frac = frac).reset_index(drop = True)\n        self.t1 = Lambda(lambda b: Image.open(io.BytesIO(b)))\n        self.t2 = Compose([RandomCrop(300), \n                           RandomHorizontalFlip(), \n                           ToTensor(), \n                           Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\n    def __len__(self):\n        return self.df.shape[0]\n    def __getitem__(self, i):\n        transform = Compose([self.t1, Resize(np.random.randint(300, 641)), self.t2])\n        sample = self.df.iloc[i]\n        return transform(sample['img']), sample['lab'] \n    \nclass Evalset(Dataset):\n    def __init__(self, frac = 1, test = False):\n        super().__init__()\n        files = val_files if not test else test_files\n        self.df = tfrecords_to_dataframe(files, test).sample(frac = frac).reset_index(drop = True)\n        self.transforms = [Compose([Lambda(lambda b: Image.open(io.BytesIO(b))), \n                                    Resize(scale), \n                                    TenCrop(300), \n                                    Lambda(lambda xs: torch.stack([ToTensor()(x) for x in xs])), \n                                    Lambda(lambda xs: torch.stack([Normalize([0.485, 0.456, 0.406], \n                                                                             [0.229, 0.224, 0.225])(x) for x in xs]))])\n                           for scale in [372, 568]]\n        self.test = test\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, i):\n        sample = self.df.iloc[i]\n        imgs = torch.stack([t(sample['img']) for t in self.transforms])\n        return imgs, sample['lab'] if not self.test else sample['id']\n \n\nclass EfficientNetB0(nn.Module):\n    def __init__(self, n_classes, learnable_modules = ('classifier.1',)):\n        super().__init__()\n        self.efficientnet_b0 = models.efficientnet_b0(weights = 'DEFAULT')\n        self.efficientnet_b0.classifier[1] = nn.Linear(self.efficientnet_b0.classifier[1].in_features, n_classes)\n        self.efficientnet_b0.requires_grad_(False)\n        modules = dict(self.efficientnet_b0.named_modules())\n        for name in learnable_modules:\n            modules[name].requires_grad_(True)\n        \n    def forward(self, x):\n        return F.log_softmax(self.efficientnet_b0(x), dim = 1)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T13:27:08.536221Z","iopub.execute_input":"2024-03-19T13:27:08.536942Z","iopub.status.idle":"2024-03-19T13:27:08.555145Z","shell.execute_reply.started":"2024-03-19T13:27:08.536906Z","shell.execute_reply":"2024-03-19T13:27:08.554167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set    = Trainset()\ntrain_loader = DataLoader(train_set, batch_size = batch_size, shuffle = True, num_workers = 2)\nvalid_loader = DataLoader(Evalset(frac = 0.20), batch_size = 1, num_workers = 2)\ntest_loader  = DataLoader(Evalset(test = True), batch_size = 1, num_workers = 2)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T13:27:09.037589Z","iopub.execute_input":"2024-03-19T13:27:09.037992Z","iopub.status.idle":"2024-03-19T13:27:29.370840Z","shell.execute_reply.started":"2024-03-19T13:27:09.037959Z","shell.execute_reply":"2024-03-19T13:27:29.369820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_images(train_set, n = 20, cols = 10)","metadata":{"execution":{"iopub.status.busy":"2024-03-19T13:27:29.372774Z","iopub.execute_input":"2024-03-19T13:27:29.373051Z","iopub.status.idle":"2024-03-19T13:27:32.444578Z","shell.execute_reply.started":"2024-03-19T13:27:29.373026Z","shell.execute_reply":"2024-03-19T13:27:32.443523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = nn.DataParallel(EfficientNetB0(n_classes = 104, learnable_modules = ('features.5.2', \n                                                                             'features.6', \n                                                                             'features.7', \n                                                                             'features.8', \n                                                                             'classifier')))\nmodel.to(device)\n\noptimizer = torch.optim.Adam(params = [{'params': model.module.efficientnet_b0.features[5][2].parameters()}, \n                                       {'params': model.module.efficientnet_b0.features[6].parameters()}, \n                                       {'params': model.module.efficientnet_b0.features[7].parameters()},\n                                       {'params': model.module.efficientnet_b0.features[8].parameters()},\n                                       {'params': model.module.efficientnet_b0.classifier.parameters(), 'lr': 1e-3}], \n                             lr = 1e-4, \n                             weight_decay = 1e-4)\n\nscheduler = CosineAnnealingLR(optimizer, T_max = epochs)\n\nloss_fn = F.nll_loss","metadata":{"execution":{"iopub.status.busy":"2024-03-19T13:27:32.445739Z","iopub.execute_input":"2024-03-19T13:27:32.446027Z","iopub.status.idle":"2024-03-19T13:27:33.305395Z","shell.execute_reply.started":"2024-03-19T13:27:32.446002Z","shell.execute_reply":"2024-03-19T13:27:33.304432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"losses = []                                                               \nvalid_f1s = []                                                            \nfor epoch in range(epochs):\n    print()\n    print(f'Epoch {epoch}:')\n    print('-' * len(f'Epoch {epoch}:'))\n    model.train() \n    for i, (x, y) in enumerate(train_loader):\n        loss = loss_fn(model(x.to(device)), y.to(device))\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        if i % print_freq == 0:\n            print('Loss {}: {:.3f}'.format(i, loss.item()))\n            losses.append(loss.item())\n    if epoch % check_freq == 0:\n        model.eval()\n        valid_true_labs = []\n        valid_pred_labs = []\n        with torch.no_grad():\n            for x, y in valid_loader:\n                valid_true_labs.append(y.item())\n                mean_logp = model(x.view(-1, 3, 300, 300).to(device)).mean(dim = 0)\n                valid_pred_labs.append(torch.argmax(mean_logp).item())\n        valid_f1 = f1_score(valid_true_labs, valid_pred_labs, average = 'weighted')\n        valid_f1s.append(valid_f1)\n        print()\n        print('Validation F1: {:.2f}%'.format(valid_f1 * 100))\n        torch.save(model.state_dict(), f'./epoch{epoch // check_freq}.pth')\n    scheduler.step()","metadata":{"execution":{"iopub.status.busy":"2024-03-19T13:27:44.287044Z","iopub.execute_input":"2024-03-19T13:27:44.287874Z","iopub.status.idle":"2024-03-19T14:03:35.288284Z","shell.execute_reply.started":"2024-03-19T13:27:44.287837Z","shell.execute_reply":"2024-03-19T14:03:35.287046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimal_epoch = np.argmax(np.array(valid_f1s)) ","metadata":{"execution":{"iopub.status.busy":"2024-03-19T14:06:04.424120Z","iopub.execute_input":"2024-03-19T14:06:04.424493Z","iopub.status.idle":"2024-03-19T14:06:04.431556Z","shell.execute_reply.started":"2024-03-19T14:06:04.424462Z","shell.execute_reply":"2024-03-19T14:06:04.430820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 2.5))\nplt.subplot(1, 2, 1)\nplt.plot(np.arange(len(losses)) / epochs, losses, linewidth = 2)\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training')\nplt.subplot(1, 2, 2)\nplt.plot(np.arange(len(valid_f1s)) * check_freq, valid_f1s, linewidth = 2)\nplt.vlines(optimal_epoch * check_freq, 0, valid_f1s[optimal_epoch], colors = 'black', linestyles = 'dashed', label = f'Optimal epoch ({optimal_epoch * check_freq})')\nplt.xlabel('Epoch')\nplt.ylabel('Weighted F1')\nplt.ylim(0, 1)\nplt.title('Validation')\nplt.legend(loc = 'lower left')\nplt.savefig('plot.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-19T14:06:12.014281Z","iopub.execute_input":"2024-03-19T14:06:12.014920Z","iopub.status.idle":"2024-03-19T14:06:12.797638Z","shell.execute_reply.started":"2024-03-19T14:06:12.014888Z","shell.execute_reply":"2024-03-19T14:06:12.796744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}