{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":18237,"databundleVersionId":1053191,"sourceType":"competition"},{"sourceId":7223450,"sourceType":"datasetVersion","datasetId":4168209},{"sourceId":7223471,"sourceType":"datasetVersion","datasetId":4168227}],"dockerImageVersionId":30615,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport textwrap\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom PIL import Image\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport random\nimport pytorch_lightning as pl\nfrom torchmetrics.classification import Accuracy\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\n\nSEED = 111\ntorch.manual_seed(SEED)\nnp.random.seed(SEED)\nrandom.seed(SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\ndf_train = pd.read_csv(\"/kaggle/input/imaterialist-fashion-2020-fgvc7/train.csv\")\ndf_train.head()","metadata":{"_uuid":"4de4580b-cd80-4b41-b0d3-baf6625e903a","_cell_guid":"20961a49-aa75-4cc4-a334-7b7bf3e56106","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-17T14:51:31.957304Z","iopub.execute_input":"2023-12-17T14:51:31.957687Z","iopub.status.idle":"2023-12-17T14:52:01.673892Z","shell.execute_reply.started":"2023-12-17T14:51:31.957655Z","shell.execute_reply":"2023-12-17T14:52:01.672303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"/kaggle/input/imaterialist-fashion-2020-fgvc7/label_descriptions.json\", \"r\", encoding=\"UTF-8\") as f:\n    labels_dict = json.load(f)\n\n# labels_dict","metadata":{"_uuid":"bbb3aee0-787a-4d34-82e0-a6c03e0be9bc","_cell_guid":"31058ff8-c427-43cb-9293-111e4129b464","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-17T14:52:01.676639Z","iopub.execute_input":"2023-12-17T14:52:01.677014Z","iopub.status.idle":"2023-12-17T14:52:01.690575Z","shell.execute_reply.started":"2023-12-17T14:52:01.676978Z","shell.execute_reply":"2023-12-17T14:52:01.688607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_CLASSES = len(labels_dict['categories'])\nNUM_CLASSES","metadata":{"_uuid":"bbb3aee0-787a-4d34-82e0-a6c03e0be9bc","_cell_guid":"31058ff8-c427-43cb-9293-111e4129b464","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-17T14:52:01.692551Z","iopub.execute_input":"2023-12-17T14:52:01.692979Z","iopub.status.idle":"2023-12-17T14:52:01.702586Z","shell.execute_reply.started":"2023-12-17T14:52:01.692938Z","shell.execute_reply":"2023-12-17T14:52:01.701327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pretrained & Random Initialization","metadata":{}},{"cell_type":"code","source":"import pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning.loggers import TensorBoardLogger\nimport matplotlib.pyplot as plt\nimport torchmetrics\n\n\nclass ImaterialistDataset(Dataset):\n    def __init__(self, df, data_dir, transform=None):\n        self.df = self._to_1h(df)\n        self.data_dir = data_dir\n        self.transform = transform\n    \n    def _to_1h(self, df):\n        df_1h = df.groupby([\"ImageId\"]).aggregate({\n            \"ClassId\": lambda x: [torch.clamp(\n                torch.sum(\n                    nn.functional.one_hot(\n                        torch.tensor(x.to_numpy()), \n                        NUM_CLASSES\n                    ), \n                    dim=0\n                ), \n                min=0, \n                max=1\n            ).numpy()]\n        }).reset_index()\n        df_1h['ClassId'] = df_1h['ClassId'].apply(lambda x: x[0])\n        return df_1h\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n        img_id = self.df.iloc[idx][\"ImageId\"]\n        label = torch.FloatTensor(self.df.iloc[idx][\"ClassId\"])\n\n        img_path = f\"{self.data_dir}/{img_id}.jpg\"\n        image = Image.open(img_path).convert(\"RGB\")\n\n        if self.transform:\n            image = self.transform(image)\n\n        return image, label\n\n\nd = ImaterialistDataset(df=df_train, data_dir='/kaggle/input/imaterialist-fashion-2020-fgvc7/train', transform=None)\nd[111]","metadata":{"_uuid":"8db84eba-1f58-4808-8b3c-97d6d7658f12","_cell_guid":"432c593d-a838-497a-a102-3873496c00d6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-17T15:04:23.341981Z","iopub.execute_input":"2023-12-17T15:04:23.342383Z","iopub.status.idle":"2023-12-17T15:04:27.339861Z","shell.execute_reply.started":"2023-12-17T15:04:23.342351Z","shell.execute_reply":"2023-12-17T15:04:27.338473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImaterialistDataModule(pl.LightningDataModule):\n    def __init__(self, df, data_dir, batch_size=64):\n        super().__init__()\n        self.df = df\n        self.data_dir = data_dir\n        self.batch_size = batch_size\n        self.train_transform = transforms.Compose([\n            transforms.Resize((128, 128)),\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomRotation(15),\n            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\n        ])\n\n    def setup(self, stage=None):\n        dataset = ImaterialistDataset(df=self.df, data_dir=self.data_dir, transform=self.train_transform)\n        self.train_dataset, self.val_dataset = torch.utils.data.random_split(\n            dataset, [int(len(dataset) * 0.8), len(dataset) - int(len(dataset) * 0.8)]\n        )\n\n    def train_dataloader(self):\n        return DataLoader(dataset=self.train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=4, pin_memory=True)\n    \n    def val_dataloader(self):\n        return DataLoader(dataset=self.val_dataset, batch_size=self.batch_size, shuffle=False, num_workers=4, pin_memory=True)\n\n    \nclass ImaterialistModel(pl.LightningModule):\n    def __init__(self, weights=None, freeze=False, threshold=.5):\n        super().__init__()\n        self.model = models.mobilenet_v3_small(weights=weights)\n        self.model.classifier[-1] = nn.Linear(in_features=self.model.classifier[-1].in_features, out_features=NUM_CLASSES)\n        if freeze:\n            for params in self.model.parameters():\n                params.requires_grad = False\n        for params in self.model.classifier.parameters():\n            params.requires_grad = True\n        self.threshold = threshold\n        self.accuracy = torchmetrics.Accuracy(task=\"multilabel\", num_labels=NUM_CLASSES)\n        self.criterion = nn.BCEWithLogitsLoss()        \n\n    def forward(self, x):\n        return self.model(x)\n\n    def training_step(self, batch, batch_idx):\n        images, labels = batch\n        outputs = self(images)\n        loss = self.criterion(outputs, labels)\n        self.log('train_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True)\n\n        preds = torch.sigmoid(outputs) > self.threshold\n        acc = torch.sum(torch.sum((preds == labels.bool()), 1) == NUM_CLASSES).item() / 64\n        self.log('train_acc', acc, on_step=True, on_epoch=True, prog_bar=True, logger=True)\n        \n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        images, labels = batch\n        outputs = self(images)\n        loss = self.criterion(outputs, labels)\n        self.log('val_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True)\n\n        preds = torch.sigmoid(outputs) > self.threshold\n        acc = torch.sum(torch.sum((preds == labels.bool()), 1) == NUM_CLASSES).item() / 64\n        self.log('val_acc', acc, on_step=True, on_epoch=True, prog_bar=True, logger=True)\n\n        return loss\n\n    def configure_optimizers(self):\n        optimizer = optim.Adam(self.parameters(), lr=0.001)\n        scheduler = CosineAnnealingLR(optimizer, self.trainer.max_epochs, 0)\n        return [optimizer], [scheduler]\n\n\ndata_module = ImaterialistDataModule(df=df_train, data_dir='/kaggle/input/imaterialist-fashion-2020-fgvc7/train')","metadata":{"_uuid":"622e5ef1-c14f-4122-b32f-f0f4c59f8d6b","_cell_guid":"1f37854b-c4bd-4fce-9fa9-272440cf8988","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-17T14:52:15.139879Z","iopub.execute_input":"2023-12-17T14:52:15.140843Z","iopub.status.idle":"2023-12-17T14:52:15.248142Z","shell.execute_reply.started":"2023-12-17T14:52:15.140808Z","shell.execute_reply":"2023-12-17T14:52:15.247002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = ImaterialistModel(models.MobileNet_V3_Small_Weights.IMAGENET1K_V1, freeze=True)\n\n# checkpoint_callback = ModelCheckpoint(\n#     monitor='val_loss',\n#     dirpath='checkpoints',\n#     filename='pretrained-{epoch:02d}-{val_loss:.2f}',\n#     save_top_k=3,\n#     mode='min',\n# )\n\n# logger = TensorBoardLogger(\"tb-logs\", name=\"pretrained\")\n\n# trainer = pl.Trainer(\n#     max_epochs=3,\n#     min_epochs=1,\n#     callbacks=[checkpoint_callback],\n#     logger=logger,\n#     accelerator='auto',\n#     devices=1,\n#     log_every_n_steps=1,\n# )\n\n# trainer.fit(model, datamodule=data_module)","metadata":{"execution":{"iopub.status.busy":"2023-12-17T14:52:15.530597Z","iopub.execute_input":"2023-12-17T14:52:15.530994Z","iopub.status.idle":"2023-12-17T14:52:15.535825Z","shell.execute_reply.started":"2023-12-17T14:52:15.530969Z","shell.execute_reply":"2023-12-17T14:52:15.534588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = ImaterialistModel()\n\n# checkpoint_callback = ModelCheckpoint(\n#     monitor='val_loss',\n#     dirpath='checkpoints',\n#     filename='random-{epoch:02d}-{val_loss:.2f}',\n#     save_top_k=3,\n#     mode='min',\n# )\n\n# logger = TensorBoardLogger(\"tb-logs\", name=\"random\")\n\n# trainer = pl.Trainer(\n#     max_epochs=3,\n#     min_epochs=1,\n#     callbacks=[checkpoint_callback],\n#     logger=logger,\n#     accelerator='auto',\n#     devices=1,\n#     log_every_n_steps=1,\n# )\n\n# trainer.fit(model, datamodule=data_module)","metadata":{"execution":{"iopub.status.busy":"2023-12-17T14:52:15.537539Z","iopub.execute_input":"2023-12-17T14:52:15.537929Z","iopub.status.idle":"2023-12-17T14:52:15.548472Z","shell.execute_reply.started":"2023-12-17T14:52:15.537899Z","shell.execute_reply":"2023-12-17T14:52:15.547084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def class_id_to_name(class_id):\n    for attr_dict in labels_dict[\"categories\"]:\n        if attr_dict[\"id\"] == class_id:\n            return attr_dict[\"name\"]\n    raise ValueError(\"class_id not found\")\n\n\ndef labels_1h_to_name(labels_1h):\n    label_ids = np.nonzero(np.isclose(labels_1h, 1))[0]\n    return [class_id_to_name(label_id) for label_id in label_ids]\n\n\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n])\n\nimage, label = d[111]\n\nlabels = \", \".join(labels_1h_to_name(label))\nlabels = \"\\n\".join(textwrap.wrap(labels))\n\nplt.title(f\"ground truth:\\n\\n{labels}\")\n\nimage = transform(image)\nimage = np.transpose(image.numpy(), (1, 2, 0))\nplt.imshow(np.clip(image, 0, 1))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-17T14:58:35.666493Z","iopub.execute_input":"2023-12-17T14:58:35.666861Z","iopub.status.idle":"2023-12-17T14:58:36.240152Z","shell.execute_reply.started":"2023-12-17T14:58:35.666830Z","shell.execute_reply":"2023-12-17T14:58:36.237910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_image = d[111][0]\n\nmodel = ImaterialistModel.load_from_checkpoint('/kaggle/input/weights/pretrained-epoch02-val_loss0.21.ckpt')\ntransform = transforms.Compose([\n            transforms.Resize((128, 128)),\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomRotation(15),\n            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\n])\nout = model(transform(sample_image).unsqueeze(0))[0]\nout_1h = torch.nn.functional.sigmoid(out) > 0.5\nlabel_ids = np.nonzero(np.isclose(out_1h, 1))[0]\nprint('predicted using pretrained:\\n', labels_1h_to_name(out_1h))\n\nmodel = ImaterialistModel.load_from_checkpoint('/kaggle/input/weights/random-epoch02-val_loss0.21.ckpt')\nout = model(transform(sample_image).unsqueeze(0))[0]\nout_1h = torch.nn.functional.sigmoid(out) > 0.5\nprint('predicted using random:\\n', labels_1h_to_name(out_1h))","metadata":{"execution":{"iopub.status.busy":"2023-12-17T14:58:38.374795Z","iopub.execute_input":"2023-12-17T14:58:38.375173Z","iopub.status.idle":"2023-12-17T14:58:38.771501Z","shell.execute_reply.started":"2023-12-17T14:58:38.375143Z","shell.execute_reply":"2023-12-17T14:58:38.770472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom tensorboard.backend.event_processing.event_accumulator import EventAccumulator\nfrom pathlib import Path\n\n\ndef extract_scalar_events(event_file, tag):\n    event_acc = EventAccumulator(str(event_file))\n    event_acc.Reload()\n    if tag in event_acc.Tags()['scalars']:\n        events = event_acc.Scalars(tag)\n        steps = [event.step for event in events]\n        values = [event.value for event in events]\n        return steps, values\n    else:\n        return [], []\n\nevent_file = '/kaggle/input/tb-logs/events.out.tfevents.1702740133.8ab404751304.27.0'\n\nplt.figure(figsize=(24, 12))\n\ntrain_steps, train_errors = extract_scalar_events(event_file, 'train_loss_step')\nval_steps, val_errors = extract_scalar_events(event_file, 'val_loss_step')\n\nplt.subplot(2, 2, 1)\nplt.plot(train_steps, train_errors, label='Training Loss')\nplt.plot(np.array(val_steps) * 4, val_errors, label='Validation Loss')\nplt.xlabel('Steps')\nplt.ylabel('Loss')\nplt.title('Training vs Validation Loss')\nplt.legend()\n\ntrain_steps, train_errors = extract_scalar_events(event_file, 'train_acc_step')\nval_steps, val_errors = extract_scalar_events(event_file, 'val_acc_step')\n\nplt.subplot(2, 2, 2)\nplt.plot(train_steps, train_errors, label='Training Accuracy')\nplt.plot(np.array(val_steps) * 4, val_errors, label='Validation Accuracy')\nplt.xlabel('Steps')\nplt.ylabel('Accuracy')\nplt.title('Training vs Validation Accuracy')\nplt.legend()\n\nplt.show()","metadata":{"_uuid":"eff4c9b7-ecd5-4690-b23a-e37c23845ce7","_cell_guid":"ff3f6e4d-6732-4571-be81-c03c91e49e46","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-17T13:01:35.464461Z","iopub.execute_input":"2023-12-17T13:01:35.465097Z","iopub.status.idle":"2023-12-17T13:01:36.883360Z","shell.execute_reply.started":"2023-12-17T13:01:35.465057Z","shell.execute_reply":"2023-12-17T13:01:36.882178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom tensorboard.backend.event_processing.event_accumulator import EventAccumulator\nfrom pathlib import Path\n\n\ndef extract_scalar_events(event_file, tag):\n    event_acc = EventAccumulator(str(event_file))\n    event_acc.Reload()\n    if tag in event_acc.Tags()['scalars']:\n        events = event_acc.Scalars(tag)\n        steps = [event.step for event in events]\n        values = [event.value for event in events]\n        return steps, values\n    else:\n        return [], []\n\n\nevent_file = '/kaggle/input/tb-logs/events.out.tfevents.1702818324.16b07179c0c8.26.0'\n\nplt.figure(figsize=(24, 12))\n\ntrain_steps, train_errors = extract_scalar_events(event_file, 'train_loss_step')\nval_steps, val_errors = extract_scalar_events(event_file, 'val_loss_step')\n\nplt.subplot(2, 2, 1)\nplt.plot(train_steps, train_errors, label='Training Loss')\nplt.plot(np.array(val_steps) * 4, val_errors, label='Validation Loss')\nplt.xlabel('Steps')\nplt.ylabel('Loss')\nplt.title('Training vs Validation Loss')\nplt.legend()\n\ntrain_steps, train_errors = extract_scalar_events(event_file, 'train_acc_step')\nval_steps, val_errors = extract_scalar_events(event_file, 'val_acc_step')\n\nplt.subplot(2, 2, 2)\nplt.plot(train_steps, train_errors, label='Training Accuracy')\nplt.plot(np.array(val_steps) * 4, val_errors, label='Validation Accuracy')\nplt.xlabel('Steps')\nplt.ylabel('Accuracy')\nplt.title('Training vs Validation Accuracy')\nplt.legend()\n\nplt.show()","metadata":{"_uuid":"8f869fc6-2ec9-4f52-8574-2f439c9af04d","_cell_guid":"298c2f78-5ccb-4c5e-8d25-53eb8d9e56e5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-17T14:46:22.928242Z","iopub.execute_input":"2023-12-17T14:46:22.928612Z","iopub.status.idle":"2023-12-17T14:46:34.322755Z","shell.execute_reply.started":"2023-12-17T14:46:22.928574Z","shell.execute_reply":"2023-12-17T14:46:34.321414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Metric Learning Using Triplet Loss","metadata":{}},{"cell_type":"code","source":"from collections import defaultdict\nimport random\n\n\nclass ImaterialistTripletDataset(Dataset):\n    def __init__(self, df, data_dir, transform=None):\n        self.df = self._to_1h(df)\n        self.data_dir = data_dir\n        self.transform = transform\n        \n        self.classwise_indices = defaultdict(list)\n        for index, label in zip(self.df['ImageId'], self.df['ClassId']):\n            self.classwise_indices[tuple(label)].append(index)\n        \n        for label, indices in self.classwise_indices.items():\n            if len(indices) > 1:  # only the same image\n                continue\n            num_unmatched_desired = 0\n            num_unmatched_per_label = np.sum(np.array(label) != list(self.df['ClassId']), axis=1)\n            while len(indices) <= 1:\n                num_unmatched_desired += 1\n                indices = list(self.df['ImageId'][num_unmatched_per_label == num_unmatched_desired])\n            self.classwise_indices[label] = indices\n    \n    def _to_1h(self, df):\n        df_1h = df.groupby([\"ImageId\"]).aggregate({\n            \"ClassId\": lambda x: [torch.clamp(\n                torch.sum(\n                    nn.functional.one_hot(\n                        torch.tensor(x.to_numpy()), \n                        NUM_CLASSES\n                    ), \n                    dim=0\n                ), \n                min=0, \n                max=1\n            ).numpy()]\n        }).reset_index()\n        df_1h['ClassId'] = df_1h['ClassId'].apply(lambda x: x[0])\n        return df_1h\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n        anchor_id = self.df.iloc[idx][\"ImageId\"]\n        anchor_label = torch.FloatTensor(self.df.iloc[idx][\"ClassId\"])\n        anchor_image = Image.open(f\"{self.data_dir}/{anchor_id}.jpg\").convert(\"RGB\")\n\n        positive_id = random.choice([img_id for img_id in self.classwise_indices[tuple(self.df.iloc[idx][\"ClassId\"])] if img_id != anchor_id])\n        positive_label = torch.FloatTensor(self.df.loc[self.df['ImageId'] == positive_id][\"ClassId\"].to_numpy()[0])\n        positive_image = Image.open(f\"{self.data_dir}/{positive_id}.jpg\").convert(\"RGB\")\n        \n        negative_id = random.choice([img_id for img_id in self.df['ImageId'] if img_id != anchor_id])\n        negative_label = torch.FloatTensor(self.df.loc[self.df['ImageId'] == negative_id][\"ClassId\"].to_numpy()[0])\n        while torch.all(negative_label == anchor_label):\n            negative_id = random.choice([img_id for img_id in self.df['ImageId'] if img_id != anchor_id])\n            negative_label = torch.FloatTensor(self.df.loc[self.df['ImageId'] == negative_id][\"ClassId\"].to_numpy()[0])\n        negative_image = Image.open(f\"{self.data_dir}/{negative_id}.jpg\").convert(\"RGB\")\n\n        if self.transform:\n            anchor_image = self.transform(anchor_image)\n            positive_image = self.transform(positive_image)\n            negative_image = self.transform(negative_image)\n\n        return (anchor_image, positive_image, negative_image), (anchor_label, positive_label, negative_label)\n\n\nd = ImaterialistTripletDataset(df=df_train, data_dir='/kaggle/input/imaterialist-fashion-2020-fgvc7/train', transform=None)\nd[111]","metadata":{"execution":{"iopub.status.busy":"2023-12-17T17:28:21.076010Z","iopub.execute_input":"2023-12-17T17:28:21.076777Z","iopub.status.idle":"2023-12-17T17:36:26.015813Z","shell.execute_reply.started":"2023-12-17T17:28:21.076740Z","shell.execute_reply":"2023-12-17T17:36:26.014658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.Resize((512, 512)),\n    transforms.ToTensor(),\n])\n\nplt.figure(figsize=(24, 12))\n(anchor_image, positive_image, negative_image), (anchor_label, positive_label, negative_label) = d[111]\nfor i, (image, label, title) in enumerate(zip([anchor_image, positive_image, negative_image],\n                                            [anchor_label, positive_label, negative_label],\n                                            [\"anchor\", \"positive\", \"negative\"])):\n    labels = \", \".join(labels_1h_to_name(label))\n    labels = \"\\n\".join(textwrap.wrap(labels))\n\n    plt.subplot(1, 3, i + 1).set_title(f\"{title}\\n\\n{labels}\")\n\n    image = transform(image)\n    image = np.transpose(image.numpy(), (1, 2, 0))\n    plt.imshow(np.clip(image, 0, 1))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-17T17:36:26.018084Z","iopub.execute_input":"2023-12-17T17:36:26.018418Z","iopub.status.idle":"2023-12-17T17:36:27.750574Z","shell.execute_reply.started":"2023-12-17T17:36:26.018387Z","shell.execute_reply":"2023-12-17T17:36:27.749108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImaterialistTripletDataModule(pl.LightningDataModule):\n    def __init__(self, df, data_dir, batch_size=64):\n        super().__init__()\n        self.df = df\n        self.data_dir = data_dir\n        self.batch_size = batch_size\n        self.train_transform = transforms.Compose([\n            transforms.Resize((128, 128)),\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomRotation(15),\n            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\n        ])\n\n    def setup(self, stage=None):\n        dataset = ImaterialistTripletDataset(df=self.df, data_dir=self.data_dir, transform=self.train_transform)\n        self.train_dataset, self.val_dataset = torch.utils.data.random_split(\n            dataset, [int(len(dataset) * 0.8), len(dataset) - int(len(dataset) * 0.8)]\n        )\n\n    def train_dataloader(self):\n        return DataLoader(dataset=self.train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=4, pin_memory=True)\n    \n    def val_dataloader(self):\n        return DataLoader(dataset=self.val_dataset, batch_size=self.batch_size, shuffle=False, num_workers=4, pin_memory=True)\n\n    \nclass ImaterialistTripletModel(pl.LightningModule):\n    def __init__(self, weights=None, threshold=.5):\n        super().__init__()\n        self.model = models.mobilenet_v3_small(weights=weights)\n        features = list(self.model.classifier.children())[:-1]\n        features.append(nn.Linear(in_features=features[0].out_features, out_features=128))\n        self.model.classifier = nn.Sequential(*features)\n        for params in self.model.parameters():\n            params.requires_grad = False\n        for params in self.model.classifier.parameters():\n            params.requires_grad = True\n\n        self.threshold = threshold\n        self.accuracy = torchmetrics.Accuracy(task=\"multilabel\", num_labels=NUM_CLASSES)\n        self.criterion = nn.TripletMarginLoss(margin=1.0)      \n\n    def forward(self, x):\n        return self.model(x)\n\n    def training_step(self, batch, batch_idx):\n        (anchor, positive, negative), (_, _, _) = batch\n        \n        anchor_emb = self(anchor)\n        positive_emb = self(positive)\n        negative_emb = self(negative)\n        \n        loss = self.criterion(anchor_emb, positive_emb, negative_emb)\n        self.log('train_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        (anchor, positive, negative), (_, _, _) = batch\n        \n        anchor_emb = self(anchor)\n        positive_emb = self(positive)\n        negative_emb = self(negative)\n        \n        loss = self.criterion(anchor_emb, positive_emb, negative_emb)\n        self.log('val_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True)\n        return loss\n\n    def configure_optimizers(self):\n        optimizer = optim.Adam(self.parameters(), lr=0.001)\n        scheduler = CosineAnnealingLR(optimizer, self.trainer.max_epochs, 0)\n        return [optimizer], [scheduler]\n\n\ndata_module = ImaterialistTripletDataModule(df=df_train, data_dir='/kaggle/input/imaterialist-fashion-2020-fgvc7/train')","metadata":{"execution":{"iopub.status.busy":"2023-12-17T15:08:29.448672Z","iopub.execute_input":"2023-12-17T15:08:29.449044Z","iopub.status.idle":"2023-12-17T15:08:29.470341Z","shell.execute_reply.started":"2023-12-17T15:08:29.449011Z","shell.execute_reply":"2023-12-17T15:08:29.468535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ImaterialistTripletModel(models.MobileNet_V3_Small_Weights.IMAGENET1K_V1)\n\ncheckpoint_callback = ModelCheckpoint(\n    monitor='val_loss',\n    dirpath='checkpoints',\n    filename='metric-{epoch:02d}-{val_loss:.2f}',\n    save_top_k=3,\n    mode='min',\n)\n\nlogger = TensorBoardLogger(\"tb-logs\", name=\"metric\")\n\ntrainer = pl.Trainer(\n    max_epochs=3,\n    min_epochs=1,\n    callbacks=[checkpoint_callback],\n    logger=logger,\n    accelerator='auto',\n    devices=1,\n    log_every_n_steps=1,\n)\n\ntrainer.fit(model, datamodule=data_module)","metadata":{"execution":{"iopub.status.busy":"2023-12-16T22:10:33.232244Z","iopub.execute_input":"2023-12-16T22:10:33.232717Z","iopub.status.idle":"2023-12-16T22:29:53.441067Z","shell.execute_reply.started":"2023-12-16T22:10:33.232680Z","shell.execute_reply":"2023-12-16T22:29:53.439457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Self-Supervised Learning Using Augmentation and Triplet Loss","metadata":{}},{"cell_type":"code","source":"from collections import defaultdict\nimport random\n\n\nclass ImaterialistSslDataset(Dataset):\n    def __init__(self, df, data_dir, transform=None):\n        self.df = self._to_1h(df)\n        self.data_dir = data_dir\n        self.transform = transform\n        self.augmentation = transforms.Compose([\n            transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.1),\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomRotation(degrees=15),\n            transforms.RandomAffine(degrees=0, translate=(0.1, 0.1), scale=(0.9, 1.1), shear=10),\n        ])\n\n        \n    def _to_1h(self, df):\n        df_1h = df.groupby([\"ImageId\"]).aggregate({\n            \"ClassId\": lambda x: [torch.clamp(\n                torch.sum(\n                    nn.functional.one_hot(\n                        torch.tensor(x.to_numpy()), \n                        NUM_CLASSES\n                    ), \n                    dim=0\n                ), \n                min=0, \n                max=1\n            ).numpy()]\n        }).reset_index()\n        df_1h['ClassId'] = df_1h['ClassId'].apply(lambda x: x[0])\n        return df_1h\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n        anchor_id = self.df.iloc[idx][\"ImageId\"]\n        anchor_label = torch.FloatTensor(self.df.iloc[idx][\"ClassId\"])\n        anchor_image = Image.open(f\"{self.data_dir}/{anchor_id}.jpg\").convert(\"RGB\")\n\n        positive_label = anchor_label\n        positive_image = self.augmentation(anchor_image)\n        \n        negative_id = random.choice([img_id for img_id in self.df['ImageId'] if img_id != anchor_id])\n        negative_label = torch.FloatTensor(self.df.loc[self.df['ImageId'] == negative_id][\"ClassId\"].to_numpy()[0])\n        while torch.all(negative_label == anchor_label):\n            negative_id = random.choice([img_id for img_id in self.df['ImageId'] if img_id != anchor_id])\n            negative_label = torch.FloatTensor(self.df.loc[self.df['ImageId'] == negative_id][\"ClassId\"].to_numpy()[0])\n        negative_image = Image.open(f\"{self.data_dir}/{negative_id}.jpg\").convert(\"RGB\")\n\n        if self.transform:\n            anchor_image = self.transform(anchor_image)\n            positive_image = self.transform(positive_image)\n            negative_image = self.transform(negative_image)\n\n        return (anchor_image, positive_image, negative_image), (anchor_label, positive_label, negative_label)\n\n\nd = ImaterialistSslDataset(df=df_train, data_dir='/kaggle/input/imaterialist-fashion-2020-fgvc7/train', transform=None)\nd[111]","metadata":{"execution":{"iopub.status.busy":"2023-12-17T15:05:01.480334Z","iopub.execute_input":"2023-12-17T15:05:01.480739Z","iopub.status.idle":"2023-12-17T15:05:06.034641Z","shell.execute_reply.started":"2023-12-17T15:05:01.480707Z","shell.execute_reply":"2023-12-17T15:05:06.032735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.Resize((512, 512)),\n    transforms.ToTensor(),\n])\n\n\nplt.figure(figsize=(24, 12))\n(anchor_image, positive_image, negative_image), (anchor_label, positive_label, negative_label) = d[111]\nfor i, (image, label, title) in enumerate(zip([anchor_image, positive_image, negative_image],\n                                            [anchor_label, positive_label, negative_label],\n                                            [\"anchor\", \"positive\", \"negative\"])):\n    labels = \", \".join(labels_1h_to_name(label))\n    labels = \"\\n\".join(textwrap.wrap(labels))\n\n    plt.subplot(1, 3, i + 1).set_title(f\"{title}\\n\\n{labels}\")\n\n    image = transform(image)\n    image = np.transpose(image.numpy(), (1, 2, 0))\n    plt.imshow(np.clip(image, 0, 1))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-16T23:47:03.454274Z","iopub.execute_input":"2023-12-16T23:47:03.454881Z","iopub.status.idle":"2023-12-16T23:47:04.959364Z","shell.execute_reply.started":"2023-12-16T23:47:03.454856Z","shell.execute_reply":"2023-12-16T23:47:04.957816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImaterialistSslDataModule(pl.LightningDataModule):\n    def __init__(self, df, data_dir, batch_size=64):\n        super().__init__()\n        self.df = df\n        self.data_dir = data_dir\n        self.batch_size = batch_size\n        self.train_transform = transforms.Compose([\n            transforms.Resize((128, 128)),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\n        ])\n\n    def setup(self, stage=None):\n        dataset = ImaterialistSslDataset(df=self.df, data_dir=self.data_dir, transform=self.train_transform)\n        self.train_dataset, self.val_dataset = torch.utils.data.random_split(\n            dataset, [int(len(dataset) * 0.8), len(dataset) - int(len(dataset) * 0.8)]\n        )\n\n    def train_dataloader(self):\n        return DataLoader(dataset=self.train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=4, pin_memory=True)\n    \n    def val_dataloader(self):\n        return DataLoader(dataset=self.val_dataset, batch_size=self.batch_size, shuffle=False, num_workers=4, pin_memory=True)\n\n\ndata_module = ImaterialistSslDataModule(df=df_train, data_dir='/kaggle/input/imaterialist-fashion-2020-fgvc7/train')","metadata":{"execution":{"iopub.status.busy":"2023-12-16T23:47:21.380227Z","iopub.execute_input":"2023-12-16T23:47:21.380742Z","iopub.status.idle":"2023-12-16T23:47:21.393697Z","shell.execute_reply.started":"2023-12-16T23:47:21.380703Z","shell.execute_reply":"2023-12-16T23:47:21.392257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ImaterialistTripletModel(models.MobileNet_V3_Small_Weights.IMAGENET1K_V1)\n\ncheckpoint_callback = ModelCheckpoint(\n    monitor='val_loss',\n    dirpath='checkpoints',\n    filename='ssl-{epoch:02d}-{val_loss:.2f}',\n    save_top_k=3,\n    mode='min',\n)\n\nlogger = TensorBoardLogger(\"tb-logs\", name=\"ssl\")\n\ntrainer = pl.Trainer(\n    max_epochs=3,\n    min_epochs=1,\n    callbacks=[checkpoint_callback],\n    logger=logger,\n    accelerator='auto',\n    devices=1,\n    log_every_n_steps=1,\n)\n\ntrainer.fit(model, datamodule=data_module)","metadata":{"execution":{"iopub.status.busy":"2023-12-16T23:47:21.574235Z","iopub.execute_input":"2023-12-16T23:47:21.575637Z","iopub.status.idle":"2023-12-16T23:50:47.513081Z","shell.execute_reply.started":"2023-12-16T23:47:21.575578Z","shell.execute_reply":"2023-12-16T23:50:47.511877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom tensorboard.backend.event_processing.event_accumulator import EventAccumulator\nfrom pathlib import Path\nimport numpy as np\n\n\ndef extract_scalar_events(event_file, tag):\n    event_acc = EventAccumulator(str(event_file))\n    event_acc.Reload()\n    if tag in event_acc.Tags()['scalars']:\n        events = event_acc.Scalars(tag)\n        steps = [event.step for event in events]\n        values = [event.value for event in events]\n        return steps, values\n    else:\n        return [], []\n\n\ntb_log_dir = '/kaggle/input/tb-logs'\nevent_file = '/kaggle/input/tb-logs/events.out.tfevents.1702771963.b01ee7a1bfab.27.0'\n\nplt.figure(figsize=(24, 12))\n\ntrain_steps, train_errors = extract_scalar_events(event_file, 'train_loss_step')\nval_steps, val_errors = extract_scalar_events(event_file, 'val_loss_step')\n\nplt.subplot(2, 2, 1)\nplt.plot(train_steps, train_errors, label='Training Loss')\nplt.plot(np.array(val_steps) * 4, val_errors, label='Validation Loss')\nplt.xlabel('Steps')\nplt.ylabel('Loss')\nplt.title('Training vs Validation Loss')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-17T12:35:17.145108Z","iopub.execute_input":"2023-12-17T12:35:17.145541Z","iopub.status.idle":"2023-12-17T12:35:17.773510Z","shell.execute_reply.started":"2023-12-17T12:35:17.145508Z","shell.execute_reply":"2023-12-17T12:35:17.772115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom tensorboard.backend.event_processing.event_accumulator import EventAccumulator\nfrom pathlib import Path\n\n\ndef extract_scalar_events(event_file, tag):\n    event_acc = EventAccumulator(str(event_file))\n    event_acc.Reload()\n    if tag in event_acc.Tags()['scalars']:\n        events = event_acc.Scalars(tag)\n        steps = [event.step for event in events]\n        values = [event.value for event in events]\n        return steps, values\n    else:\n        return [], []\n\n\ntb_log_dir = '/kaggle/input/tb-logs'\nevent_file = '/kaggle/input/tb-logs/events.out.tfevents.1702784987.b01ee7a1bfab.27.1'\n\nplt.figure(figsize=(24, 12))\n\ntrain_steps, train_errors = extract_scalar_events(event_file, 'train_loss_step')\nval_steps, val_errors = extract_scalar_events(event_file, 'val_loss_step')\n\nplt.subplot(2, 2, 1)\nplt.plot(train_steps, train_errors, label='Training Loss')\nplt.plot(np.array(val_steps) * 4, val_errors, label='Validation Loss')\nplt.xlabel('Steps')\nplt.ylabel('Loss')\nplt.title('Training vs Validation Loss')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-17T12:36:02.724404Z","iopub.execute_input":"2023-12-17T12:36:02.724923Z","iopub.status.idle":"2023-12-17T12:36:03.302258Z","shell.execute_reply.started":"2023-12-17T12:36:02.724879Z","shell.execute_reply":"2023-12-17T12:36:03.301057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = ImaterialistDataset(df=df_train, data_dir='/kaggle/input/imaterialist-fashion-2020-fgvc7/train', transform=None)\nimage, label = d[111]\n\nlabels = \", \".join(labels_1h_to_name(label))\nlabels = \"\\n\".join(textwrap.wrap(labels))\n\nplt.title(f\"ground truth:\\n\\n{labels}\")\n\nimage = transform(image)\nimage = np.transpose(image.numpy(), (1, 2, 0))\nplt.imshow(np.clip(image, 0, 1))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-17T15:05:35.655258Z","iopub.execute_input":"2023-12-17T15:05:35.655646Z","iopub.status.idle":"2023-12-17T15:05:40.747857Z","shell.execute_reply.started":"2023-12-17T15:05:35.655612Z","shell.execute_reply":"2023-12-17T15:05:40.745809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ImaterialistTripletModel.load_from_checkpoint('/kaggle/input/weights/triplet-epoch02-val_loss0.72.ckpt')\ntransform = transforms.Compose([\n            transforms.Resize((128, 128)),\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomRotation(15),\n            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\n])\nout = model(transform(sample_image).unsqueeze(0))[0]\nprint('encoding using triplet:\\n', out)\n\nmodel = ImaterialistTripletModel.load_from_checkpoint('/kaggle/input/weights/ssl-epoch01-val_loss0.06.ckpt')\nout = model(transform(sample_image).unsqueeze(0))[0]\nprint('encoding using ssl:\\n', out)","metadata":{"execution":{"iopub.status.busy":"2023-12-17T15:09:44.949695Z","iopub.execute_input":"2023-12-17T15:09:44.950188Z","iopub.status.idle":"2023-12-17T15:09:45.459091Z","shell.execute_reply.started":"2023-12-17T15:09:44.950148Z","shell.execute_reply":"2023-12-17T15:09:45.458401Z"},"trusted":true},"execution_count":null,"outputs":[]}]}