{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup\nThis notebook is intended to run on colab, so some things are commented out to make it work on kaggle.","metadata":{"id":"5tPFJ89V3BFT"}},{"cell_type":"code","source":"# !nvidia-smi","metadata":{"id":"vW7S-8qm3FfU","outputId":"eb071dcb-9f88-4cc5-e413-9fb027a385f4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from google.colab import drive\n# drive.mount('/gdrive')\n# %cd /gdrive","metadata":{"id":"_ABy2M3C3HEb","outputId":"485bc19b-4d51-41c0-b5ab-688fb5d39abb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install efficientnet_pytorch\n!pip install git+https://github.com/rwightman/pytorch-image-models\n!pip install pytorch-metric-learning\n!pip install faiss-gpu\n!pip install imgaug -U\n!pip install albumentations -U","metadata":{"id":"1JM4AM6M2_aj","outputId":"b48d38cb-5d44-4de7-b390-1490ab10ae53","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{"id":"MyC4gTwZ3MKJ"}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nimport os\nimport math","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","id":"u0Bz2ktn2_ap","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.utils import class_weight\nfrom PIL import Image as pil_image\nfrom tqdm import tqdm\nimport scipy\n\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go","metadata":{"id":"tOszKuxt3PXn"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\n\nimport timm\nfrom timm.optim import Lookahead, RAdam\nfrom pytorch_metric_learning import miners, losses, samplers , distances, regularizers ","metadata":{"id":"uQE7wYFR3QxV"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Global","metadata":{"id":"tirOg6jm3aIB"}},{"cell_type":"code","source":"IMG_SIZE = 512\nSEED = 42\nPROJECT_FOLDER = \"../input/hotel-id-2021-fgvc8/\"\nDATA_FOLDER = \"../input/hotelid-images-512x512-padded/\"\nOUTPUT_FOLDER = \"./\"\n\n# PROJECT_FOLDER = \"/gdrive/MyDrive/Projects/Hotel-ID/\"\n# DATA_FOLDER = \"/home/data/\"\n# OUTPUT_FOLDER = PROJECT_FOLDER + \"output/\"","metadata":{"id":"DV7qHDuYGoJH"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !mkdir {DATA_FOLDER}\n# !unzip -qq {PROJECT_FOLDER}data/train-{IMG_SIZE}x{IMG_SIZE}.zip -d /home/data/","metadata":{"id":"TB9CXg8U3bbQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.listdir(PROJECT_FOLDER))\nprint(len(os.listdir(DATA_FOLDER)))","metadata":{"id":"1wH0zWUS2_aq","outputId":"77eb5bd2-13cd-4547-de9f-37391185cd8b","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions - seed and metric calculator","metadata":{"id":"ZmZ-HheL3itu"}},{"cell_type":"code","source":"def seed_everything(seed):\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True","metadata":{"id":"csp2OMgo2_ar","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset and transformations","metadata":{"id":"8V_xuoN73lON"}},{"cell_type":"code","source":"import albumentations as A\nimport albumentations.pytorch as APT\nimport cv2 \n\ntrain_transform = A.Compose([\n # A.Resize(IMG_SIZE, IMG_SIZE),\n # A.CLAHE(p=1), \n \n A.HorizontalFlip(p=0.75),\n A.VerticalFlip(p=0.25),\n A.ShiftScaleRotate(p=0.5, border_mode=cv2.BORDER_CONSTANT),\n A.OpticalDistortion(p=0.25),\n A.IAAPerspective(p=0.25),\n A.CoarseDropout(p=0.5),\n\n A.RandomBrightness(p=0.75),\n A.ToFloat(),\n APT.transforms.ToTensor(),\n])\n\n\nval_transform = A.Compose([\n # A.Resize(IMG_SIZE, IMG_SIZE),\n # A.CLAHE(p=1),\n A.ToFloat(),\n APT.transforms.ToTensor(),\n])","metadata":{"id":"8ucWZHeG2_as","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HotelTrainDataset:\n def __init__(self, data, transform=None, data_path=\"train_images/\"):\n self.data = data\n self.data_path = data_path\n self.transform = transform\n self.fake_load = False\n\n def __len__(self):\n return len(self.data)\n \n def __getitem__(self, idx):\n record = self.data.iloc[idx]\n image_path = self.data_path + record[\"image\"]\n\n if self.fake_load:\n image = np.random.randint(0, 255, (32, 32, 3)).astype(np.uint8)\n else:\n image = np.array(pil_image.open(image_path)).astype(np.uint8)\n\n if self.transform:\n transformed = self.transform(image=image)\n \n return {\n \"image\" : transformed[\"image\"],\n \"target\" : record['hotel_id_code'],\n }","metadata":{"id":"EiLYsfKq2_at","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"id":"FpR2HfK93pvS"}},{"cell_type":"code","source":"class EmbeddingNet(nn.Module):\n def __init__(self, n_classes=100, embed_size=64, backbone_name=\"efficientnet_b0\"):\n super(EmbeddingNet, self).__init__()\n \n self.embed_size = embed_size\n self.backbone = timm.create_model(backbone_name, pretrained=True)\n in_features = self.backbone.get_classifier().in_features\n\n fc_name, _ = list(self.backbone.named_modules())[-1]\n if fc_name == 'classifier':\n self.backbone.classifier = nn.Identity()\n elif fc_name == 'head.fc':\n self.backbone.head.fc = nn.Identity()\n elif fc_name == 'fc':\n self.backbone.fc = nn.Identity()\n else:\n raise Exception(\"unknown classifier layer: \" + fc_name)\n \n self.post = nn.Sequential(\n nn.utils.weight_norm(nn.Linear(in_features, self.embed_size*2), dim=None),\n nn.BatchNorm1d(self.embed_size*2),\n nn.Dropout(0.2),\n nn.utils.weight_norm(nn.Linear(self.embed_size*2, self.embed_size)),\n )\n\n self.classifier = nn.Sequential(\n nn.BatchNorm1d(self.embed_size),\n nn.Dropout(0.2),\n nn.Linear(self.embed_size, n_classes),\n )\n \n def embed_and_classify(self, x):\n x = self.forward(x)\n return x, self.classifier(x)\n\n def forward(self, x):\n x = self.backbone(x)\n x = x.view(x.size(0), -1)\n x = self.post(x)\n return x","metadata":{"id":"_2mse3zX3pFQ"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model helper functions","metadata":{"id":"mTFCinps35ci"}},{"cell_type":"code","source":"def get_embeds(loader, model, bar_desc=\"Generating embeds\"):\n targets_all = []\n outputs_all = []\n \n model.eval()\n with torch.no_grad():\n t = tqdm(loader, desc=bar_desc)\n for i, sample in enumerate(t):\n input = sample['image'].to(args.device)\n target = sample['target'].to(args.device)\n output = model(input)\n \n targets_all.extend(target.cpu().numpy())\n outputs_all.extend(output.detach().cpu().numpy())\n \n return targets_all, outputs_all\n\ndef get_prediction(loader, model, bar_desc=\"Generating predictions\"):\n targets_all = []\n outputs_all = []\n \n model.eval()\n with torch.no_grad():\n t = tqdm(loader, desc=bar_desc)\n for i, sample in enumerate(t):\n input = sample['image'].to(args.device)\n target = sample['target'].to(args.device)\n _, output = model.embed_and_classify(input)\n output = torch.argsort(torch.sigmoid(output), descending=True)\n\n targets_all.extend(target.cpu().numpy())\n outputs_all.extend(output.detach().cpu().numpy())\n \n return targets_all, outputs_all","metadata":{"id":"xW5LIe1l2_at","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_distance_matrix(embeds, base_embeds, distance_func):\n distance_matrix = []\n base_embeds = torch.Tensor(base_embeds)\n embeds_dataset = torch.utils.data.TensorDataset(torch.Tensor(embeds))\n embeds_dataloader = DataLoader(embeds_dataset, num_workers=2, batch_size=1024, shuffle=False)\n \n t = tqdm(embeds_dataloader)\n for i, sample in enumerate(t): \n distances = distance_func(sample[0], base_embeds)\n distance_matrix.extend(distances.numpy())\n \n return np.array(distance_matrix)","metadata":{"id":"syXhlJrJG2AV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_checkpoint(model, scheduler, optimizer, epoch, name, loss=None, score=None):\n checkpoint = {\"epoch\": epoch,\n \"model\": model.state_dict(),\n \"scheduler\": scheduler.state_dict(),\n \"optimizer\": optimizer.state_dict(),\n \"loss\": loss,\n \"score\": score,\n }\n\n torch.save(checkpoint, f\"{OUTPUT_FOLDER}checkpoint-{name}.pt\")\n\n\ndef load_checkpoint(model, scheduler, optimizer, name):\n checkpoint = torch.load(f\"{OUTPUT_FOLDER}checkpoint-{name}.pt\")\n\n model.load_state_dict(checkpoint[\"model\"])\n scheduler.load_state_dict(checkpoint[\"scheduler\"])\n # optimizer.load_state_dict(checkpoint[\"optimizer\"])\n\n return model, scheduler, optimizer, checkpoint[\"epoch\"]","metadata":{"id":"ryZ6wE0zKPiz"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def iterate_loader(loader, epochs):\n loader.dataset.fake_load = True\n for i in range(epochs):\n with torch.no_grad():\n t = tqdm(loader, desc=f\"Iterating loader {i+1}/{epochs}\")\n for j, sample in enumerate(t):\n images = sample['image']\n targets = sample['target']\n\n loader.dataset.fake_load = False","metadata":{"id":"o8sQ9dtJH1fu"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_epoch(args, model, loader, criterion, optimizer, scheduler, epoch):\n losses = []\n targets_all = []\n outputs_all = []\n \n model.train()\n t = tqdm(loader)\n \n for i, sample in enumerate(t):\n optimizer.zero_grad()\n \n images = sample['image'].to(args.device)\n targets = sample['target'].to(args.device)\n \n embeds, outputs = model.embed_and_classify(images)\n loss = criterion(outputs, targets)\n \n loss.backward()\n optimizer.step()\n \n if scheduler:\n scheduler.step()\n \n losses.append(loss.item())\n targets_all.extend(targets.cpu().numpy())\n outputs_all.extend(torch.sigmoid(outputs).detach().cpu().numpy())\n\n score = np.mean(targets_all == np.argmax(outputs_all, axis=1))\n desc = f\"Epoch {epoch}/{args.epochs} - Train loss:{loss:0.4f}, score: {score:0.4f}\"\n t.set_description(desc)\n \n return np.mean(losses), score\n\n\ndef test_closest_match(base_df, base_embeds, valid_targets, valid_embeds, model, distance_func, closest, n_matches=5):\n distance_matrix = get_distance_matrix(valid_embeds, base_embeds, distance_func)\n\n preds = []\n N_val = len(valid_embeds)\n for i in tqdm(range(N_val), total=N_val, desc=\"Getting closest match\"):\n tmp_df = base_df.copy()\n tmp_df[\"distance\"] = distance_matrix[i]\n tmp_df = tmp_df.sort_values(by=[\"distance\", \"hotel_id\"], ascending=closest).reset_index(drop=True)\n preds.extend([tmp_df[\"hotel_id_code\"].unique()[:n_matches]])\n\n y = np.repeat([valid_targets], repeats=n_matches, axis=0).T\n preds = np.array(preds)\n acc_top_1 = (preds[:, 0] == valid_targets).mean()\n acc_top_5 = (preds == y).any(axis=1).mean()\n print(f\"Accuracy: {acc_top_1:0.4f}, top 5 accuracy: {acc_top_5:0.4f}\")\n return preds, distance_matrix\n\n\ndef test(base_loader, valid_loader, model, distance_func, closest):\n base_targets, base_embeds = get_embeds(base_loader, model, \"Generating embeds for train\")\n valid_targets, valid_embeds = get_embeds(valid_loader, model, \"Generating embeds for test\")\n val_preds, distance_matrix = test_closest_match(base_loader.dataset.data, base_embeds, valid_targets, valid_embeds, model, distance_func, closest)\n\n return base_embeds, valid_embeds, base_targets, valid_targets, val_preds, distance_matrix","metadata":{"id":"SntLH82s2_au","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data","metadata":{"id":"F2xgmwBW4LjC"}},{"cell_type":"code","source":"def sample_data(n_hotels, min_images, max_images):\n data_df = pd.read_csv(PROJECT_FOLDER + \"train.csv\", parse_dates=[\"timestamp\"])\n sample_df = data_df.groupby(\"hotel_id\").filter(lambda x: (x[\"image\"].nunique() > min_images) & (x[\"image\"].nunique() < max_images))\n sample_df[\"hotel_id_code\"] = sample_df[\"hotel_id\"].astype('category').cat.codes.values.astype(np.int64)\n sample_df = sample_df[sample_df[\"hotel_id_code\"] < n_hotels]\n\n print(f\"Subsample with {len(sample_df.hotel_id.unique())} hotels out of {len(data_df.hotel_id.unique())}\" + \n f\" with total {len(sample_df)} images ({len(sample_df) / len(data_df) * 100:0.2f} %)\")\n \n return sample_df","metadata":{"id":"JBkHrXYy2_av","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FOR TESTING DIFFERENT SETTING\n# data_df = sample_data(1000, 15, 50)\n\n# FOR FINAL TRAINING\ndata_df = pd.read_csv(PROJECT_FOLDER + \"train.csv\", parse_dates=[\"timestamp\"])\ndata_df[\"hotel_id_code\"] = data_df[\"hotel_id\"].astype('category').cat.codes.values.astype(np.int64)\n\nfig = go.Figure()\nfig.add_trace(go.Histogram(x=data_df[\"hotel_id_code\"]))\nfig.update_xaxes(type=\"category\")\nfig.show()","metadata":{"id":"Sn6HrWKQ2_aw","outputId":"6ebc2628-8451-4e27-b53b-e46c04f32783","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_and_validate(args, data_df):\n model_name = f\"classification-model-{args.backbone_name}-{IMG_SIZE}x{IMG_SIZE}-{args.embed_size}embeds-{args.n_classes}hotels\"\n print(model_name)\n\n seed_everything(seed=SEED)\n\n val_df = data_df.groupby(\"hotel_id\").sample(args.val_samples, random_state=SEED)\n train_df = data_df[~data_df[\"image\"].isin(val_df[\"image\"])]\n\n train_dataset = HotelTrainDataset(train_df, train_transform, data_path=DATA_FOLDER)\n train_loader = DataLoader(train_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=True, drop_last=True)\n base_dataset = HotelTrainDataset(train_df, val_transform, data_path=DATA_FOLDER)\n base_loader = DataLoader(base_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=False)\n val_dataset = HotelTrainDataset(val_df, val_transform, data_path=DATA_FOLDER)\n valid_loader = DataLoader(val_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=False)\n\n print(f\"Base: {len(base_dataset)}\\nValidation: {len(val_dataset)}\")\n\n model = EmbeddingNet(args.n_classes, args.embed_size, args.backbone_name)\n model = model.to(args.device)\n\n distance = distances.CosineSimilarity()\n criterion = nn.CrossEntropyLoss()\n optimizer = Lookahead(torch.optim.AdamW(model.parameters(), lr=args.lr), k=3)\n\n scheduler = torch.optim.lr_scheduler.OneCycleLR(\n optimizer,\n max_lr=args.lr,\n epochs=args.epochs,\n steps_per_epoch=len(train_loader),\n div_factor=10,\n final_div_factor=1,\n pct_start=0.1,\n anneal_strategy=\"cos\",\n )\n \n start_epoch = 1\n if args.continue_from_checkpoint:\n model, scheduler, _, last_epoch = load_checkpoint(model, scheduler, None, model_name)\n iterate_loader(train_loader, last_epoch)\n start_epoch = start_epoch + last_epoch\n\n torch.cuda.empty_cache()\n\n for epoch in range(start_epoch, args.epochs+1):\n train_loss, train_score = train_epoch(args, model, train_loader, criterion, optimizer, scheduler, epoch)\n save_checkpoint(model, scheduler, optimizer, epoch, model_name, train_loss, train_score)\n if (epoch == 1):\n _ = test(base_loader, valid_loader, model, distance, closest=False)\n\n base_embeds, valid_embeds, base_targets, valid_targets, val_preds, distance_matrix = test(base_loader, valid_loader, model, distance, closest=False)\n \n # output = {\"base_embeds\": base_embeds,\n # \"valid_embeds\": valid_embeds,\n # \"base_targets\": base_targets,\n # \"valid_targets\": valid_targets,\n # \"val_preds\": val_preds,\n # \"distance_matrix\": distance_matrix,\n # \"train_df\" : train_df,\n # \"valid_df\": val_df,\n # }\n\n # torch.save(output, f\"{OUTPUT_FOLDER}output-{model_name}.pt\")","metadata":{"id":"3aEmY6K7KY3H"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and evaluate","metadata":{"id":"EMVYKwZ64zUN"}},{"cell_type":"code","source":"# %%time \n\n# class args:\n# epochs = 9\n# lr = 1e-3\n# batch_size = 32\n# num_workers = 2\n# embed_size = 4096\n# val_samples = 1\n# continue_from_checkpoint = False\n# backbone_name = \"efficientnet_b1\"\n# n_classes = data_df[\"hotel_id_code\"].nunique()\n# device = ('cuda' if torch.cuda.is_available() else 'cpu')\n\n# train_and_validate(args, data_df)","metadata":{"id":"YONzJBtG2_a0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time \n\n# class args:\n# epochs = 9\n# lr = 1e-3\n# batch_size = 16\n# num_workers = 2\n# embed_size = 4096\n# val_samples = 1\n# continue_from_checkpoint = False\n# backbone_name = \"eca_nfnet_l0\"\n# n_classes = data_df[\"hotel_id_code\"].nunique()\n# device = ('cuda' if torch.cuda.is_available() else 'cpu')\n\n# train_and_validate(args, data_df)\n\n\n# RESULTS\n# Epoch 1/9 - Train loss:8.9233, score: 0.0529: 100%|██████████| 3741/3741 [2:37:22<00:00, 2.52s/it]\n# Generating embeds for train: 100%|██████████| 3742/3742 [31:23<00:00, 1.99it/s]\n# Generating embeds for test: 100%|██████████| 324/324 [02:44<00:00, 1.97it/s]\n# 100%|██████████| 8/8 [01:01<00:00, 7.70s/it]\n# Getting closest match: 100%|██████████| 7770/7770 [07:22<00:00, 17.57it/s]\n# 0%| | 0/3741 [00:00