{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup\nThis notebook is intended to run on colab, so some things are commented out to make it work on kaggle.","metadata":{"id":"DAY5rHgTm7e8"}},{"cell_type":"code","source":"# !nvidia-smi","metadata":{"id":"l1qCYqQ0KH1p","outputId":"2a24301f-5972-4e7f-a350-0d64601c3775","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from google.colab import drive\n# drive.mount('/gdrive')\n# %cd /gdrive","metadata":{"id":"dtXBll8AfLHv","outputId":"9959dda5-bc86-46c3-c6bf-19124a660c7d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install git+https://github.com/rwightman/pytorch-image-models\n!pip install albumentations -U\n# !pip install imgaug -U","metadata":{"papermill":{"duration":22.050076,"end_time":"2021-04-17T11:04:51.928845","exception":false,"start_time":"2021-04-17T11:04:29.878769","status":"completed"},"tags":[],"id":"alleged-legislation","outputId":"2abe0aa6-4c55-4eb7-9279-0e214d2e92e0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{"id":"cZoSOL9Qm-Yr"}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport random\nimport os\nimport math\nimport time","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.030593,"end_time":"2021-04-17T11:04:51.983376","exception":false,"start_time":"2021-04-17T11:04:51.952783","status":"completed"},"tags":[],"id":"expired-matter","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.utils import class_weight\nfrom PIL import Image as pil_image\nfrom tqdm import tqdm\nimport scipy\n\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go","metadata":{"papermill":{"duration":4.287352,"end_time":"2021-04-17T11:04:56.353165","exception":false,"start_time":"2021-04-17T11:04:52.065813","status":"completed"},"tags":[],"id":"extreme-problem","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\n\nimport timm\nfrom timm.optim import Lookahead, RAdam","metadata":{"papermill":{"duration":1.641769,"end_time":"2021-04-17T11:04:58.018871","exception":false,"start_time":"2021-04-17T11:04:56.377102","status":"completed"},"tags":[],"id":"angry-domain","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Global","metadata":{"id":"0B00pe7mnBTj"}},{"cell_type":"code","source":"IMG_SIZE = 512\nSEED = 42\nPROJECT_FOLDER = \"../input/hotel-id-2021-fgvc8/\"\nDATA_FOLDER = \"../input/hotelid-images-512x512-padded/\"\nOUTPUT_FOLDER = \"./\"\n\n# PROJECT_FOLDER = \"/gdrive/MyDrive/Projects/Hotel-ID/\"\n# DATA_FOLDER = \"/home/data/\"\n# OUTPUT_FOLDER = PROJECT_FOLDER + \"output/\"","metadata":{"id":"JPRRC5jpF4nx","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !mkdir {DATA_FOLDER}\n# !unzip -qq {PROJECT_FOLDER}data/train-{IMG_SIZE}x{IMG_SIZE}.zip -d /home/data/","metadata":{"id":"nPXawSRWKbeO","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.listdir(PROJECT_FOLDER))\nprint(len(os.listdir(DATA_FOLDER)))","metadata":{"id":"PZvmFng7ctO3","outputId":"59062bb2-fe5f-4ede-940e-29d38c86e20d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions - seed and metric calculator","metadata":{"id":"9p7EE95ZnNpK"}},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"papermill":{"duration":0.031291,"end_time":"2021-04-17T11:04:58.424933","exception":false,"start_time":"2021-04-17T11:04:58.393642","status":"completed"},"tags":[],"id":"eastern-content","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset and transformations","metadata":{"id":"xaJKvvuKnW4k"}},{"cell_type":"code","source":"import albumentations as A\nimport albumentations.pytorch as APT\nimport cv2 \n\ntrain_transform = A.Compose([\n    # A.Resize(IMG_SIZE, IMG_SIZE),\n    # A.CLAHE(p=1), \n    \n    A.HorizontalFlip(p=0.75),\n    A.VerticalFlip(p=0.25),\n    A.ShiftScaleRotate(p=0.5, border_mode=cv2.BORDER_CONSTANT),\n    A.OpticalDistortion(p=0.25),\n    A.IAAPerspective(p=0.25),\n    A.CoarseDropout(p=0.5),\n\n    A.RandomBrightness(p=0.75),\n    A.ToFloat(),\n    APT.transforms.ToTensor(),\n])\n\n\nvalid_transform = A.Compose([\n    # A.Resize(IMG_SIZE, IMG_SIZE),\n    # A.CLAHE(p=1),\n    A.ToFloat(),\n    APT.transforms.ToTensor(),\n])","metadata":{"papermill":{"duration":0.033385,"end_time":"2021-04-17T11:04:58.538926","exception":false,"start_time":"2021-04-17T11:04:58.505541","status":"completed"},"tags":[],"id":"revolutionary-membership","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HotelTrainDataset:\n    def __init__(self, data, transform=None, data_path=\"train_images/\"):\n        self.data = data\n        self.data_path = data_path\n        self.transform = transform\n        self.fake_load = False\n\n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        record = self.data.iloc[idx]\n        image_path = self.data_path + record[\"image\"]\n\n        if self.fake_load:\n            image = np.random.randint(0, 255, (32, 32, 3)).astype(np.uint8)\n        else:\n            image = np.array(pil_image.open(image_path)).astype(np.uint8)\n\n        if self.transform:\n            transformed = self.transform(image=image)\n        \n        return {\n            \"image\" : transformed[\"image\"],\n            \"target\" : record['hotel_id_code'],\n        }","metadata":{"papermill":{"duration":0.032811,"end_time":"2021-04-17T11:04:58.595928","exception":false,"start_time":"2021-04-17T11:04:58.563117","status":"completed"},"tags":[],"id":"found-mouth","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"id":"NMDM4PwPnced"}},{"cell_type":"code","source":"# source: https://github.com/ronghuaiyang/arcface-pytorch/blob/master/models/metrics.py\nclass ArcMarginProduct(nn.Module):\n    r\"\"\"Implement of large margin arc distance: :\n        Args:\n            in_features: size of each input sample\n            out_features: size of each output sample\n            s: norm of input feature\n            m: margin\n            cos(theta + m)\n        \"\"\"\n    def __init__(self, in_features, out_features, s=30.0, m=0.50, easy_margin=False):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        sine = torch.sqrt((1.0 - torch.pow(cosine, 2)).clamp(0, 1))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device='cuda')\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)  # you can use torch.where if your torch.__version__ is 0.4\n        output *= self.s\n\n        return output\n\nclass HotelIdModel(nn.Module):\n    def __init__(self, out_features, embed_size=256, backbone_name=\"efficientnet_b3\"):\n        super(HotelIdModel, self).__init__()\n\n        self.embed_size = embed_size\n        self.backbone = timm.create_model(backbone_name, pretrained=True)\n        in_features = self.backbone.get_classifier().in_features\n\n        fc_name, _ = list(self.backbone.named_modules())[-1]\n        if fc_name == 'classifier':\n            self.backbone.classifier = nn.Identity()\n        elif fc_name == 'head.fc':\n            self.backbone.head.fc = nn.Identity()\n        elif fc_name == 'fc':\n            self.backbone.fc = nn.Identity()\n        else:\n            raise Exception(\"unknown classifier layer: \" + fc_name)\n\n        self.arc_face = ArcMarginProduct(self.embed_size, out_features, s=30.0, m=0.20, easy_margin=False)\n\n        self.post = nn.Sequential(\n            nn.utils.weight_norm(nn.Linear(in_features, self.embed_size*2), dim=None),\n            nn.BatchNorm1d(self.embed_size*2),\n            nn.Dropout(0.2),\n            nn.utils.weight_norm(nn.Linear(self.embed_size*2, self.embed_size)),\n            nn.BatchNorm1d(self.embed_size),\n        )\n\n        print(f\"Model {backbone_name} ArcMarginProduct - Features: {in_features}, Embeds: {self.embed_size}\")\n        \n    def forward(self, input, targets = None):\n        x = self.backbone(input)\n        x = x.view(x.size(0), -1)\n        x = self.post(x)\n        \n        if targets is not None:\n            logits = self.arc_face(x, targets)\n            return logits\n        \n        return x","metadata":{"id":"GuAfw_a4m3PK","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model helper functions","metadata":{"id":"YMZYKhUSneMY"}},{"cell_type":"code","source":"def get_embeds(loader, model, bar_desc=\"Generating embeds\"):\n    targets_all = []\n    outputs_all = []\n    \n    model.eval()\n    with torch.no_grad():\n        t = tqdm(loader, desc=bar_desc)\n        for i, sample in enumerate(t):\n            input = sample['image'].to(args.device)\n            target = sample['target'].to(args.device)\n            output = model(input)\n\n            targets_all.extend(target.cpu().numpy())\n            outputs_all.extend(output.detach().cpu().numpy())\n            \n    return targets_all, outputs_all","metadata":{"papermill":{"duration":0.032565,"end_time":"2021-04-17T11:04:58.652672","exception":false,"start_time":"2021-04-17T11:04:58.620107","status":"completed"},"tags":[],"id":"massive-makeup","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics.pairwise import cosine_similarity\n    \ndef get_distance_matrix(embeds, base_embeds):\n    distance_matrix = []\n    embeds_dataset = torch.utils.data.TensorDataset(torch.Tensor(embeds))\n    embeds_dataloader = DataLoader(embeds_dataset, num_workers=2, batch_size=1024, shuffle=False)\n    \n    t = tqdm(embeds_dataloader)\n    for i, sample in enumerate(t): \n        distances = cosine_similarity(sample[0].numpy(), base_embeds)\n        distance_matrix.extend(distances)\n        \n    return np.array(distance_matrix)","metadata":{"id":"0CIJPX2mGzZw","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_checkpoint(model, scheduler, optimizer, epoch, name, loss=None, score=None):\n    checkpoint = {\"epoch\": epoch,\n                  \"model\": model.state_dict(),\n                  \"scheduler\": scheduler.state_dict(),\n                  \"optimizer\": optimizer.state_dict(),\n                  \"loss\": loss,\n                  \"score\": score,\n                  }\n\n    torch.save(checkpoint, f\"{OUTPUT_FOLDER}checkpoint-{name}.pt\")\n\n\ndef load_checkpoint(model, scheduler, optimizer, name):\n    checkpoint = torch.load(f\"{OUTPUT_FOLDER}checkpoint-{name}.pt\")\n\n    model.load_state_dict(checkpoint[\"model\"])\n    scheduler.load_state_dict(checkpoint[\"scheduler\"])\n    # optimizer.load_state_dict(checkpoint[\"optimizer\"])\n\n    return model, scheduler, optimizer, checkpoint[\"epoch\"]","metadata":{"id":"T85MXS1lHUKI","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def iterate_loader(loader, epochs):\n    loader.dataset.fake_load = True\n    with torch.no_grad():\n        for i in range(epochs):\n            t = tqdm(loader, desc=f\"Iterating loader {i+1}/{epochs}\")\n            for j, sample in enumerate(t):\n                images = sample['image']\n                targets = sample['target']\n\n    loader.dataset.fake_load = False","metadata":{"id":"w8y9MjFEuFPU","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_epoch(args, model, loader, criterion, optimizer, scheduler, epoch):\n    losses = []\n    targets_all = []\n    outputs_all = []\n    \n    model.train()\n    t = tqdm(loader)\n    \n    for i, sample in enumerate(t):\n        optimizer.zero_grad()\n        \n        input = sample['image'].to(args.device)\n        target = sample['target'].to(args.device)\n        \n        output = model(input, target)\n        loss = criterion(output, target)\n        \n        loss.backward()\n        optimizer.step()\n\n        if scheduler is not None:\n            scheduler.step()\n        \n        losses.append(loss.item())\n        targets_all.extend(target.cpu().numpy())\n        outputs_all.extend(torch.sigmoid(output).detach().cpu().numpy())\n        \n        score = accuracy_score(targets_all, np.argmax(outputs_all, axis=1))\n        t.set_description(f\"Epoch {epoch}/{args.epochs} - Train loss:{loss:0.4f}, score: {score:0.4f}\")\n        \n    return np.mean(losses), score\n        \n\ndef find_closest_match(base_df, distance_matrix, n_matches=5):\n    preds = []\n    N_dist = len(distance_matrix)\n    for i in tqdm(range(N_dist), total=N_dist, desc=\"Getting closest match\"):\n        tmp_df = base_df.copy()\n        tmp_df[\"distance\"] = distance_matrix[i]\n        tmp_df = tmp_df.sort_values(by=[\"distance\", \"hotel_id\"], ascending=False).reset_index(drop=True)\n        preds.extend([tmp_df[\"hotel_id_code\"].unique()[:n_matches]])\n    \n    preds = np.array(preds)\n    return preds\n\n\ndef calc_metric(y_true, y_pred, n_matches=5):\n    y = np.repeat([y_true], repeats=n_matches, axis=0).T\n    acc_top_1 = (y_pred[:, 0] == y_true).mean()\n    acc_top_5 = (y_pred == y).any(axis=1).mean()\n    print(f\"Accuracy: {acc_top_1:0.4f}, top 5 accuracy: {acc_top_5:0.4f}\")\n    return acc_top_1, acc_top_5\n\n\ndef test(base_loader, valid_loader, model):\n    base_targets, base_embeds = get_embeds(base_loader, model, \"Generating embeds for train\")\n    valid_targets, valid_embeds = get_embeds(valid_loader, model, \"Generating embeds for test\")\n    distance_matrix = get_distance_matrix(valid_embeds, base_embeds)\n    val_preds = find_closest_match(base_loader.dataset.data, distance_matrix)\n    calc_metric(valid_targets, val_preds)\n    return base_embeds, valid_embeds, valid_targets, val_preds, distance_matrix\n\n\ndef test_closest_match_tta(args, base_loader, valid_df, tta_transforms, model):\n    base_targets, base_embeds = get_embeds(base_loader, model, \"Generating embeds for train\")\n    distance_matrix = None\n\n    for key in tta_transforms:\n        valid_dataset = HotelTrainDataset(valid_df, tta_transforms[key], data_path=DATA_FOLDER)\n        valid_loader = DataLoader(valid_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=False)\n        valid_targets, valid_embeds = get_embeds(valid_loader, model, f\"Generating embeds for test {key}\")\n        \n        distances = get_distance_matrix(valid_embeds, base_embeds)\n\n        if distance_matrix is None:\n            distance_matrix = distances\n        else:\n            distance_matrix = np.min(np.dstack((distance_matrix, distances)), axis = 2)\n    \n    val_preds = find_closest_match(base_loader.dataset.data, distance_matrix)\n    calc_metric(valid_targets, val_preds)","metadata":{"papermill":{"duration":0.036549,"end_time":"2021-04-17T11:04:58.713386","exception":false,"start_time":"2021-04-17T11:04:58.676837","status":"completed"},"tags":[],"id":"flying-bottle","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data","metadata":{"id":"AwShW1wXniD6"}},{"cell_type":"code","source":"def sample_data(n_hotels, min_images, max_images):\n    data_df = pd.read_csv(PROJECT_FOLDER + \"train.csv\", parse_dates=[\"timestamp\"])\n    sample_df = data_df.groupby(\"hotel_id\").filter(lambda x: (x[\"image\"].nunique() > min_images) & (x[\"image\"].nunique() < max_images))\n    sample_df[\"hotel_id_code\"] = sample_df[\"hotel_id\"].astype('category').cat.codes.values.astype(np.int64)\n    sample_df = sample_df[sample_df[\"hotel_id_code\"] < n_hotels]\n\n    print(f\"Subsample with {len(sample_df.hotel_id.unique())} hotels out of {len(data_df.hotel_id.unique())} \" + \n          f\"with total {len(sample_df)} images ({len(sample_df) / len(data_df) * 100:0.2f} %)\")\n    \n    return sample_df","metadata":{"id":"y4sv-2bmijiP","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FOR TESTING DIFFERENT SETTING\n# data_df = sample_data(1000, 15, 50)\n\n# FOR FINAL TRAINING\ndata_df = pd.read_csv(PROJECT_FOLDER + \"train.csv\", parse_dates=[\"timestamp\"])\ndata_df[\"hotel_id_code\"] = data_df[\"hotel_id\"].astype('category').cat.codes.values.astype(np.int64)\n\nfig = go.Figure()\nfig.add_trace(go.Histogram(x=data_df[\"hotel_id_code\"]))\nfig.update_xaxes(type=\"category\")\nfig.show()","metadata":{"papermill":{"duration":2.790179,"end_time":"2021-04-17T11:05:01.702988","exception":false,"start_time":"2021-04-17T11:04:58.912809","status":"completed"},"tags":[],"id":"discrete-right","outputId":"ef000115-70a9-419d-8882-83e228ebc131","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and evaluate","metadata":{"id":"5JPdD2bpnniP"}},{"cell_type":"code","source":"def train_and_validate(args, data_df):\n    model_name = f\"arcmargin-model-{args.backbone_name}-{IMG_SIZE}x{IMG_SIZE}-{args.embed_size}embeds-{args.n_classes}hotels\"\n    print(model_name)\n    # SEED and split\n    seed_everything(seed=SEED)\n    valid_df = data_df.groupby(\"hotel_id\").sample(args.val_samples, random_state=SEED)\n    train_df = data_df[~data_df[\"image\"].isin(valid_df[\"image\"])]\n\n    # create model\n    model = HotelIdModel(args.n_classes, args.embed_size, args.backbone_name)\n    model = model.to(args.device)\n\n    # train data loader\n    train_dataset = HotelTrainDataset(train_df, train_transform, data_path=DATA_FOLDER)\n    train_loader = DataLoader(train_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=True, pin_memory=False)\n    # train without augmentations to generate base embeddings\n    base_dataset = HotelTrainDataset(train_df, valid_transform, data_path=DATA_FOLDER)\n    base_loader = DataLoader(base_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=False)\n    # valid loader\n    valid_dataset = HotelTrainDataset(valid_df, valid_transform, data_path=DATA_FOLDER)\n    valid_loader = DataLoader(valid_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=False)\n\n    print(f\"Base: {len(base_dataset)}\\nValidation: {len(valid_dataset)}\")\n\n    criterion = nn.CrossEntropyLoss()\n    optimizer = Lookahead(torch.optim.AdamW(model.parameters(), lr=args.lr), k=3)\n    scheduler = torch.optim.lr_scheduler.OneCycleLR(\n                    optimizer,\n                    max_lr=args.lr,\n                    epochs=args.epochs,\n                    steps_per_epoch=len(train_loader),\n                    div_factor=10,\n                    final_div_factor=1,\n                    pct_start=0.1,\n                    anneal_strategy=\"cos\",\n                )\n    \n    start_epoch = 1\n\n    if args.continue_from_checkpoint:\n        model, scheduler, optimizer, last_epoch = load_checkpoint(model, scheduler, optimizer, model_name)\n        iterate_loader(train_loader, last_epoch)\n        start_epoch = start_epoch + last_epoch\n\n    torch.cuda.empty_cache()\n\n    for epoch in range(start_epoch, args.epochs +1):\n        train_loss, train_score = train_epoch(args, model, train_loader, criterion, optimizer, scheduler, epoch)\n        save_checkpoint(model, scheduler, optimizer, epoch, model_name, train_loss, train_score)\n        if (epoch == 1): #  or (epoch % 3) == 0:\n            base_embeds, valid_embeds, valid_targets, val_preds, distance_matrix = test(base_loader, valid_loader, model)\n\n    base_embeds, valid_embeds, valid_targets, val_preds, distance_matrix = test(base_loader, valid_loader, model)\n\n    # output = {\"base_embeds\": base_embeds,\n    #           \"valid_embeds\": valid_embeds,\n    #           \"valid_targets\": valid_targets,\n    #           \"val_preds\": val_preds,\n    #           \"distance_matrix\": distance_matrix,\n    #           \"train_df\" : train_df,\n    #           \"valid_df\": valid_df,\n    #           }\n\n    # torch.save(output, f\"{OUTPUT_FOLDER}output-{model_name}.pt\")","metadata":{"id":"SYoTOYmYdjwy","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time \n\n# class args:\n#     epochs = 6\n#     lr = 1e-3\n#     batch_size = 16\n#     num_workers = 2\n#     embed_size = 4096\n#     val_samples = 1\n#     backbone_name=\"eca_nfnet_l0\"\n#     n_classes = data_df[\"hotel_id_code\"].nunique()\n#     device = ('cuda' if torch.cuda.is_available() else 'cpu')\n#     continue_from_checkpoint = False\n\n# train_and_validate(args, data_df)","metadata":{"papermill":{"duration":0.59707,"end_time":"2021-04-17T11:05:02.330381","exception":false,"start_time":"2021-04-17T11:05:01.733311","status":"completed"},"tags":[],"id":"appointed-machinery","outputId":"6a30d489-8e9a-48bd-eaa8-bcba2d8344eb","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time \n\n# class args:\n#     epochs = 9\n#     lr = 1e-3\n#     batch_size = 16\n#     num_workers = 2\n#     embed_size = 4096\n#     val_samples = 1\n#     continue_from_checkpoint = True\n#     backbone_name=\"efficientnet_b1\"\n#     n_classes = data_df[\"hotel_id_code\"].nunique()\n#     device = ('cuda' if torch.cuda.is_available() else 'cpu')\n\n# train_and_validate(args, data_df)\n\n\n# RESULTS\n# Iterating loader 1/4: 100%|██████████| 5612/5612 [20:05<00:00,  4.66it/s]\n# ...\n# Iterating loader 4/4: 100%|██████████| 5612/5612 [19:29<00:00,  4.80it/s]\n# Epoch 5/9 - Train loss:2.5940, score: 0.2919: 100%|██████████| 5612/5612 [2:00:37<00:00,  1.29s/it]\n# Epoch 6/9 - Train loss:4.4544, score: 0.3838: 100%|██████████| 5612/5612 [2:00:07<00:00,  1.28s/it]\n# Epoch 7/9 - Train loss:2.8829, score: 0.4948:  27%|██▋       | 1495/5612 [17:03<1:01:32,  1.11it/s]\n# Iterating loader 1/6: 100%|██████████| 5612/5612 [01:21<00:00, 68.73it/s]\n# ...\n# Iterating loader 6/6: 100%|██████████| 5612/5612 [01:22<00:00, 68.03it/s]\n# Epoch 7/9 - Train loss:3.1958, score: 0.4358:  90%|████████▉ | 5042/5612 [1:58:12<20:23,  2.15s/it]\n# Iterating loader 1/6: 100%|██████████| 5612/5612 [01:18<00:00, 71.15it/s]\n# ...\n# Iterating loader 6/6: 100%|██████████| 5612/5612 [01:17<00:00, 72.06it/s]\n# Epoch 7/9 - Train loss:4.7988, score: 0.4360: 100%|██████████| 5612/5612 [2:22:07<00:00,  1.52s/it]\n# Epoch 8/9 - Train loss:1.2727, score: 0.5257:  88%|████████▊ | 4961/5612 [2:02:48<27:56,  2.57s/it]\n# Iterating loader 1/7: 100%|██████████| 5612/5612 [01:23<00:00, 66.99it/s]\n# ...\n# Iterating loader 7/7: 100%|██████████| 5612/5612 [01:19<00:00, 70.49it/s]\n# Epoch 8/9 - Train loss:3.3698, score: 0.4863: 100%|██████████| 5612/5612 [2:22:22<00:00,  1.52s/it]\n# Epoch 9/9 - Train loss:3.6118, score: 0.5580:  62%|██████▏   | 3483/5612 [1:09:32<1:00:59,  1.72s/it]\n# Iterating loader 1/8: 100%|██████████| 5612/5612 [01:19<00:00, 70.66it/s]\n# ...\n# Iterating loader 8/8: 100%|██████████| 5612/5612 [01:20<00:00, 69.83it/s]\n# Epoch 9/9 - Train loss:4.3291, score: 0.5247: 100%|██████████| 5612/5612 [2:24:09<00:00,  1.54s/it]\n# Generating embeds for train: 100%|██████████| 5612/5612 [17:36<00:00,  5.31it/s]\n# Generating embeds for test: 100%|██████████| 486/486 [01:41<00:00,  4.80it/s]\n# 100%|██████████| 8/8 [01:23<00:00, 10.48s/it]\n# Getting closest match: 100%|██████████| 7770/7770 [09:35<00:00, 13.50it/s]\n# Accuracy: 0.6802, top 5 accuracy: 0.7979","metadata":{"id":"KDeMdhh-vxjM","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Results","metadata":{"id":"t7WB-7sPnrMA"}},{"cell_type":"markdown","source":"\n|Size|Hotels|Epochs|LR|Model|Embeds|Optimizer|Scheduler|Acc|Acc 5| PL | Comment |\n| --- | --- | --- | --- | --- | --- | --- | --- |--- | --- | --- | --- |\n|512|7770|6|1e-3|efficientnet_b1|4096|Look3AdamW|OneCycle-10-1|0.0682|0.7979||\n|512|7770|6|1e-3|eca_nfnet_l0|1024|Look3AdamW|OneCycle-10-1|0.6368|0.7604|0.630|\n|512|7770|6|1e-3|efficientnet_b1|1024|Look3AdamW|OneCycle-10-1|0.5871|0.7151|0.592|\n|512|500|9|1e-3|eca_nfnet_l0|256|Look3AdamW|OneCycle-10-1|0.8040|0.8940|\n|512|500|6|1e-3|eca_nfnet_l0|256|Look3AdamW|OneCycle-10-1|0.8440|0.9220||cos-m=0.2|\n|512|500|6|1e-3|efficientnet_b1|256|Look3AdamW|OneCycle-10-1|0.7980|0.9060||cos-m=0.2|\n|512|500|2x6|1e-3|efficientnet_b1|256|Look3AdamW|OneCycle-10-1|0.8240|0.9140||cos-m=0.2|\n|512|500|9|1e-3|ecaresnet50d_pruned|256|Look3AdamW|OneCycle-10-1|0.7780|0.8780|\n|512|500|9|1e-3|efficientnet_b0|256|Look3AdamW|OneCycle-10-1|0.7680|0.8720|\n|512|500|9|1e-3|efficientnet_b1|256|Look3AdamW|OneCycle-10-1|0.7780|0.8780|\n|512|500|9|1e-3|efficientnet_b1|1024|Look3AdamW|OneCycle-10-1|0.8040|0.8820|\n|256(6x)+512(3x)|500|9|1e-3|efficientnet_b1|256|Look3AdamW|OneCycle-10-1|0.7580|0.8660|\n|512|500|9|1e-3|efficientnet_b1|256|AdamW|OneCycle-10-1|0.7720|0.8660|\n|512|500|9|1e-3|efficientnet_b3|256|Look3AdamW|OneCycle-10-1|0.7520|0.8440|\n|512|500|9|1e-3|dla102|256|Look3AdamW|OneCycle-10-1|super slow|\n|256|500|9|1e-3|eca_nfnet_l0|256|Look3AdamW|OneCycle-10-10|0.7500|0.8500|\n|256|500|6|1e-3|eca_nfnet_l0|256|Look3AdamW|OneCycle-10-1|0.7800|0.8740||cos-m=0.5|\n|256|500|6|1e-3|eca_nfnet_l0|256|Look3AdamW|OneCycle-10-1|0.7880|0.8960||cos-m=0.2|\n|256|500|6|1e-3|eca_nfnet_l0|256|Look3AdamW|OneCycle-10-1|0.7580|0.8740||cos-m=0.2-norm embeds|\n|256|500|9|1e-3|eca_nfnet_l0|256|Look3AdamW|OneCycle-10-1|0.7540|0.8580|\n|256|500|9|1e-3|eca_nfnet_l0|512|Look3AdamW|OneCycle-10-1|0.7440|0.8420|\n|256|500|6|1e-3|eca_nfnet_l0|512|Look3AdamW|OneCycle-10-1|0.7380|0.8460|\n|256|500|6|1e-2|eca_nfnet_l0|512|Look3AdamW|OneCycle-10-1||\n|256|500|9|1e-3|eca_nfnet_l0+Mish|256|Look3AdamW|OneCycle-10-1|0.7500|0.8300|\n|256|500|9|1e-3|eca_nfnet_l0|256|Look3AdamW|OneCycle-10-0.5|0.7300|0.8500|\n|256|500|9|1e-3|eca_nfnet_l1|256|Look3AdamW|OneCycle-10-1|0.7560|0.8640|\n|256|500|9|1e-3|seresnext26d_32x4d|256|Look3AdamW|OneCycle-10-1|0.6780|0.7820|\n|256|500|9|1e-3|nfnet_f0|256|Look3AdamW|OneCycle-10-1|doesn't converge||\n|256|500|9|1e-3|swsl_resnet18|256|Look3AdamW|OneCycle-10-1|0.6780|0.7280|\n|256|500|9|1e-3|swsl_resnet50|256|Look3AdamW|OneCycle-10-1|0.6400|0.7760|\n|256|500|9|1e-3|efficientnet_b0|256|Look3AdamW|OneCycle-10-1|0.6500|0.8020|\n|256|500|9|1e-3|efficientnet_b0|256|Look3AdamW|OneCycle-10-1|0.6940|0.8360||cos|\n|256|500|9|1e-3|efficientnet_b1|256|Look3AdamW|OneCycle-10-1|0.6540|0.7760|\n|256|500|6|1e-3|efficientnet_b1|256|Look3AdamW|OneCycle-10-1|0.6880|0.8480||cos-m=0.2-norm embeds|\n|256|500|6|1e-3|efficientnet_b1|256|Look3AdamW|OneCycle-10-1|0.668|0.8340||cos-m=0.2|\n|256|500|9|1e-3|efficientnet_b3|256|Look3AdamW|OneCycle-10-1|0.6320|0.7520|\n|256|500|9|1e-3|adv_inception_v3|256|Look3AdamW|OneCycle-10-1|0.5000|0.6700|\n|256|500|9|1e-3|ecaresnet50t|256|Look3AdamW|OneCycle-10-1|0.7180|0.8220|\n|256|500|9|1e-3|ecaresnet50d_pruned|256|Look3AdamW|OneCycle-10-1|0.6980|0.8340|\n|256|500|9|1e-3|ecaresnet101d_pruned|256|Look3AdamW|OneCycle-10-1|0.6680|0.7980|\n|256|500|9|1e-3|ecaresnext50t_32x4d|256|Look3AdamW|OneCycle-10-1|0.1240|0.2480|\n|256|500|9|1e-3|nf_ecaresnet50|256|Look3AdamW|OneCycle-10-1|doesn't converge|\n|256|500|9|1e-3|nf_seresnet50|256|Look3AdamW|OneCycle-10-1|doesn't converge|\n|256|500|9|1e-3|ese_vovnet39b_evos|256|Look3AdamW|OneCycle-10-1|doesn't converge|\n|256|500|9|1e-3|eca_vovnet39b |256|Look3AdamW|OneCycle-10-1|doesn't converge|\n|256|500|9|1e-3|tresnet_m|256|Look3AdamW|OneCycle-10-1|0.7260|0.8120|\n|256|500|9|1e-3|vit_small_resnet26d_224|256|Look3AdamW|OneCycle-10-1|0.6060|0.7460|\n|256|500|9|1e-3|tv_resnet50|256|Look3AdamW|OneCycle-10-1|0.5720|0.7500|\n|256|500|9|1e-3|selecsls42b|256|Look3AdamW|OneCycle-10-1|0.5800|0.7620|\n|256|500|9|1e-3|resnet50|256|Look3AdamW|OneCycle-10-1|0.6360|0.7680|\n|256|500|9|1e-3|botnet50t_224|256|Look3AdamW|OneCycle-10-1|0.0340|0.1100|\n|256|500|9|1e-3|dm_nfnet_f0|256|Look3AdamW|OneCycle-10-1|doesn't converge|\n|256|500|9|1e-3|dla60 |256|Look3AdamW|OneCycle-10-1|0.6580|0.8040|\n|256|500|9|1e-3|densenet121 |256|Look3AdamW|OneCycle-10-1|0.6120|0.7620|\n|256|500|9|1e-3|tf_mixnet_m|256|Look3AdamW|OneCycle-10-1|0.6480|0.7700|\n|256|500|9|1e-3|tf_mixnet_l |256|Look3AdamW|OneCycle-10-1|0.6720|0.7920|\n|256|500|9|1e-3|dla102|256|Look3AdamW|OneCycle-10-1|0.6640|0.8180|\n|256|500|12|1e-3|dla102|256|Look3AdamW|OneCycle-10-1|0.6580|0.8040|\n|256|500|9|1e-3||256|Look3AdamW|OneCycle-10-1|||\n|256|500|9|1e-3||256|Look3AdamW|OneCycle-10-1|||\n\n\n\n \n","metadata":{"id":"TyZDhg4-jwLS"}}]}