{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-28T06:37:01.158127Z","iopub.execute_input":"2022-05-28T06:37:01.159744Z","iopub.status.idle":"2022-05-28T06:37:05.512848Z","shell.execute_reply.started":"2022-05-28T06:37:01.159618Z","shell.execute_reply":"2022-05-28T06:37:05.51178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shared Imports\nimport random\nimport os\nimport pathlib\nfrom typing import Iterator, List, Optional, Tuple\nimport json\n\nimport math\nimport numpy as np\nimport pandas as pd\nimport matplotlib\nimport matplotlib.pyplot as plt\n\n# model imports\n## Pytorch/modell stuff\nimport torch\nimport torch.nn as nn\nfrom torchmetrics import Accuracy\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint, LearningRateMonitor\nfrom pytorch_lightning.loggers import TensorBoardLogger\n\nfrom sklearn.metrics.pairwise import cosine_similarity\n\n# Pre-processing\nimport albumentations as A\nimport albumentations.pytorch as APT\nimport cv2 \nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:05.516735Z","iopub.execute_input":"2022-05-28T06:37:05.517428Z","iopub.status.idle":"2022-05-28T06:37:10.02733Z","shell.execute_reply.started":"2022-05-28T06:37:05.517354Z","shell.execute_reply":"2022-05-28T06:37:10.026284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Globals","metadata":{}},{"cell_type":"code","source":"SEED = 42\n# Wheter to PAD the images\nPAD = True\n# The size of the images\nPATCH = (256, 256)\n# The number of matches to consider\nN_MATCHES = 5\n\n# Set random Seed\npl.seed_everything(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.029458Z","iopub.execute_input":"2022-05-28T06:37:10.029843Z","iopub.status.idle":"2022-05-28T06:37:10.046285Z","shell.execute_reply.started":"2022-05-28T06:37:10.029764Z","shell.execute_reply":"2022-05-28T06:37:10.045115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# directory of the test images\nTEST_DIR = pathlib.Path(\"../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/test_images\")\n# base model\nBASE_MODEL = \"eca_nfnet_l0\"\n# Directory of the model weights and saved embeddings\nMODEL_WEIGHTS_DIR = pathlib.Path(\"../input/experimenten-weights\")\nMODEL_WEIGHTS = pathlib.Path(\"../input/experimenten-weights/logs/lightning_logs/version_0/checkpoints/epoch_0009.step_000025139.val-map_0.4332.last.ckpt\")\nBASE_EMB = MODEL_WEIGHTS_DIR / \"base_image-embeddings.pkl\"","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.049623Z","iopub.execute_input":"2022-05-28T06:37:10.049989Z","iopub.status.idle":"2022-05-28T06:37:10.058057Z","shell.execute_reply.started":"2022-05-28T06:37:10.049942Z","shell.execute_reply":"2022-05-28T06:37:10.056271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading data","metadata":{}},{"cell_type":"code","source":"test_df = pd.DataFrame(data={\"image_id\": os.listdir(TEST_DIR), \"hotel_id\": \"\"}).sort_values(by=\"image_id\")\n\nhotel_id_code_df = pd.read_csv(MODEL_WEIGHTS_DIR / 'hotel_id_code_mapping.csv')\nhotel_id_code_map = hotel_id_code_df.set_index('hotel_code').to_dict()[\"hotel_id\"]\nhotel_id_code_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.060103Z","iopub.execute_input":"2022-05-28T06:37:10.06047Z","iopub.status.idle":"2022-05-28T06:37:10.112558Z","shell.execute_reply.started":"2022-05-28T06:37:10.060424Z","shell.execute_reply":"2022-05-28T06:37:10.111554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.113945Z","iopub.execute_input":"2022-05-28T06:37:10.114248Z","iopub.status.idle":"2022-05-28T06:37:10.128112Z","shell.execute_reply.started":"2022-05-28T06:37:10.114211Z","shell.execute_reply":"2022-05-28T06:37:10.127011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre-process","metadata":{}},{"cell_type":"code","source":"def open_preprocess_img(img_path: pathlib.Path):\n    img = cv2.imread(str(img_path))\n    \n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    if PAD: img = pad(img)\n    \n    return cv2.resize(img, PATCH)\n\ndef pad(img):\n    w, h, c = np.shape(img)\n    const = 0\n        \n    if w == h: return img\n    elif (w - h) % 2 != 0: const = 1\n        \n    if w < h:\n        half_py = (h - w) // 2       \n        return cv2.copyMakeBorder(img, 0, 0, half_py, half_py + const, cv2.BORDER_CONSTANT, value=0)\n    elif h < w:\n        half_px = (w - h) // 2\n        return cv2.copyMakeBorder(img, half_px, half_px + const, 0, 0, cv2.BORDER_CONSTANT, value=0)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.129559Z","iopub.execute_input":"2022-05-28T06:37:10.130833Z","iopub.status.idle":"2022-05-28T06:37:10.142617Z","shell.execute_reply.started":"2022-05-28T06:37:10.130788Z","shell.execute_reply":"2022-05-28T06:37:10.141614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augmentations","metadata":{}},{"cell_type":"code","source":"base_transform = A.Compose([#A.RandomCrop(224,224, p=1),\n    A.ToFloat(),\n    APT.transforms.ToTensorV2(),\n])","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.144538Z","iopub.execute_input":"2022-05-28T06:37:10.145284Z","iopub.status.idle":"2022-05-28T06:37:10.158039Z","shell.execute_reply.started":"2022-05-28T06:37:10.145222Z","shell.execute_reply":"2022-05-28T06:37:10.156858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataloader","metadata":{}},{"cell_type":"code","source":"class ImageDataset(Dataset):\n    def __init__(self,\n                 data: pd.DataFrame,\n                 data_path: pathlib.Path,\n                 transform: Optional = None,\n                ):\n        self.data = data\n        self.data_path = data_path\n        self.transform = transform\n\n    def __len__(self) -> int:\n        return len(self.data)\n\n    def __getitem__(self, idx: int):\n        record = self.data.iloc[idx]\n\n        image_path = self.data_path / record[\"image_id\"]\n        image = np.array(open_preprocess_img(image_path)).astype(np.uint8)\n        \n        if self.transform:\n            transformed = self.transform(image=image)\n            image = transformed[\"image\"]\n            \n        return image","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.159882Z","iopub.execute_input":"2022-05-28T06:37:10.160326Z","iopub.status.idle":"2022-05-28T06:37:10.171746Z","shell.execute_reply.started":"2022-05-28T06:37:10.16028Z","shell.execute_reply":"2022-05-28T06:37:10.170555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class args:\n    num_workers = 2\n    n_classes = hotel_id_code_df[\"hotel_id\"].nunique()\n    device = ('cuda' if torch.cuda.is_available() else 'cpu')\n\ntest_dataset = ImageDataset(test_df, transform=base_transform, data_path=TEST_DIR)\ntest_loader = DataLoader(test_dataset, num_workers=args.num_workers, batch_size=1, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.176052Z","iopub.execute_input":"2022-05-28T06:37:10.176393Z","iopub.status.idle":"2022-05-28T06:37:10.254864Z","shell.execute_reply.started":"2022-05-28T06:37:10.176345Z","shell.execute_reply":"2022-05-28T06:37:10.253939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# source: https://github.com/ronghuaiyang/arcface-pytorch/blob/master/models/metrics.py\nclass ArcMarginProduct(nn.Module):\n    r\"\"\"Implement of large margin arc distance: :\n        Args:\n            in_features: size of each input sample\n            out_features: size of each output sample\n            s: norm of input feature\n            m: margin\n            cos(theta + m)\n        \"\"\"\n    def __init__(self, in_features, out_features, s=30.0, m=0.50, easy_margin=False):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        sine = torch.sqrt((1.0 - torch.pow(cosine, 2)).clamp(0, 1))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device='cuda')\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)  # you can use torch.where if your torch.__version__ is 0.4\n        output *= self.s\n\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.256458Z","iopub.execute_input":"2022-05-28T06:37:10.257234Z","iopub.status.idle":"2022-05-28T06:37:10.272295Z","shell.execute_reply.started":"2022-05-28T06:37:10.257176Z","shell.execute_reply":"2022-05-28T06:37:10.271334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HotelModel(pl.LightningModule):\n    def __init__(self,\n                n_hotels: int,\n                steps_per_epoch: int,\n                n_embeddings: int = 256,\n                base_model = None,\n                pretrained: bool = False,\n                learning_rate: float = 0.003,\n                \n                ):\n        super().__init__()\n        \n        # Hyperparams\n        self.n_embeddings = n_embeddings\n        self.n_hotels = n_hotels\n        self.learning_rate = learning_rate\n        self.steps_per_epoch = steps_per_epoch\n        \n        # Metrics\n        self.loss_fn = nn.CrossEntropyLoss()\n        self.train_acc = Accuracy()\n        self.val_acc = Accuracy()\n        \n        # Model Definition \n        ## Base model\n        self.base_model = timm.create_model(base_model, pretrained=False)        \n        in_features = self.base_model.get_classifier().in_features\n        \n        fc_name, _ = list(self.base_model.named_modules())[-1]\n        if fc_name == 'classifier':\n            self.base_model.classifier = nn.Identity()\n        elif fc_name == 'head.fc':\n            self.base_model.head.fc = nn.Identity()\n        elif fc_name == 'fc':\n            self.base_model.fc = nn.Identity()\n        #else:\n            #raise Exception(\"unknown classifier layer: \" + fc_name)\n        \n        ## Arcface module\n        self.arc_face = ArcMarginProduct(self.n_embeddings, n_hotels, s=30.0, m=0.20, easy_margin=False)\n        \n        ## Top model\n        self.top_model = nn.Sequential(\n            nn.utils.weight_norm(nn.Linear(1000, self.n_embeddings*2), dim=None),\n            nn.BatchNorm1d(self.n_embeddings*2),\n            nn.Dropout(0.2),\n            nn.utils.weight_norm(nn.Linear(self.n_embeddings*2, self.n_embeddings)),\n            nn.BatchNorm1d(self.n_embeddings),\n        )\n        \n        # Save hyper params\n        self.save_hyperparameters()\n\n    def configure_optimizers(self):\n        optimizer = Lookahead(torch.optim.AdamW(self.parameters(), lr=self.learning_rate), k=3)\n        \n        scheduler = torch.optim.lr_scheduler.OneCycleLR(\n                    optimizer,\n                    max_lr=self.learning_rate,\n                    epochs=EPOCHS,\n                    steps_per_epoch=self.steps_per_epoch,\n                    div_factor=10,\n                    final_div_factor=1,\n                    pct_start=0.1,\n                    anneal_strategy=\"cos\",\n                )\n        \n        schedule = {\n            # Required: the scheduler instance.\n            \"scheduler\": scheduler,\n        }\n        return [optimizer], [schedule]\n    \n    def forward(self, x, targets = None):\n        y_hat = self.base_model(x)\n        y_hat = y_hat.view(y_hat.size(0), -1)\n        y_hat = self.top_model(y_hat)\n        \n        if targets is not None:\n            y_hat = self.arc_face(y_hat, targets)\n\n        return y_hat\n\n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        \n        # Forward pass\n        y_hat = self.forward(x, y)\n        loss = self.loss_fn(y_hat, y)\n        self.train_acc(y_hat, y)\n\n        # Store results\n        self.log(\"train_loss\", loss, prog_bar=False)\n        \n        return loss\n    \n    def training_epoch_end(self, train_step_outputs) -> None:\n        # Log metrics\n        self.log(\"train_acc\", self.train_acc, prog_bar=True)\n\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        \n        # Forward pass\n        y_hat = self.forward(x, y)\n        loss = self.loss_fn(y_hat, y)\n        self.val_acc(y_hat, y)\n\n        # Store results\n        self.log(\"val_loss\", loss, prog_bar=False)\n        return y_hat\n        \n    def validation_epoch_end(self, validation_step_outputs) -> None:\n        self.log(\"val_acc\", self.val_acc, prog_bar=True)\n        \n    def predict_step(self, batch, batch_idx):\n        y_hat = self.forward(batch)\n        return y_hat","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.274193Z","iopub.execute_input":"2022-05-28T06:37:10.274743Z","iopub.status.idle":"2022-05-28T06:37:10.302364Z","shell.execute_reply.started":"2022-05-28T06:37:10.274578Z","shell.execute_reply":"2022-05-28T06:37:10.301236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(model_type, backbone_name, checkpoint_path, args):\n    model = HotelModel.load_from_checkpoint(checkpoint_path, map_location='cpu', strict=True)\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.303899Z","iopub.execute_input":"2022-05-28T06:37:10.304281Z","iopub.status.idle":"2022-05-28T06:37:10.316534Z","shell.execute_reply.started":"2022-05-28T06:37:10.304236Z","shell.execute_reply":"2022-05-28T06:37:10.315382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model(\"classification\", \n                  BASE_MODEL,\n                  MODEL_WEIGHTS, \n                  args)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:10.317797Z","iopub.execute_input":"2022-05-28T06:37:10.318079Z","iopub.status.idle":"2022-05-28T06:37:22.877215Z","shell.execute_reply.started":"2022-05-28T06:37:10.318049Z","shell.execute_reply":"2022-05-28T06:37:22.876084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helper Functions","metadata":{}},{"cell_type":"code","source":"def generate_embeddings(args, loader, model, bar_desc=\"Generating embeds\"):\n    outputs_all = []\n    model = model.to(args.device)\n    model.eval()\n    with  torch.no_grad():\n        tq = tqdm(loader, desc=bar_desc)\n        for i, sample in enumerate(tq):\n            input = sample.to(args.device)\n            output = model(input)\n            outputs_all.extend(output.detach().cpu().numpy())\n            \n    return outputs_all","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:22.879103Z","iopub.execute_input":"2022-05-28T06:37:22.879465Z","iopub.status.idle":"2022-05-28T06:37:22.888163Z","shell.execute_reply.started":"2022-05-28T06:37:22.879408Z","shell.execute_reply":"2022-05-28T06:37:22.886049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_matches(query, base_embeds, base_targets, k=N_MATCHES):\n    distance_df = pd.DataFrame(index=np.arange(len(base_targets)), data={\"hotel_id\": base_targets})\n    # calculate cosine distance of query embeds to all base embeds\n    distance_df[\"distance\"] = cosine_similarity([query], list(base_embeds))[0]\n    # sort by distance and hotel_id\n    distance_df = distance_df.sort_values(by=[\"distance\", \"hotel_id\"], ascending=False).reset_index(drop=True)\n    # return first 5 different hotel_id_codes\n    return distance_df[\"hotel_id\"].unique()[:N_MATCHES]\n\n\ndef predict(args, base_embeddings_df, test_loader, model):\n    test_embeds = generate_embeddings(args, test_loader, model, \"Generate test embeddings\")\n    \n    preds = []\n    for query_embeds in tqdm(test_embeds, desc=\"Similarity - match finding\"):\n        tmp = find_matches(query_embeds, \n                           base_embeddings_df[\"embeddings\"].values, \n                           base_embeddings_df[\"hotel_id\"].values)\n        preds.extend([tmp])\n        \n    return preds","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:22.890249Z","iopub.execute_input":"2022-05-28T06:37:22.890662Z","iopub.status.idle":"2022-05-28T06:37:22.90372Z","shell.execute_reply.started":"2022-05-28T06:37:22.890615Z","shell.execute_reply":"2022-05-28T06:37:22.902544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load base embeddings","metadata":{}},{"cell_type":"code","source":"base_embeddings_df = pd.read_pickle(BASE_EMB)\ndisplay(base_embeddings_df.head())","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:22.905649Z","iopub.execute_input":"2022-05-28T06:37:22.906012Z","iopub.status.idle":"2022-05-28T06:37:28.180674Z","shell.execute_reply.started":"2022-05-28T06:37:22.905967Z","shell.execute_reply":"2022-05-28T06:37:28.179671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"preds = predict(args, base_embeddings_df, test_loader, model)\n# transform array of hotel_ids into string\ntest_df[\"hotel_id\"] = [str(list(l)).strip(\"[]\").replace(\",\", \"\") for l in preds]\n\ntest_df.to_csv(\"submission.csv\", index=False)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T06:37:28.182043Z","iopub.execute_input":"2022-05-28T06:37:28.182923Z","iopub.status.idle":"2022-05-28T06:37:40.856483Z","shell.execute_reply.started":"2022-05-28T06:37:28.182877Z","shell.execute_reply":"2022-05-28T06:37:40.855298Z"},"trusted":true},"execution_count":null,"outputs":[]}]}