{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:03.182368Z","iopub.execute_input":"2022-05-23T16:02:03.182909Z","iopub.status.idle":"2022-05-23T16:02:10.236062Z","shell.execute_reply.started":"2022-05-23T16:02:03.18287Z","shell.execute_reply":"2022-05-23T16:02:10.235036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nimport os\nimport math\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.utils import class_weight\nfrom PIL import Image as pil_image\nfrom tqdm import tqdm\nimport scipy\n\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:10.238437Z","iopub.execute_input":"2022-05-23T16:02:10.238807Z","iopub.status.idle":"2022-05-23T16:02:13.285033Z","shell.execute_reply.started":"2022-05-23T16:02:10.238766Z","shell.execute_reply":"2022-05-23T16:02:13.284211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from timm.optim import Lookahead, RAdam","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:13.286438Z","iopub.execute_input":"2022-05-23T16:02:13.28676Z","iopub.status.idle":"2022-05-23T16:02:13.361989Z","shell.execute_reply.started":"2022-05-23T16:02:13.286724Z","shell.execute_reply":"2022-05-23T16:02:13.361375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\nN_MATCHES = 5\nIMG_SIZE = 512\n\nDATA_FOLDER = \"../input/hotelid-2022-train-images-512x512/\"\nPROJECT_FOLDER = \"../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/\"\nTEST_DATA_FOLDER = PROJECT_FOLDER + \"test_images/\"\nTRAIN_DATA_FOLDER = \"../input/hotelid-2022-train-images-512x512/images/\"\n","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:13.365067Z","iopub.execute_input":"2022-05-23T16:02:13.365279Z","iopub.status.idle":"2022-05-23T16:02:13.369988Z","shell.execute_reply.started":"2022-05-23T16:02:13.365255Z","shell.execute_reply":"2022-05-23T16:02:13.369221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:13.371595Z","iopub.execute_input":"2022-05-23T16:02:13.371834Z","iopub.status.idle":"2022-05-23T16:02:13.378798Z","shell.execute_reply.started":"2022-05-23T16:02:13.371802Z","shell.execute_reply":"2022-05-23T16:02:13.378129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A\nimport albumentations.pytorch as APT\nimport cv2 \n\nbase_transform = A.Compose([\n    A.ToFloat(),\n    APT.transforms.ToTensorV2(),\n])","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:24.299032Z","iopub.execute_input":"2022-05-23T16:02:24.299586Z","iopub.status.idle":"2022-05-23T16:02:24.760117Z","shell.execute_reply.started":"2022-05-23T16:02:24.299546Z","shell.execute_reply":"2022-05-23T16:02:24.759418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pad_image(img):\n    w, h, c = np.shape(img)\n    if w > h:\n        pad = int((w - h) / 2)\n        img = cv2.copyMakeBorder(img, 0, 0, pad, pad, cv2.BORDER_CONSTANT, value=0)\n    else:\n        pad = int((h - w) / 2)\n        img = cv2.copyMakeBorder(img, pad, pad, 0, 0, cv2.BORDER_CONSTANT, value=0)\n        \n    return img\n\n\ndef open_and_preprocess_image(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = pad_image(img)\n    return cv2.resize(img, (IMG_SIZE, IMG_SIZE))","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:25.755286Z","iopub.execute_input":"2022-05-23T16:02:25.75552Z","iopub.status.idle":"2022-05-23T16:02:25.763095Z","shell.execute_reply.started":"2022-05-23T16:02:25.755493Z","shell.execute_reply":"2022-05-23T16:02:25.762433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HotelImageDataset:\n    def __init__(self, data, transform=None, data_folder=\"train_images/\"):\n        self.data = data\n        self.data_folder = data_folder\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        record = self.data.iloc[idx]\n        image_path = self.data_folder + record[\"image_id\"]\n        \n        image = np.array(open_and_preprocess_image(image_path)).astype(np.uint8)\n\n        if self.transform:\n            transformed = self.transform(image=image)\n        \n        return {\n            \"image\" : transformed[\"image\"],\n        }","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:26.564982Z","iopub.execute_input":"2022-05-23T16:02:26.565625Z","iopub.status.idle":"2022-05-23T16:02:26.573911Z","shell.execute_reply.started":"2022-05-23T16:02:26.565589Z","shell.execute_reply":"2022-05-23T16:02:26.573208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ArcMarginProduct(nn.Module):\n    r\"\"\"Implement of large margin arc distance: :\n        Args:\n            in_features: size of each input sample\n            out_features: size of each output sample\n            s: norm of input feature\n            m: margin\n            cos(theta + m)\n        \"\"\"\n    def __init__(self, in_features, out_features, s=30.0, m=0.50, easy_margin=False):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        sine = torch.sqrt((1.0 - torch.pow(cosine, 2)).clamp(0, 1))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device='cuda')\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)  # you can use torch.where if your torch.__version__ is 0.4\n        output *= self.s\n\n        return output\n\nclass HotelIdModel(nn.Module):\n    def __init__(self, out_features, embed_size=256, backbone_name=\"efficientnet_b3\"):\n        super(HotelIdModel, self).__init__()\n\n        self.embed_size = embed_size\n        self.backbone = timm.create_model(backbone_name, pretrained=False)\n        in_features = self.backbone.get_classifier().in_features\n\n        fc_name, _ = list(self.backbone.named_modules())[-1]\n        if fc_name == 'classifier':\n            self.backbone.classifier = nn.Identity()\n        elif fc_name == 'head.fc':\n            self.backbone.head.fc = nn.Identity()\n        elif fc_name == 'fc':\n            self.backbone.fc = nn.Identity()\n        else:\n            raise Exception(\"unknown classifier layer: \" + fc_name)\n\n        self.arc_face = ArcMarginProduct(self.embed_size, out_features, s=30.0, m=0.50, easy_margin=False)\n\n        self.post = nn.Sequential(\n            nn.utils.weight_norm(nn.Linear(in_features, self.embed_size*2), dim=None),\n            nn.BatchNorm1d(self.embed_size*2),\n            nn.Dropout(0.2),\n            nn.utils.weight_norm(nn.Linear(self.embed_size*2, self.embed_size)),\n            nn.BatchNorm1d(self.embed_size),\n        )\n\n        print(f\"Model {backbone_name} ArcMarginProduct - Features: {in_features}, Embeds: {self.embed_size}\")\n        \n    def forward(self, input, targets = None):\n        x = self.backbone(input)\n        x = x.view(x.size(0), -1)\n        x = self.post(x)\n        \n        if targets is not None:\n            logits = self.arc_face(x, targets)\n            return logits\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:27.347733Z","iopub.execute_input":"2022-05-23T16:02:27.348348Z","iopub.status.idle":"2022-05-23T16:02:27.367901Z","shell.execute_reply.started":"2022-05-23T16:02:27.348314Z","shell.execute_reply":"2022-05-23T16:02:27.367227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EmbeddingNet(nn.Module):\n    def __init__(self, n_classes=100, embedding_size=64, backbone_name=\"efficientnet_b0\"):\n        super(EmbeddingNet, self).__init__()\n        \n        self.embedding_size = embedding_size\n        self.backbone = timm.create_model(backbone_name, num_classes=n_classes, pretrained=False)\n        \n        in_features = self.backbone.get_classifier().in_features\n        \n        fc_name, _ = list(self.backbone.named_modules())[-1]\n        if fc_name == 'classifier':\n           self.backbone.classifier = nn.Identity()\n        elif fc_name == 'head.fc':\n           self.backbone.head.fc = nn.Identity()\n        elif fc_name == 'fc':\n           self.backbone.fc = nn.Identity()\n        else:\n           raise Exception(\"unknown classifier layer: \" + fc_name)\n        \n        self.embedding = nn.Sequential(\n            nn.utils.weight_norm(nn.Linear(in_features, self.embedding_size*2), dim=None),\n            nn.BatchNorm1d(self.embedding_size*2),\n            nn.Dropout(0.2),\n            nn.utils.weight_norm(nn.Linear(self.embedding_size*2, self.embedding_size)),\n        )\n\n        self.classifier = nn.Sequential(\n            nn.BatchNorm1d(self.embedding_size),\n            nn.Dropout(0.2),\n            nn.Linear(self.embedding_size, n_classes),\n        )\n        \n        print(f\"Model {backbone_name} EmbeddingNet - Features: {in_features}, Embeds: {self.embedding_size}\")\n        \n    def embed_and_classify(self, x):\n        x = self.forward(x)\n        return x, self.classifier(x)\n\n    def forward(self, x):\n        x = self.backbone(x)\n        x = x.view(x.size(0), -1)\n        x = self.embedding(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:27.927282Z","iopub.execute_input":"2022-05-23T16:02:27.92821Z","iopub.status.idle":"2022-05-23T16:02:27.939387Z","shell.execute_reply.started":"2022-05-23T16:02:27.928136Z","shell.execute_reply":"2022-05-23T16:02:27.938685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics.pairwise import cosine_similarity\n\ndef generate_embeddings(args, loader, model, bar_desc=\"Generating embeds\"):\n    outputs_all = []\n    \n    model.eval()\n    with torch.no_grad():\n        t = tqdm(loader, desc=bar_desc)\n        for i, sample in enumerate(t):\n            input = sample['image'].to(args.device)\n            output = model(input)\n            outputs_all.extend(output.detach().cpu().numpy())     \n            \n    return outputs_all","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:28.483105Z","iopub.execute_input":"2022-05-23T16:02:28.483678Z","iopub.status.idle":"2022-05-23T16:02:28.490321Z","shell.execute_reply.started":"2022-05-23T16:02:28.483633Z","shell.execute_reply":"2022-05-23T16:02:28.489413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_matches(query, base_embeds, base_targets, k=N_MATCHES):\n    distance_df = pd.DataFrame(index=np.arange(len(base_targets)), data={\"hotel_id\": base_targets})\n    # calculate cosine distance of query embeds to all base embeds\n    distance_df[\"distance\"] = cosine_similarity([query], list(base_embeds))[0]\n    # sort by distance and hotel_id\n    distance_df = distance_df.sort_values(by=[\"distance\", \"hotel_id\"], ascending=False).reset_index(drop=True)\n    # return first 5 different hotel_id_codes\n    return distance_df[\"hotel_id\"].unique()[:N_MATCHES]\n\n\ndef predict(args, base_embeddings_df, test_loader, model):\n    test_embeds = generate_embeddings(args, test_loader, model, \"Generate test embeddings\")\n    \n    preds = []\n    for query_embeds in tqdm(test_embeds, desc=\"Similarity - match finding\"):\n        tmp = find_matches(query_embeds, \n                            base_embeddings_df[\"embeddings\"].values, \n                            base_embeddings_df[\"hotel_id\"].values)\n        preds.extend([tmp])\n        \n    return preds","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:29.867386Z","iopub.execute_input":"2022-05-23T16:02:29.867714Z","iopub.status.idle":"2022-05-23T16:02:29.874941Z","shell.execute_reply.started":"2022-05-23T16:02:29.867683Z","shell.execute_reply":"2022-05-23T16:02:29.874222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.DataFrame(data={\"image_id\": os.listdir(TEST_DATA_FOLDER), \"hotel_id\": \"\"}).sort_values(by=\"image_id\")","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:31.869539Z","iopub.execute_input":"2022-05-23T16:02:31.869827Z","iopub.status.idle":"2022-05-23T16:02:31.893046Z","shell.execute_reply.started":"2022-05-23T16:02:31.869793Z","shell.execute_reply":"2022-05-23T16:02:31.892325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(backbone_name, checkpoint_path, args):\n\n    model = HotelIdModel(args.n_classes, args.embedding_size, backbone_name)\n        \n    checkpoint = torch.load(checkpoint_path)\n    model.load_state_dict(checkpoint[\"model\"])\n    model = model.to(args.device)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:03:22.478589Z","iopub.execute_input":"2022-05-23T16:03:22.478852Z","iopub.status.idle":"2022-05-23T16:03:22.484812Z","shell.execute_reply.started":"2022-05-23T16:03:22.478823Z","shell.execute_reply":"2022-05-23T16:03:22.483876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class args:\n    batch_size = 16\n    num_workers = 2\n    embedding_size = 2048\n    device = ('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    \nseed_everything(seed=SEED)\n\ntest_dataset = HotelImageDataset(test_df, base_transform, data_folder=TEST_DATA_FOLDER)\ntest_loader = DataLoader(test_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:33.55552Z","iopub.execute_input":"2022-05-23T16:02:33.55607Z","iopub.status.idle":"2022-05-23T16:02:33.619792Z","shell.execute_reply.started":"2022-05-23T16:02:33.55603Z","shell.execute_reply":"2022-05-23T16:02:33.619026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_embeddings_df = pd.read_pickle('../input/training-arcmargin/arcmargin-model-efficientnet_b1-512x512-2048embeds-3116hotels_image-embeddings.pkl')\ndisplay(base_embeddings_df.head())","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:34.79859Z","iopub.execute_input":"2022-05-23T16:02:34.798928Z","iopub.status.idle":"2022-05-23T16:02:37.740294Z","shell.execute_reply.started":"2022-05-23T16:02:34.798894Z","shell.execute_reply":"2022-05-23T16:02:37.739568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args.n_classes = 3116\n\nmodel = get_model(\"efficientnet_b1\",\n                  \"../input/training-arcmargin/checkpoint-arcmargin-model-efficientnet_b1-512x512-2048embeds-3116hotels.pt\",\n                  args)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:03:26.137712Z","iopub.execute_input":"2022-05-23T16:03:26.138831Z","iopub.status.idle":"2022-05-23T16:03:27.356864Z","shell.execute_reply.started":"2022-05-23T16:03:26.138784Z","shell.execute_reply":"2022-05-23T16:03:27.356112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npreds = predict(args, base_embeddings_df, test_loader, model)\ntest_df[\"hotel_id\"] = [str(list(l)).strip(\"[]\").replace(\",\", \"\") for l in preds]\n\ntest_df.to_csv(\"submission.csv\", index=False)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T16:02:14.171796Z","iopub.status.idle":"2022-05-23T16:02:14.172234Z","shell.execute_reply.started":"2022-05-23T16:02:14.171989Z","shell.execute_reply":"2022-05-23T16:02:14.172011Z"},"trusted":true},"execution_count":null,"outputs":[]}]}