{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Intro\nInference notebook for [Hotel-ID starter - similarity - training](https://www.kaggle.com/code/michaln/hotel-id-starter-similarity-training)\n\nUsing model and embeddings from the training notebook to generate embeddings for test data and find similar images.","metadata":{"id":"DAY5rHgTm7e8","papermill":{"duration":0.024896,"end_time":"2022-03-24T14:00:54.588459","exception":false,"start_time":"2022-03-24T14:00:54.563563","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')","metadata":{"executionInfo":{"elapsed":16271,"status":"ok","timestamp":1619310548121,"user":{"displayName":"Jeom Jin-Ho","photoUrl":"","userId":"00155613517919499503"},"user_tz":-120},"id":"alleged-legislation","outputId":"c6541e5f-ffb4-4609-d6c6-39784e6a07b1","papermill":{"duration":0.036572,"end_time":"2022-03-24T14:00:54.649254","exception":false,"start_time":"2022-03-24T14:00:54.612682","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:18:57.822322Z","iopub.execute_input":"2022-05-19T11:18:57.822711Z","iopub.status.idle":"2022-05-19T11:18:57.854492Z","shell.execute_reply.started":"2022-05-19T11:18:57.822594Z","shell.execute_reply":"2022-05-19T11:18:57.853803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{"id":"cZoSOL9Qm-Yr","papermill":{"duration":0.023644,"end_time":"2022-03-24T14:00:54.696898","exception":false,"start_time":"2022-03-24T14:00:54.673254","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nimport os\nimport math","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","executionInfo":{"elapsed":14459,"status":"ok","timestamp":1619310548121,"user":{"displayName":"Jeom Jin-Ho","photoUrl":"","userId":"00155613517919499503"},"user_tz":-120},"id":"expired-matter","papermill":{"duration":0.030271,"end_time":"2022-03-24T14:00:54.751131","exception":false,"start_time":"2022-03-24T14:00:54.72086","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:18:57.957764Z","iopub.execute_input":"2022-05-19T11:18:57.958724Z","iopub.status.idle":"2022-05-19T11:18:57.965234Z","shell.execute_reply.started":"2022-05-19T11:18:57.958680Z","shell.execute_reply":"2022-05-19T11:18:57.964305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image as pil_image\nfrom tqdm import tqdm","metadata":{"executionInfo":{"elapsed":16003,"status":"ok","timestamp":1619310550014,"user":{"displayName":"Jeom Jin-Ho","photoUrl":"","userId":"00155613517919499503"},"user_tz":-120},"id":"extreme-problem","papermill":{"duration":3.220402,"end_time":"2022-03-24T14:00:57.995239","exception":false,"start_time":"2022-03-24T14:00:54.774837","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:18:58.092995Z","iopub.execute_input":"2022-05-19T11:18:58.093269Z","iopub.status.idle":"2022-05-19T11:18:58.097714Z","shell.execute_reply.started":"2022-05-19T11:18:58.093238Z","shell.execute_reply":"2022-05-19T11:18:58.096789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader\nimport torchvision.models as models\n\nimport timm\nfrom sklearn.metrics.pairwise import cosine_similarity","metadata":{"executionInfo":{"elapsed":19672,"status":"ok","timestamp":1619310554099,"user":{"displayName":"Jeom Jin-Ho","photoUrl":"","userId":"00155613517919499503"},"user_tz":-120},"id":"angry-domain","papermill":{"duration":2.727834,"end_time":"2022-03-24T14:01:00.766951","exception":false,"start_time":"2022-03-24T14:00:58.039117","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:18:58.222909Z","iopub.execute_input":"2022-05-19T11:18:58.223636Z","iopub.status.idle":"2022-05-19T11:19:02.586458Z","shell.execute_reply.started":"2022-05-19T11:18:58.223567Z","shell.execute_reply":"2022-05-19T11:19:02.585631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Global","metadata":{"id":"0B00pe7mnBTj","papermill":{"duration":0.023573,"end_time":"2022-03-24T14:01:00.814976","exception":false,"start_time":"2022-03-24T14:01:00.791403","status":"completed"},"tags":[]}},{"cell_type":"code","source":"SEED = 42\nIMG_SIZE = 256\nN_MATCHES = 5\n\nPROJECT_FOLDER = \"../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/\"\nTRAIN_DATA_FOLDER = \"../input/hotelid-2022-train-images-256x256/images/\"\nTEST_DATA_FOLDER = PROJECT_FOLDER + \"test_images/\"\n#TEST_DATA_FOLDER = PROJECT_FOLDER + \"train_images/100055/\"\nTRAIN_DATA_FOLDER = \"../input/hotelid-2022-train-images-256x256/images/\"","metadata":{"executionInfo":{"elapsed":589,"status":"ok","timestamp":1619310979015,"user":{"displayName":"Jeom Jin-Ho","photoUrl":"","userId":"00155613517919499503"},"user_tz":-120},"id":"contained-brief","papermill":{"duration":0.03175,"end_time":"2022-03-24T14:01:00.871686","exception":false,"start_time":"2022-03-24T14:01:00.839936","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:19:02.588455Z","iopub.execute_input":"2022-05-19T11:19:02.588873Z","iopub.status.idle":"2022-05-19T11:19:02.593283Z","shell.execute_reply.started":"2022-05-19T11:19:02.588833Z","shell.execute_reply":"2022-05-19T11:19:02.592644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.listdir(PROJECT_FOLDER))","metadata":{"executionInfo":{"elapsed":879,"status":"ok","timestamp":1619310979515,"user":{"displayName":"Jeom Jin-Ho","photoUrl":"","userId":"00155613517919499503"},"user_tz":-120},"id":"PZvmFng7ctO3","outputId":"dce0cc91-8e70-4acc-a0b8-6763ffffd5ca","papermill":{"duration":0.031651,"end_time":"2022-03-24T14:01:00.927239","exception":false,"start_time":"2022-03-24T14:01:00.895588","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:19:02.594506Z","iopub.execute_input":"2022-05-19T11:19:02.594884Z","iopub.status.idle":"2022-05-19T11:19:02.600402Z","shell.execute_reply.started":"2022-05-19T11:19:02.594850Z","shell.execute_reply":"2022-05-19T11:19:02.598920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"executionInfo":{"elapsed":600,"status":"ok","timestamp":1619310981653,"user":{"displayName":"Jeom Jin-Ho","photoUrl":"","userId":"00155613517919499503"},"user_tz":-120},"id":"eastern-content","papermill":{"duration":0.032105,"end_time":"2022-03-24T14:01:01.031949","exception":false,"start_time":"2022-03-24T14:01:00.999844","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:19:02.603072Z","iopub.execute_input":"2022-05-19T11:19:02.603492Z","iopub.status.idle":"2022-05-19T11:19:02.611034Z","shell.execute_reply.started":"2022-05-19T11:19:02.603458Z","shell.execute_reply":"2022-05-19T11:19:02.610275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset and transformations","metadata":{"id":"xaJKvvuKnW4k","papermill":{"duration":0.023988,"end_time":"2022-03-24T14:01:01.080488","exception":false,"start_time":"2022-03-24T14:01:01.0565","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import albumentations as A\nimport albumentations.pytorch as APT\nimport cv2 \n\nbase_transform = A.Compose([\n    A.ToFloat(),\n    APT.transforms.ToTensorV2(),\n])","metadata":{"executionInfo":{"elapsed":1519,"status":"ok","timestamp":1619310984075,"user":{"displayName":"Jeom Jin-Ho","photoUrl":"","userId":"00155613517919499503"},"user_tz":-120},"id":"revolutionary-membership","papermill":{"duration":0.747421,"end_time":"2022-03-24T14:01:01.852072","exception":false,"start_time":"2022-03-24T14:01:01.104651","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:19:02.612320Z","iopub.execute_input":"2022-05-19T11:19:02.612732Z","iopub.status.idle":"2022-05-19T11:19:03.664752Z","shell.execute_reply.started":"2022-05-19T11:19:02.612698Z","shell.execute_reply":"2022-05-19T11:19:03.663956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pad_image(img):\n    w, h, c = np.shape(img)\n    if w > h:\n        pad = int((w - h) / 2)\n        img = cv2.copyMakeBorder(img, 0, 0, pad, pad, cv2.BORDER_CONSTANT, value=0)\n    else:\n        pad = int((h - w) / 2)\n        img = cv2.copyMakeBorder(img, pad, pad, 0, 0, cv2.BORDER_CONSTANT, value=0)\n        \n    return img\n\n\ndef open_and_preprocess_image(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = pad_image(img)\n    return cv2.resize(img, (IMG_SIZE, IMG_SIZE))","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:19:03.667743Z","iopub.execute_input":"2022-05-19T11:19:03.668000Z","iopub.status.idle":"2022-05-19T11:19:03.674890Z","shell.execute_reply.started":"2022-05-19T11:19:03.667965Z","shell.execute_reply":"2022-05-19T11:19:03.673856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HotelImageDataset:\n    def __init__(self, data, transform=None, data_folder=\"train_images/\"):\n        self.data = data\n        self.data_folder = data_folder\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        record = self.data.iloc[idx]\n        image_path = self.data_folder + record[\"image_id\"]\n        \n        image = np.array(open_and_preprocess_image(image_path)).astype(np.uint8)\n\n        if self.transform:\n            transformed = self.transform(image=image)\n            image = transformed[\"image\"]\n        \n        return {\n            \"image\" : image,\n        }","metadata":{"executionInfo":{"elapsed":1058,"status":"ok","timestamp":1619310984077,"user":{"displayName":"Jeom Jin-Ho","photoUrl":"","userId":"00155613517919499503"},"user_tz":-120},"id":"found-mouth","papermill":{"duration":0.037507,"end_time":"2022-03-24T14:01:01.914352","exception":false,"start_time":"2022-03-24T14:01:01.876845","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:19:03.676461Z","iopub.execute_input":"2022-05-19T11:19:03.676887Z","iopub.status.idle":"2022-05-19T11:19:03.686953Z","shell.execute_reply.started":"2022-05-19T11:19:03.676846Z","shell.execute_reply":"2022-05-19T11:19:03.686143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"id":"NMDM4PwPnced","papermill":{"duration":0.023902,"end_time":"2022-03-24T14:01:01.962307","exception":false,"start_time":"2022-03-24T14:01:01.938405","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class EmbeddingModel(nn.Module):\n    def __init__(self, n_classes=100, embedding_size=64, backbone_name=\"efficientnet_b1\"):\n        super(EmbeddingModel, self).__init__()\n        \n        self.backbone = timm.create_model(backbone_name, num_classes=n_classes, pretrained=False)\n            \n        if backbone_name == \"inception_v3\":\n            in_features = self.backbone.get_classifier().out_features\n        else:\n            in_features = self.backbone.get_classifier().in_features\n        \n        self.backbone.classifier = nn.Identity()\n        self.embedding = nn.Linear(in_features, embedding_size)\n        self.classifier = nn.Linear(embedding_size, n_classes)\n\n    def embed_and_classify(self, x):\n        x = self.forward(x)\n        return x, self.classifier(x)\n\n    def forward(self, x):\n        x = self.backbone(x)\n        x = x.view(x.size(0), -1)\n        x = self.embedding(x)\n        return x","metadata":{"papermill":{"duration":0.032166,"end_time":"2022-03-24T14:01:02.018479","exception":false,"start_time":"2022-03-24T14:01:01.986313","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:19:03.688412Z","iopub.execute_input":"2022-05-19T11:19:03.688879Z","iopub.status.idle":"2022-05-19T11:19:03.700050Z","shell.execute_reply.started":"2022-05-19T11:19:03.688841Z","shell.execute_reply":"2022-05-19T11:19:03.699171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model helper functions","metadata":{"id":"YMZYKhUSneMY","papermill":{"duration":0.024153,"end_time":"2022-03-24T14:01:02.067537","exception":false,"start_time":"2022-03-24T14:01:02.043384","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_distances(test_embeds, base_embeds, model_array, image_index):\n    distances = None\n    for i, model in enumerate(model_array):\n        model_input_embeds = test_embeds[i][image_index]\n        model_base_embeds = base_embeds[i][\"embeddings\"].values\n        \n        output_distances = cosine_similarity([model_input_embeds], list(model_base_embeds))[0]\n    \n#         print(f\"Output distances for model {i}:\")\n#         for i,d in enumerate(output_distances[:5]):\n#             print(f\"{i}) {d}\")\n        \n        if distances is None:\n            distances = output_distances\n        else:\n            distances = distances + output_distances\n        \n#     print(f\"Output distances combined:\\n\")\n#     for i,d in enumerate(distances[:5]):\n#         print(f\"{i}) {d}\")\n    \n    return distances\n\ndef generate_embeddings(args, loader, model, bar_desc=\"Generating embeds\"):\n    outputs_all = []\n    \n    model.eval()\n    with torch.no_grad():\n        t = tqdm(loader, desc=bar_desc)\n        for i, sample in enumerate(t):\n            input = sample['image'].to(args.device)\n            output = model(input)\n            output = output.detach().cpu().numpy()\n            outputs_all.extend(output)\n            \n    return outputs_all","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:19:03.701890Z","iopub.execute_input":"2022-05-19T11:19:03.702221Z","iopub.status.idle":"2022-05-19T11:19:03.711779Z","shell.execute_reply.started":"2022-05-19T11:19:03.702183Z","shell.execute_reply":"2022-05-19T11:19:03.710636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_matches(distances, base_targets, k=N_MATCHES):\n    distance_df = pd.DataFrame(index=np.arange(len(base_targets)), data={\"hotel_id\": base_targets})\n    # calculate cosine distance of query embeds to all base embeds\n    distance_df[\"distance\"] = distances\n    \n    # sort by distance and hotel_id\n    distance_df = distance_df.sort_values(by=[\"distance\", \"hotel_id\"], ascending=False).reset_index(drop=True)\n    print(f\"distance_df after sorting: {distance_df}\")\n    \n    # return first 5 different hotel_id_codes\n    return distance_df[\"hotel_id\"].unique()[:N_MATCHES]\n\ndef predict(args, base_embeddings_list, test_loader, model_list, bar_desc=\"Generating embeds\"):    \n    test_embeds = []\n    # For every model, generate embeddings for every image\n    for model in model_list:\n        model_embeds = generate_embeddings(args, test_loader, model, \"Generate test embeddings\")\n        test_embeds.extend([model_embeds])\n    \n    preds = []\n    # For every image, calculate the distances for every model-embedding pair\n    for image_index in range(len(test_embeds[0])):\n        #distances = get_distances(test_embeds, base_embeddings_list, model_list, image_index)\n        preds_for_image = find_matches(distances, base_embeddings_list[0][\"hotel_id\"].values)\n        preds.extend([preds_for_image])\n        \n    return preds","metadata":{"papermill":{"duration":0.034692,"end_time":"2022-03-24T14:01:02.127372","exception":false,"start_time":"2022-03-24T14:01:02.09268","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:19:03.715498Z","iopub.execute_input":"2022-05-19T11:19:03.715982Z","iopub.status.idle":"2022-05-19T11:19:03.724799Z","shell.execute_reply.started":"2022-05-19T11:19:03.715946Z","shell.execute_reply":"2022-05-19T11:19:03.723800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data","metadata":{"id":"AwShW1wXniD6","papermill":{"duration":0.023807,"end_time":"2022-03-24T14:01:02.175822","exception":false,"start_time":"2022-03-24T14:01:02.152015","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_df = pd.DataFrame(data={\"image_id\": os.listdir(TEST_DATA_FOLDER), \"hotel_id\": \"\"}).sort_values(by=\"image_id\")","metadata":{"executionInfo":{"elapsed":3742,"status":"ok","timestamp":1619311036476,"user":{"displayName":"Jeom Jin-Ho","photoUrl":"","userId":"00155613517919499503"},"user_tz":-120},"id":"discrete-right","outputId":"c21ed589-3139-4919-b5d5-07bcf6f1df15","papermill":{"duration":0.103381,"end_time":"2022-03-24T14:01:02.453866","exception":false,"start_time":"2022-03-24T14:01:02.350485","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:19:03.726442Z","iopub.execute_input":"2022-05-19T11:19:03.726941Z","iopub.status.idle":"2022-05-19T11:19:03.746263Z","shell.execute_reply.started":"2022-05-19T11:19:03.726901Z","shell.execute_reply":"2022-05-19T11:19:03.745637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare model","metadata":{"id":"5JPdD2bpnniP","papermill":{"duration":0.023835,"end_time":"2022-03-24T14:01:02.502786","exception":false,"start_time":"2022-03-24T14:01:02.478951","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_model(backbone_name, checkpoint_path, args):\n    model = EmbeddingModel(args.n_classes, args.embedding_size, backbone_name)\n        \n    checkpoint = torch.load(checkpoint_path)\n    #checkpoint = torch.load(checkpoint_path, map_location=torch.device('cpu'))\n    model.load_state_dict(checkpoint[\"model\"])\n    model = model.to(args.device)\n    \n    return model","metadata":{"papermill":{"duration":0.031082,"end_time":"2022-03-24T14:01:02.557921","exception":false,"start_time":"2022-03-24T14:01:02.526839","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:19:03.748520Z","iopub.execute_input":"2022-05-19T11:19:03.748983Z","iopub.status.idle":"2022-05-19T11:19:03.754085Z","shell.execute_reply.started":"2022-05-19T11:19:03.748946Z","shell.execute_reply":"2022-05-19T11:19:03.753236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class args:\n    batch_size = 64\n    num_workers = 2\n    embedding_size = 128\n    device = ('cuda' if torch.cuda.is_available() else 'cpu')\n    \nseed_everything(seed=SEED)\n\ntest_dataset = HotelImageDataset(test_df, base_transform, data_folder=TEST_DATA_FOLDER)\ntest_loader  = DataLoader(test_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=False)","metadata":{"executionInfo":{"elapsed":450,"status":"ok","timestamp":1619311064188,"user":{"displayName":"Jeom Jin-Ho","photoUrl":"","userId":"00155613517919499503"},"user_tz":-120},"id":"appointed-machinery","papermill":{"duration":0.069839,"end_time":"2022-03-24T14:01:02.65177","exception":false,"start_time":"2022-03-24T14:01:02.581931","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:19:03.755184Z","iopub.execute_input":"2022-05-19T11:19:03.757455Z","iopub.status.idle":"2022-05-19T11:19:03.826182Z","shell.execute_reply.started":"2022-05-19T11:19:03.757418Z","shell.execute_reply":"2022-05-19T11:19:03.825369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_embeddings_list = [\n    pd.read_pickle('../input/hotelidstarter/efficientnet_b2-256x256-embeddings.pkl'),\n    pd.read_pickle('../input/hotelidstarter/densenet169-256x256-embeddings.pkl'),\n    pd.read_pickle('../input/hotelidstarter/inception_v3-256x256_image-embeddings.pkl')\n    ]\n# display(base_embeddings_df.head())","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:19:03.827391Z","iopub.execute_input":"2022-05-19T11:19:03.828224Z","iopub.status.idle":"2022-05-19T11:19:04.995552Z","shell.execute_reply.started":"2022-05-19T11:19:03.828181Z","shell.execute_reply":"2022-05-19T11:19:04.994761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args.n_classes = base_embeddings_list[0][\"hotel_id\"].nunique()\n\nmodel1 = get_model(\"efficientnet_b2\",\n                  \"../input/hotelidstarter/efficientnet_b2-256x256-checkpoint-epoch_10-acc_0.2509.pt\", \n                  args)\nmodel2 = get_model(\"densenet169\",\n                  \"../input/hotelidstarter/densenet169-256x256-checkpoint-epoch_12-acc_0.2171.pt\", \n                  args)\nmodel3 = get_model(\"inception_v3\",\n                  \"../input/hotelidstarter/inception_v3-256x256-checkpoint-epoch_13-acc_0.1955.pt\", \n                  args)\nmodel_list = [model1, model2, model3]","metadata":{"papermill":{"duration":5.553999,"end_time":"2022-03-24T14:01:08.229948","exception":false,"start_time":"2022-03-24T14:01:02.675949","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-19T11:19:05.001045Z","iopub.execute_input":"2022-05-19T11:19:05.001267Z","iopub.status.idle":"2022-05-19T11:19:15.624373Z","shell.execute_reply.started":"2022-05-19T11:19:05.001240Z","shell.execute_reply":"2022-05-19T11:19:15.623588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_embeddings_list[0].head()","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:19:15.625700Z","iopub.execute_input":"2022-05-19T11:19:15.625963Z","iopub.status.idle":"2022-05-19T11:19:15.649152Z","shell.execute_reply.started":"2022-05-19T11:19:15.625927Z","shell.execute_reply":"2022-05-19T11:19:15.648516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Ensemble","metadata":{}},{"cell_type":"code","source":"class HotelTrainDataset:\n    def __init__(self, data, transform=None, data_path=\"train_images/\"):\n        self.data = data\n        self.data_path = data_path\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        record = self.data.iloc[idx]\n        #if USE_ORIGINAL_DATA:\n        #image_path = self.data_path + str(record[\"hotel_id\"]) + \"/\" + record[\"image_id\"]\n        #else:\n        image_path = self.data_path + record[\"image_id\"]\n        image = np.array(pil_image.open(image_path)).astype(np.uint8)\n\n        if self.transform:\n            transformed = self.transform(image=image)\n            image = transformed[\"image\"]\n        \n        return {\n            \"image\" : image,\n            \"target\" : record['hotel_id_code'],\n        }\n\nclass MetaModel(nn.Module):\n    def __init__(self, n_models=3, n_classes=100):\n        super(MetaModel, self).__init__()\n        \n        self.linear = nn.Linear(n_models, n_classes)\n\n    def forward(self, x):\n        x = self.linear(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:19:15.650249Z","iopub.execute_input":"2022-05-19T11:19:15.650514Z","iopub.status.idle":"2022-05-19T11:19:15.661866Z","shell.execute_reply.started":"2022-05-19T11:19:15.650476Z","shell.execute_reply":"2022-05-19T11:19:15.661175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_epoch(args, model, embeddings, criterion, optimizer, scheduler, epoch, num_batch_steps=1):\n    losses = []\n    targets_all = []\n    outputs_all = []\n    \n    model.train()    \n    loss_sum = 0\n    for i, sample in enumerate(embeddings):        \n        images = sample['image'].to(args.device)\n        targets = sample['target'].to(args.device)\n        \n        outputs = model(images)\n        \n        loss = criterion(outputs, targets)/num_batch_steps        \n        loss.backward()\n        loss_sum += loss\n        \n        if (i+1)%num_batch_steps==0:\n            losses.append(loss_sum.item())\n            \n            optimizer.step()\n            optimizer.zero_grad()\n        \n            if scheduler:\n                scheduler.step()                \n            \n            targets_all.extend(targets.cpu().numpy())\n            outputs_all.extend(torch.sigmoid(outputs).detach().cpu().numpy())\n\n            score = np.mean(targets_all == np.argmax(outputs_all, axis=1))\n            desc = f\"Training epoch {epoch}/{args.epochs} - loss:{loss_sum:0.4f}, accuracy: {score:0.4f}\"\n            t.set_description(desc)\n            \n            loss_sum = 0\n        \n    return np.mean(losses), score\n    \ndef train_meta_learner(args, data_df):\n    model = MetaModel(3, args.n_classes)\n    model = model.to(args.device)\n\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr)\n\n    scheduler = torch.optim.lr_scheduler.OneCycleLR(\n                    optimizer,\n                    max_lr=args.lr,\n                    epochs=args.epochs,\n                    steps_per_epoch=len(valid_loader),\n                    div_factor=10,\n                    final_div_factor=1,\n                    pct_start=0.1,\n                    anneal_strategy=\"cos\",\n                )\n    \n    start_epoch = 1\n    \n    for epoch in range(start_epoch, args.epochs+1):\n        val_loss, val_score = train_epoch(args, model, valid_loader, criterion, optimizer, scheduler, epoch, args.num_batch_steps)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:19:15.663317Z","iopub.execute_input":"2022-05-19T11:19:15.663989Z","iopub.status.idle":"2022-05-19T11:19:15.678066Z","shell.execute_reply.started":"2022-05-19T11:19:15.663947Z","shell.execute_reply":"2022-05-19T11:19:15.677257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CSV_FILE = \"../input/hotelid-2022-train-images-256x256/train.csv\"\n\ndata_df = pd.read_csv(CSV_FILE)\n# encode hotel ids\ndata_df[\"hotel_id_code\"] = data_df[\"hotel_id\"].astype('category').cat.codes.values.astype(np.int64)\n\nclass args:\n    epochs = 20\n    lr = 1e-3\n    batch_size = 32\n    num_batch_steps = 2\n    num_workers = 2\n    val_samples = 1\n    embedding_size = 128\n    backbone_name = \"densenet169\"\n    n_classes = data_df[\"hotel_id_code\"].nunique()\n    device = ('cuda' if torch.cuda.is_available() else 'cpu')\n\nclass meta_args:\n    epochs = 5\n    lr = 1e-3\n    n_classes = data_df[\"hotel_id_code\"].nunique()\n    device = ('cuda' if torch.cuda.is_available() else 'cpu')\n\nval_samples = 1\n\nhotel_image_count = data_df.groupby(\"hotel_id\")[\"image_id\"].count()\n# hotels that have more images than samples for validation\nvalid_hotels = hotel_image_count[hotel_image_count > val_samples]\n# data that can be split into train and val set\nvalid_data = data_df[data_df[\"hotel_id\"].isin(valid_hotels.index)]\n# if hotel had less than required val_samples it will be only in the train set\nvalid_df = valid_data.groupby(\"hotel_id\").sample(args.val_samples, random_state=SEED)\n\nvalid_dataset = HotelTrainDataset(valid_df, base_transform, data_path=TRAIN_DATA_FOLDER)\nvalid_loader  = DataLoader(valid_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:19:15.680919Z","iopub.execute_input":"2022-05-19T11:19:15.681133Z","iopub.status.idle":"2022-05-19T11:19:16.700256Z","shell.execute_reply.started":"2022-05-19T11:19:15.681095Z","shell.execute_reply":"2022-05-19T11:19:16.699468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_val_distance(val_embeds, base_embeds):\n    model_base_embeds = base_embeds[\"embeddings\"].values\n    # 1- if we do *\n    #print(f\"val_embeds len {len(val_embeds)}\")\n    #print(f\"val_embeds[0] len {len(val_embeds[0])}\")\n    #print(f\"model_base_embeds shape {model_base_embeds[0]}\")\n    distance = cosine_similarity(val_embeds, list(model_base_embeds))[0]    \n    return distance\n\ndef find_val_matches(distances, base_targets, k=N_MATCHES):\n    distance_df = pd.DataFrame(index=np.arange(len(base_targets)), data={\"hotel_id\": base_targets})\n    # calculate cosine distance of query embeds to all base embeds\n    distance_df[\"distance\"] = distances\n    \n    # sort by distance and hotel_id\n    distance_df = distance_df.sort_values(by=[\"distance\", \"hotel_id\"], ascending=False).reset_index(drop=True)\n    print(f\"distance_df after sorting: {distance_df}\")\n    \n    # return first 5 different hotel_id_codes\n    return distance_df[\"distance\"].unique()[:N_MATCHES]","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:19:16.701739Z","iopub.execute_input":"2022-05-19T11:19:16.701998Z","iopub.status.idle":"2022-05-19T11:19:16.709248Z","shell.execute_reply.started":"2022-05-19T11:19:16.701963Z","shell.execute_reply":"2022-05-19T11:19:16.708563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_embeds = []\n# For every model, generate embeddings for every image\nfor model in model_list:\n    model_embeds = generate_embeddings(args, valid_loader, model, \"Generate val embeddings\")\n    val_embeds.extend([model_embeds])","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:20:18.056217Z","iopub.execute_input":"2022-05-19T11:20:18.057067Z","iopub.status.idle":"2022-05-19T11:20:59.521873Z","shell.execute_reply.started":"2022-05-19T11:20:18.057024Z","shell.execute_reply":"2022-05-19T11:20:59.520534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len(base_embeddings_list))\n# print(len(base_embeddings_list[0]))\n# print(valid_df)\n\nimage_embeddings = base_embeddings_list[0][\"embeddings\"].values\nprint(len(image_embeddings[0]))","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:26:49.088575Z","iopub.execute_input":"2022-05-19T11:26:49.089058Z","iopub.status.idle":"2022-05-19T11:26:49.097848Z","shell.execute_reply.started":"2022-05-19T11:26:49.089017Z","shell.execute_reply":"2022-05-19T11:26:49.095685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_distances = []\n# For every image, calculate the distances for every model-embedding pair\nfor image_index in range(len(val_embeds[0])):\n    distances = []\n    for model_index in range(len(model_list)):\n        distance = get_val_distance([val_embeds[model_index][image_index]], base_embeddings_list[model_index])\n        distances.extend([distance])\n    image_distances.extend([distances])\n\nprint(f\"image distances shape: {image_distances.shape}\")\n\n# We expect image_distances shape (3700,3,40000) (3700,3) [40000]\n# Output of trainer should be (40000)\n\n# Combine the 3 distances -> NN\n#train_meta_learner(meta_args, data_df)\n\n# \n# preds_for_image = find_val_matches(distances, base_embeddings_list[0][\"hotel_id\"].values)\n# preds.extend([preds_for_image])","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:32:12.592453Z","iopub.execute_input":"2022-05-19T11:32:12.592917Z","iopub.status.idle":"2022-05-19T11:42:04.150072Z","shell.execute_reply.started":"2022-05-19T11:32:12.592867Z","shell.execute_reply":"2022-05-19T11:42:04.148889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.024227,"end_time":"2022-03-24T14:01:08.278753","exception":false,"start_time":"2022-03-24T14:01:08.254526","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n\npreds = predict(args, base_embeddings_list, test_loader, model_list)\n# transform array of hotel_ids into string\ntest_df[\"hotel_id\"] = [str(list(l)).strip(\"[]\").replace(\",\", \"\") for l in preds]\n\ntest_df.to_csv(\"submission.csv\", index=False)\ntest_df.head()","metadata":{"papermill":{"duration":0.047156,"end_time":"2022-03-24T14:01:08.414557","exception":false,"start_time":"2022-03-24T14:01:08.367401","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}