{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Intro\n\nSimple starter notebook for [Hotel-ID to Combat Human Trafficking 2022 - FGVC9](https://www.kaggle.com/competitions/hotel-id-to-combat-human-trafficking-2022-fgvc9) competition using classification model. This notebook uses preprocessed images that were resized and padded to 256x256 pixel. \n\nUsed dataset: [Hotel-ID 2022 train images 256x256](https://www.kaggle.com/datasets/michaln/hotelid-2022-train-images-256x256) created by [Hotel-ID - image preprocessing - 256x256](https://www.kaggle.com/code/michaln/hotel-id-image-preprocessing-256x256) notebook.\n\nClassification model might not be very useful in real life application because it will not be able to handle new hotels without retraining. You can find better approaches in last year competition [Hotel-ID to Combat Human Trafficking 2021 - FGVC8](https://www.kaggle.com/competitions/hotel-id-2021-fgvc8):\n- [1st place solution: Swin + Arcface + Label-constrained DBA](https://www.kaggle.com/c/hotel-id-2021-fgvc8/discussion/242087)\n- [8th place solution: arcface + cosface + classification](https://www.kaggle.com/c/hotel-id-2021-fgvc8/discussion/242207)\n- [14th place solution - fastai ensemble](https://www.kaggle.com/c/hotel-id-2021-fgvc8/discussion/242030)","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{"id":"MyC4gTwZ3MKJ","papermill":{"duration":0.020375,"end_time":"2022-03-23T20:08:58.820405","exception":false,"start_time":"2022-03-23T20:08:58.80003","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install timm","metadata":{"papermill":{"duration":18.268618,"end_time":"2022-03-23T20:09:17.10966","exception":false,"start_time":"2022-03-23T20:08:58.841042","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nimport os","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"u0Bz2ktn2_ap","papermill":{"duration":0.032275,"end_time":"2022-03-23T20:09:17.168351","exception":false,"start_time":"2022-03-23T20:09:17.136076","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image as pil_image\nfrom tqdm import tqdm\n\nimport matplotlib\nimport matplotlib.pyplot as plt\n\nimport plotly.graph_objects as go\nimport plotly.express as px\nfrom plotly.subplots import make_subplots","metadata":{"id":"tOszKuxt3PXn","papermill":{"duration":3.425867,"end_time":"2022-03-23T20:09:20.619496","exception":false,"start_time":"2022-03-23T20:09:17.193629","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader\n\nimport timm","metadata":{"id":"uQE7wYFR3QxV","papermill":{"duration":2.460075,"end_time":"2022-03-23T20:09:23.106294","exception":false,"start_time":"2022-03-23T20:09:20.646219","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Global","metadata":{"id":"tirOg6jm3aIB","papermill":{"duration":0.025747,"end_time":"2022-03-23T20:09:23.159882","exception":false,"start_time":"2022-03-23T20:09:23.134135","status":"completed"},"tags":[]}},{"cell_type":"code","source":"IMG_SIZE = 256\nSEED = 42\nN_MATCHES = 5\n\n\nPROJECT_FOLDER = \"../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/\"\nDATA_FOLDER = \"../input/padding256/\"\nIMAGE_FOLDER = os.path.join(DATA_FOLDER, 'images/')\nOUTPUT_FOLDER = \"\"\n\ntrain_df = pd.read_csv(os.path.join(DATA_FOLDER, 'train_padding_256.csv'))","metadata":{"id":"DV7qHDuYGoJH","papermill":{"duration":0.086034,"end_time":"2022-03-23T20:09:23.2714","exception":false,"start_time":"2022-03-23T20:09:23.185366","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.listdir(PROJECT_FOLDER))","metadata":{"id":"TB9CXg8U3bbQ","papermill":{"duration":0.032796,"end_time":"2022-03-23T20:09:23.329909","exception":false,"start_time":"2022-03-23T20:09:23.297113","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"id":"csp2OMgo2_ar","papermill":{"duration":0.03307,"end_time":"2022-03-23T20:09:23.38914","exception":false,"start_time":"2022-03-23T20:09:23.35607","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset and transformations","metadata":{"id":"8V_xuoN73lON","papermill":{"duration":0.025486,"end_time":"2022-03-23T20:09:23.440614","exception":false,"start_time":"2022-03-23T20:09:23.415128","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Coarse dropout with fill_value=(255,0,0) (full red channel) is used to simulate the occlussions like the one in test dataset. \n```python\nA.CoarseDropout(p=0.75, max_holes=1, \n                min_height=IMG_SIZE//4, max_height=IMG_SIZE//2,\n                min_width=IMG_SIZE//4,  max_width=IMG_SIZE//2, \n                fill_value=(255,0,0))\n```","metadata":{}},{"cell_type":"code","source":"import albumentations as A\nimport albumentations.pytorch as APT\nimport cv2 \n\n# used for training dataset - augmentations and occlusions\ntrain_transform = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, border_mode=0, p=0.85),\n    A.OpticalDistortion(distort_limit=1.0),\n    A.Perspective(p=0.25),\n    \n    A.CoarseDropout(p=0.75, max_holes=1, \n                    min_height=IMG_SIZE//4, max_height=IMG_SIZE//2,\n                    min_width=IMG_SIZE//4,  max_width=IMG_SIZE//2, \n                    fill_value=(255,0,0)),# simulating occlusions in test data\n\n    A.RandomBrightnessContrast(p=0.75),\n    A.ToFloat(),\n    APT.transforms.ToTensorV2(),\n])\n\n# used for validation dataset - only occlusions\nval_transform = A.Compose([\n    A.CoarseDropout(p=0.75, max_holes=1, \n                    min_height=IMG_SIZE//4, max_height=IMG_SIZE//2,\n                    min_width=IMG_SIZE//4,  max_width=IMG_SIZE//2, \n                    fill_value=(255,0,0)),# simulating occlusions\n    A.ToFloat(),\n    APT.transforms.ToTensorV2(),\n])\n\n# no augmentations\nbase_transform = A.Compose([\n    A.ToFloat(),\n    APT.transforms.ToTensorV2(),\n])","metadata":{"id":"8ucWZHeG2_as","papermill":{"duration":0.919086,"end_time":"2022-03-23T20:09:24.385553","exception":false,"start_time":"2022-03-23T20:09:23.466467","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HotelTrainDataset:\n    def __init__(self, data, transform=None, data_path=\"train_images/\"):\n        self.data = data\n        self.data_path = data_path\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        record = self.data.iloc[idx]\n        image_path = self.data_path + record[\"image_id\"]\n        image = np.array(pil_image.open(image_path)).astype(np.uint8)\n\n        if self.transform:\n            transformed = self.transform(image=image)\n            image = transformed[\"image\"]\n        \n        return {\n            \"image\" : image,\n            \"target\" : record['hotel_id_code'],\n        }","metadata":{"id":"EiLYsfKq2_at","papermill":{"duration":0.035446,"end_time":"2022-03-23T20:09:24.447084","exception":false,"start_time":"2022-03-23T20:09:24.411638","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"id":"FpR2HfK93pvS","papermill":{"duration":0.025911,"end_time":"2022-03-23T20:09:24.49901","exception":false,"start_time":"2022-03-23T20:09:24.473099","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class HotelIdModel(nn.Module):\n    def __init__(self, n_classes=100, backbone_name=\"efficientnet_b0\"):\n        super(HotelIdModel, self).__init__()\n        \n        self.backbone = timm.create_model(backbone_name, num_classes=n_classes, pretrained=True)\n\n    def forward(self, x):\n        return self.backbone(x)","metadata":{"id":"_2mse3zX3pFQ","papermill":{"duration":0.033646,"end_time":"2022-03-23T20:09:24.558453","exception":false,"start_time":"2022-03-23T20:09:24.524807","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model helper functions","metadata":{"id":"mTFCinps35ci","papermill":{"duration":0.025639,"end_time":"2022-03-23T20:09:24.609989","exception":false,"start_time":"2022-03-23T20:09:24.58435","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def save_checkpoint(model, scheduler, optimizer, epoch, name, loss=None, score=None):\n    checkpoint = {\"epoch\": epoch,\n                  \"model\": model.state_dict(),\n                  \"scheduler\": scheduler.state_dict(),\n                  \"optimizer\": optimizer.state_dict(),\n                  \"loss\": loss,\n                  \"score\": score,\n                  }\n\n    torch.save(checkpoint, f\"{OUTPUT_FOLDER}checkpoint-{name}.pt\")\n\n\ndef load_checkpoint(model, scheduler, optimizer, name):\n    checkpoint = torch.load(f\"{OUTPUT_FOLDER}checkpoint-{name}.pt\")\n\n    model.load_state_dict(checkpoint[\"model\"])\n    scheduler.load_state_dict(checkpoint[\"scheduler\"])\n    return model, scheduler, optimizer, checkpoint[\"epoch\"]","metadata":{"id":"ryZ6wE0zKPiz","papermill":{"duration":0.035168,"end_time":"2022-03-23T20:09:24.671044","exception":false,"start_time":"2022-03-23T20:09:24.635876","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train functions","metadata":{}},{"cell_type":"code","source":"def train_epoch(args, model, loader, criterion, optimizer, scheduler, epoch):\n    losses = []\n    targets_all = []\n    outputs_all = []\n    \n    model.train()\n    t = tqdm(loader)\n    \n    for i, sample in enumerate(t):\n        optimizer.zero_grad()\n        \n        images = sample['image'].to(args.device)\n        targets = sample['target'].to(args.device)\n        \n        outputs = model(images)\n        loss = criterion(outputs, targets)\n        \n        loss.backward()\n        optimizer.step()\n        \n        if scheduler:\n            scheduler.step()\n                \n        losses.append(loss.item())\n        targets_all.extend(targets.cpu().numpy())\n        outputs_all.extend(torch.sigmoid(outputs).detach().cpu().numpy())\n\n        score = np.mean(targets_all == np.argmax(outputs_all, axis=1))\n        desc = f\"Epoch {epoch}/{args.epochs} - Train loss:{loss:0.4f}, accuracy: {score:0.4f}\"\n        t.set_description(desc)\n        \n    return np.mean(losses), score\n\n\ndef test(loader, model):\n    targets_all = []\n    outputs_all = []\n    \n    model.eval()\n    t = tqdm(loader)\n    \n    for i, sample in enumerate(t):\n        images = sample['image'].to(args.device)\n        targets = sample['target'].to(args.device)\n        \n        outputs = model(images)\n        \n        targets_all.extend(targets.cpu().numpy())\n        outputs_all.extend(torch.sigmoid(outputs).detach().cpu().numpy())\n        \n    \n    # repeat targets to N_MATCHES for easy calculation of MAP@5\n    y = np.repeat([targets_all], repeats=N_MATCHES, axis=0).T\n    # sort predictions and get top 5\n    preds = np.argsort(-np.array(outputs_all), axis=1)[:, :5]\n    # check if any of top 5 predictions are correct and calculate mean accuracy\n    acc_top_5 = (preds == y).any(axis=1).mean()\n    # calculate prediction accuracy\n    acc_top_1 = np.mean(targets_all == np.argmax(outputs_all, axis=1))\n\n    print(f\"Accuracy: {acc_top_1:0.4f}, MAP@5: {acc_top_5:0.4f}\")","metadata":{"id":"SntLH82s2_au","papermill":{"duration":0.040666,"end_time":"2022-03-23T20:09:24.737525","exception":false,"start_time":"2022-03-23T20:09:24.696859","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data","metadata":{"id":"F2xgmwBW4LjC","papermill":{"duration":0.025544,"end_time":"2022-03-23T20:09:24.788734","exception":false,"start_time":"2022-03-23T20:09:24.76319","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data_df = pd.read_csv(DATA_FOLDER + \"train_padding_256.csv\")\n# encode hotel ids\ndata_df[\"hotel_id_code\"] = data_df[\"hotel_id\"].astype('category').cat.codes.values.astype(np.int64)","metadata":{"id":"Sn6HrWKQ2_aw","papermill":{"duration":0.154871,"end_time":"2022-03-23T20:09:27.317597","exception":false,"start_time":"2022-03-23T20:09:27.162726","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save hotel_id encoding for later decoding\nhotel_id_code_df = data_df.drop(columns=[\"image_id\"]).drop_duplicates().reset_index(drop=True)\nhotel_id_code_df.to_csv(OUTPUT_FOLDER + 'hotel_id_code_mapping.csv', index=False)\n# hotel_id_code_map = hotel_id_code_df.set_index('hotel_id_code').to_dict()[\"hotel_id\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Quick look at data\n\nWe can see that most hotels have less than 20 images while there are few with over hundreds and one with over 1000. In case of classification weights or sampling might be useful to handle the imbalance.","metadata":{}},{"cell_type":"markdown","source":"## Image count per hotel","metadata":{}},{"cell_type":"code","source":"group_df = data_df.groupby([\"hotel_id\"]).size().to_frame(\"image_count\").sort_values(\"image_count\")[::-1].reset_index()\n\n# top and low\nlow_df = group_df.iloc[-50:]\ntop_df = group_df.iloc[:50]\n\nfig = make_subplots(rows=2, cols=2, \n                    specs=[[{\"colspan\": 2}, None], [{}, {}]],\n                    horizontal_spacing=0.02, vertical_spacing=0.2, \n                    shared_yaxes=False,\n                    subplot_titles=(\"\", \"Top 50\", \"Bottom 50\"))\n\n\nfig.add_trace(go.Scatter(x=group_df[\"hotel_id\"], y=group_df[\"image_count\"], showlegend = False), 1, 1)\nfig.add_trace(go.Bar(x=top_df[\"hotel_id\"], y=top_df[\"image_count\"], showlegend = False), 2, 1)\nfig.add_trace(go.Bar(x=low_df[\"hotel_id\"], y=low_df[\"image_count\"], showlegend = False), 2, 2)\n\nfig.update_yaxes(title_text=\"Image count\", row=1, col=1)\nfig.update_yaxes(title_text=\"Image count\", row=2, col=1)\nfig.update_xaxes(type=\"category\", visible=False, row=1, col=1)\nfig.update_xaxes(title_text=\"Hotel ID\", type=\"category\", row=2, col=1)\nfig.update_xaxes(title_text=\"Hotel ID\", type=\"category\", row=2, col=2)\n\nfig.update_layout(title=\"Image count per hotel\", height=550)\nfig.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(group_df, x=\"image_count\", nbins=100, marginal=\"box\", height=350)\nfig.update_layout(title=\"Distribution of image count per hotel\")\nfig.update_traces(hovertemplate=\"Image count: %{x} <br>Hotel count: %{y}\")\nfig.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Example of images","metadata":{}},{"cell_type":"code","source":"def show_images(ds, title_text, n_images=5):\n    fig, ax = plt.subplots(1,5, figsize=(22,8))\n    \n    ax[0].set_ylabel(title_text)\n    \n    for i in range(5):\n        d = ds.__getitem__(i)\n        ax[i].imshow(d[\"image\"].T)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train images without and with augmentations","metadata":{}},{"cell_type":"code","source":"train_dataset = HotelTrainDataset(data_df, base_transform, data_path=IMAGE_FOLDER)\nshow_images(train_dataset, 'No augmentations')\n\ntrain_dataset = HotelTrainDataset(data_df, train_transform, data_path=IMAGE_FOLDER)\nshow_images(train_dataset, 'Train augmentations')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test image example with occlusion","metadata":{}},{"cell_type":"code","source":"test_image = np.array(pil_image.open('../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/test_images/abc.jpg')).astype(np.uint8)\nplt.figure(figsize=(6,6))\nplt.imshow(test_image)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and evaluate","metadata":{"id":"EMVYKwZ64zUN","papermill":{"duration":0.035036,"end_time":"2022-03-23T20:09:27.471338","exception":false,"start_time":"2022-03-23T20:09:27.436302","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def train_and_validate(args, data_df):\n    model_name = f\"classification-model-{args.backbone_name}-{IMG_SIZE}x{IMG_SIZE}\"\n    print(model_name)\n\n    seed_everything(seed=SEED)\n\n    val_df = data_df.groupby(\"hotel_id\").sample(args.val_samples, random_state=SEED)\n    train_df = data_df[~data_df[\"image_id\"].isin(val_df[\"image_id\"])]\n\n    train_dataset = HotelTrainDataset(train_df, train_transform, data_path=IMAGE_FOLDER)\n    train_loader = DataLoader(train_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=True, drop_last=True)\n    val_dataset = HotelTrainDataset(val_df, val_transform, data_path=IMAGE_FOLDER)\n    valid_loader = DataLoader(val_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=False)\n\n    model = HotelIdModel(args.n_classes, args.backbone_name)\n    model = model.to(args.device)\n\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr)\n\n    scheduler = torch.optim.lr_scheduler.OneCycleLR(\n                    optimizer,\n                    max_lr=args.lr,\n                    epochs=args.epochs,\n                    steps_per_epoch=len(train_loader),\n                    div_factor=10,\n                    final_div_factor=1,\n                    pct_start=0.1,\n                    anneal_strategy=\"cos\",\n                )\n    \n    start_epoch = 1\n    \n    for epoch in range(start_epoch, args.epochs+1):\n        train_loss, train_score = train_epoch(args, model, train_loader, criterion, optimizer, scheduler, epoch)\n        save_checkpoint(model, scheduler, optimizer, epoch, model_name, train_loss, train_score)\n        test(valid_loader, model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Efficientnet-b0 training","metadata":{}},{"cell_type":"code","source":"%%time \n\nclass args:\n    epochs = 5\n    lr = 1e-3\n    batch_size = 64\n    num_workers = 2\n    val_samples = 1\n    backbone_name = \"efficientnet_b0\"\n    n_classes = data_df[\"hotel_id_code\"].nunique()\n    device = ('cuda' if torch.cuda.is_available() else 'cpu')\n\ntrain_and_validate(args, data_df)","metadata":{"id":"YONzJBtG2_a0","papermill":{"duration":1686.013804,"end_time":"2022-03-23T20:37:33.519992","exception":false,"start_time":"2022-03-23T20:09:27.506188","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}