{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Image embedding by timm swin transformer","metadata":{}},{"cell_type":"code","source":"! pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-02-16T12:16:14.316055Z","iopub.execute_input":"2022-02-16T12:16:14.316811Z","iopub.status.idle":"2022-02-16T12:16:26.279196Z","shell.execute_reply.started":"2022-02-16T12:16:14.316673Z","shell.execute_reply":"2022-02-16T12:16:26.278123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\nimport torch\nimport torchvision.transforms as T\nimport torchvision.transforms.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision.io import read_image\nimport matplotlib.pyplot as plt\nimport glob\nimport tqdm\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-16T12:16:26.284144Z","iopub.execute_input":"2022-02-16T12:16:26.284373Z","iopub.status.idle":"2022-02-16T12:16:29.399462Z","shell.execute_reply.started":"2022-02-16T12:16:26.284344Z","shell.execute_reply":"2022-02-16T12:16:29.398292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    train_loader = {\n         'batch_size': 64,\n         'shuffle': False,\n         'num_workers': 4,\n         'pin_memory': False,\n         'drop_last': False,\n     }\n    \n    path_list = glob.glob(\"../input/h-and-m-personalized-fashion-recommendations/images/*/*.jpg\")\n    model_name = \"swin_tiny_patch4_window7_224\"\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-16T12:16:29.40174Z","iopub.execute_input":"2022-02-16T12:16:29.402182Z","iopub.status.idle":"2022-02-16T12:16:33.746378Z","shell.execute_reply.started":"2022-02-16T12:16:29.402094Z","shell.execute_reply":"2022-02-16T12:16:33.745382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HAndMImageDataset(Dataset):\n    def __init__(self, path_list, image_size=224):\n        self._path_list = path_list\n        self._transform = T.Compose([\n            # https://discuss.pytorch.org/t/convert-grayscale-images-to-rgb/113422\n            T.Lambda(lambda x: x.repeat(3, 1, 1) if x.size(0)==1 else x), \n            T.Resize([image_size, image_size])\n        ])\n            \n    def __len__(self):\n        return len(self._path_list)\n\n    def __getitem__(self, idx):\n        image_path = self._path_list[idx]\n        image = read_image(image_path)            \n        image = self._transform(image)\n        article_id = image_path.split('/')[-1].replace('.jpg', '')\n        \n        \n        return image, article_id\n","metadata":{"execution":{"iopub.status.busy":"2022-02-16T12:16:33.749371Z","iopub.execute_input":"2022-02-16T12:16:33.750171Z","iopub.status.idle":"2022-02-16T12:16:33.759372Z","shell.execute_reply.started":"2022-02-16T12:16:33.750111Z","shell.execute_reply":"2022-02-16T12:16:33.758378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = HAndMImageDataset(Config.path_list)\nloader = DataLoader(dataset, **Config.train_loader)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T12:16:33.760703Z","iopub.execute_input":"2022-02-16T12:16:33.761516Z","iopub.status.idle":"2022-02-16T12:16:33.776347Z","shell.execute_reply.started":"2022-02-16T12:16:33.76144Z","shell.execute_reply":"2022-02-16T12:16:33.775051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, article_ids = iter(loader).next()\nplt.figure(figsize=(12, 12))\nfor it, (image, article_id) in enumerate(zip(images[:16], article_ids[:16])):\n    plt.subplot(4, 4, it+1)\n    plt.imshow(image.permute(1, 2, 0))\n    plt.axis('off')\n    plt.title(f'article id: {article_id}')","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:50:32.442611Z","iopub.execute_input":"2022-02-16T13:50:32.443019Z","iopub.status.idle":"2022-02-16T13:50:46.440047Z","shell.execute_reply.started":"2022-02-16T13:50:32.442983Z","shell.execute_reply":"2022-02-16T13:50:46.439092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = timm.create_model(\n    Config.model_name, pretrained=True, num_classes=0, in_chans=3\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T12:16:50.08351Z","iopub.execute_input":"2022-02-16T12:16:50.083785Z","iopub.status.idle":"2022-02-16T12:16:56.237149Z","shell.execute_reply.started":"2022-02-16T12:16:50.083744Z","shell.execute_reply":"2022-02-16T12:16:56.236148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGENET_MEAN = [0.485, 0.456, 0.406]  # RGB\nIMAGENET_STD = [0.229, 0.224, 0.225]  # RGB\ntransform = T.Compose([\n                T.ConvertImageDtype(torch.float),\n                T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),\n            ])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T12:18:05.298326Z","iopub.execute_input":"2022-02-16T12:18:05.298981Z","iopub.status.idle":"2022-02-16T12:18:05.310253Z","shell.execute_reply.started":"2022-02-16T12:18:05.298874Z","shell.execute_reply":"2022-02-16T12:18:05.308854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda')\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T12:18:05.87747Z","iopub.execute_input":"2022-02-16T12:18:05.878279Z","iopub.status.idle":"2022-02-16T12:18:09.428183Z","shell.execute_reply.started":"2022-02-16T12:18:05.878219Z","shell.execute_reply":"2022-02-16T12:18:09.427246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2022-02-16T12:18:09.431813Z","iopub.execute_input":"2022-02-16T12:18:09.432423Z","iopub.status.idle":"2022-02-16T12:18:09.439423Z","shell.execute_reply.started":"2022-02-16T12:18:09.432378Z","shell.execute_reply":"2022-02-16T12:18:09.438459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = []\nmodel.eval()\nwith torch.no_grad():\n    for i, (images, article_ids) in enumerate(tqdm.tqdm(loader)):\n        images = images.to(device)\n        images = transform(images)\n        emb = model(images)\n        emb = emb.detach().cpu().numpy()\n\n        df = pd.DataFrame(emb)\n        df.loc[:, \"article_id\"] = article_ids\n        dfs.append(df)\n    \ndf = pd.concat(dfs)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T12:31:18.583016Z","iopub.execute_input":"2022-02-16T12:31:18.583347Z","iopub.status.idle":"2022-02-16T13:25:36.244167Z","shell.execute_reply.started":"2022-02-16T12:31:18.5833Z","shell.execute_reply":"2022-02-16T13:25:36.243154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(f\"{Config.model_name}_emb.csv.gz\", index=None)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:29:09.026887Z","iopub.execute_input":"2022-02-16T13:29:09.027424Z","iopub.status.idle":"2022-02-16T13:35:50.008029Z","shell.execute_reply.started":"2022-02-16T13:29:09.027387Z","shell.execute_reply":"2022-02-16T13:35:50.007017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the output size is same as input size\ndf.shape[0] ==  len(Config.path_list)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:51:17.317371Z","iopub.execute_input":"2022-02-16T13:51:17.317685Z","iopub.status.idle":"2022-02-16T13:51:17.325897Z","shell.execute_reply.started":"2022-02-16T13:51:17.317653Z","shell.execute_reply":"2022-02-16T13:51:17.324753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:58.691789Z","iopub.execute_input":"2022-02-16T13:35:58.692947Z","iopub.status.idle":"2022-02-16T13:35:58.745372Z","shell.execute_reply.started":"2022-02-16T13:35:58.692864Z","shell.execute_reply":"2022-02-16T13:35:58.744409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check nearest neighbor ","metadata":{}},{"cell_type":"code","source":"from annoy import AnnoyIndex\nf = 768\nt = AnnoyIndex(f, 'angular')  # Length of item vector that will be indexed\nfor i, v in tqdm.tqdm(enumerate(df[list(range(f))].values), total=len(df)):\n    t.add_item(i, v)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:41:22.020671Z","iopub.execute_input":"2022-02-16T13:41:22.021669Z","iopub.status.idle":"2022-02-16T13:41:43.91729Z","shell.execute_reply.started":"2022-02-16T13:41:22.021625Z","shell.execute_reply":"2022-02-16T13:41:43.916087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t.build(10) # 10 trees","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:09.471793Z","iopub.execute_input":"2022-02-16T13:42:09.472434Z","iopub.status.idle":"2022-02-16T13:42:13.951742Z","shell.execute_reply.started":"2022-02-16T13:42:09.472397Z","shell.execute_reply":"2022-02-16T13:42:13.950807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 1\nprint(df[\"article_id\"].iloc[i])\nnns = t.get_nns_by_item(i, 10)\n\nnn_article_ids = df[\"article_id\"].iloc[nns]\nnn_paths = [f\"../input/h-and-m-personalized-fashion-recommendations/images/{article_id[:3]}/{article_id}.jpg\" \n            for article_id in nn_article_ids]\n\n\nplt.figure(figsize=(12, 12))\nfor it, (path, article_id) in enumerate(zip(nn_paths, nn_article_ids)):\n    image = read_image(path)\n    plt.subplot(4, 4, it+1)\n    plt.imshow(image.permute(1, 2, 0))\n    plt.axis('off')\n    plt.title(f'article id: {article_id}')","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:49:50.096524Z","iopub.execute_input":"2022-02-16T13:49:50.096829Z","iopub.status.idle":"2022-02-16T13:49:53.708575Z","shell.execute_reply.started":"2022-02-16T13:49:50.096795Z","shell.execute_reply":"2022-02-16T13:49:53.707412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 2\nprint(df[\"article_id\"].iloc[i])\nnns = t.get_nns_by_item(i, 10)\n\nnn_article_ids = df[\"article_id\"].iloc[nns]\nnn_paths = [f\"../input/h-and-m-personalized-fashion-recommendations/images/{article_id[:3]}/{article_id}.jpg\" \n            for article_id in nn_article_ids]\n\n\nplt.figure(figsize=(12, 12))\nfor it, (path, article_id) in enumerate(zip(nn_paths, nn_article_ids)):\n    image = read_image(path)\n    plt.subplot(4, 4, it+1)\n    plt.imshow(image.permute(1, 2, 0))\n    plt.axis('off')\n    plt.title(f'article id: {article_id}')","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:49:53.710984Z","iopub.execute_input":"2022-02-16T13:49:53.711219Z","iopub.status.idle":"2022-02-16T13:49:57.861824Z","shell.execute_reply.started":"2022-02-16T13:49:53.711189Z","shell.execute_reply":"2022-02-16T13:49:57.860836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 3\nprint(df[\"article_id\"].iloc[i])\nnns = t.get_nns_by_item(i, 10)\n\nnn_article_ids = df[\"article_id\"].iloc[nns]\nnn_paths = [f\"../input/h-and-m-personalized-fashion-recommendations/images/{article_id[:3]}/{article_id}.jpg\" \n            for article_id in nn_article_ids]\n\n\nplt.figure(figsize=(12, 12))\nfor it, (path, article_id) in enumerate(zip(nn_paths, nn_article_ids)):\n    image = read_image(path)\n    plt.subplot(4, 4, it+1)\n    plt.imshow(image.permute(1, 2, 0))\n    plt.axis('off')\n    plt.title(f'article id: {article_id}')","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:49:34.443838Z","iopub.execute_input":"2022-02-16T13:49:34.444504Z","iopub.status.idle":"2022-02-16T13:49:38.453566Z","shell.execute_reply.started":"2022-02-16T13:49:34.444467Z","shell.execute_reply":"2022-02-16T13:49:38.452294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}