{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:52:59.241015Z","iopub.execute_input":"2022-07-16T18:52:59.241527Z","iopub.status.idle":"2022-07-16T18:53:12.418914Z","shell.execute_reply.started":"2022-07-16T18:52:59.241400Z","shell.execute_reply":"2022-07-16T18:53:12.417196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn\nimport os\nimport timm\nfrom torchvision import transforms as T\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom kaggle_datasets import KaggleDatasets\nimport glob\nimport os\nimport pandas as pd\nimport numpy as np\nimport plotly.express as px\nfrom PIL import Image\n# RAPIDS\nimport cudf, cupy, cuml\nfrom cuml.neighbors import NearestNeighbors\nfrom cuml.manifold import TSNE, UMAP\nfrom cuml import PCA\n\nfrom tqdm import tqdm\nimport random\nimport cv2\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:53:12.421944Z","iopub.execute_input":"2022-07-16T18:53:12.422436Z","iopub.status.idle":"2022-07-16T18:53:22.045662Z","shell.execute_reply.started":"2022-07-16T18:53:12.422367Z","shell.execute_reply":"2022-07-16T18:53:22.044211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=0):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\nseed_everything()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:53:22.048958Z","iopub.execute_input":"2022-07-16T18:53:22.049866Z","iopub.status.idle":"2022-07-16T18:53:22.059620Z","shell.execute_reply.started":"2022-07-16T18:53:22.049819Z","shell.execute_reply":"2022-07-16T18:53:22.058291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"timm.list_models()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-16T18:53:22.064662Z","iopub.execute_input":"2022-07-16T18:53:22.065860Z","iopub.status.idle":"2022-07-16T18:53:22.101212Z","shell.execute_reply.started":"2022-07-16T18:53:22.065797Z","shell.execute_reply":"2022-07-16T18:53:22.099642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyModel(nn.Module):\n    def __init__(self, model_name: str, embedding_size: int = 64):\n        super().__init__()\n        self.backbone = timm.create_model(model_name, pretrained=True, num_classes=0)\n        self.head = nn.Sequential(\n            nn.Linear(self.backbone.num_features, embedding_size),\n            nn.LayerNorm(embedding_size),\n            nn.GELU(),\n        )\n\n    def forward(self, x):\n        x = T.functional.resize(x,size=[224, 224])\n        x = x / 255.0\n        x = T.functional.normalize(x, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n        out = self.head(self.backbone(x))\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:53:22.103770Z","iopub.execute_input":"2022-07-16T18:53:22.105379Z","iopub.status.idle":"2022-07-16T18:53:22.115643Z","shell.execute_reply.started":"2022-07-16T18:53:22.105331Z","shell.execute_reply":"2022-07-16T18:53:22.114357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MyModel(\n    model_name=\"swin_large_patch4_window7_224\",\n    embedding_size=64)\n\nmodel.eval()\nsaved_model = torch.jit.script(model)\nsaved_model.save(\"saved_model.pt\")","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:53:22.119470Z","iopub.execute_input":"2022-07-16T18:53:22.120792Z","iopub.status.idle":"2022-07-16T18:53:31.363279Z","shell.execute_reply.started":"2022-07-16T18:53:22.120757Z","shell.execute_reply":"2022-07-16T18:53:31.361576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.autograd.profiler as profiler\n# Model loading.\nmodel = torch.jit.load(\"saved_model.pt\")\nmodel.eval()\nembedding_fn = model\n\ninput_tensor = torch.rand(3, 224, 224)\ninput_batch = input_tensor.unsqueeze(0)\n\n# warm-up\nwith torch.no_grad():\n    embedding = torch.flatten(embedding_fn(input_batch)[0]).data.numpy()\n\n\nwith torch.no_grad():\n    with profiler.profile(with_stack=True, profile_memory=True) as prof:\n        embedding = torch.flatten(embedding_fn(input_batch)[0]).data.numpy()\nprint(prof.key_averages(group_by_stack_n=5).table(sort_by='self_cpu_time_total', row_limit=2))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:53:56.583597Z","iopub.execute_input":"2022-07-16T18:53:56.584054Z","iopub.status.idle":"2022-07-16T18:54:02.810926Z","shell.execute_reply.started":"2022-07-16T18:53:56.584005Z","shell.execute_reply":"2022-07-16T18:54:02.809292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip ./submission.zip ./saved_model.pt","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:54:10.207823Z","iopub.execute_input":"2022-07-16T18:54:10.208214Z","iopub.status.idle":"2022-07-16T18:55:00.452143Z","shell.execute_reply.started":"2022-07-16T18:54:10.208183Z","shell.execute_reply":"2022-07-16T18:55:00.450457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:55:00.456010Z","iopub.execute_input":"2022-07-16T18:55:00.457014Z","iopub.status.idle":"2022-07-16T18:55:01.526982Z","shell.execute_reply.started":"2022-07-16T18:55:00.456961Z","shell.execute_reply":"2022-07-16T18:55:01.525524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EVAL_DIR = \"../input/caltech256/256_ObjectCategories/256_ObjectCategories\"","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:55:01.529977Z","iopub.execute_input":"2022-07-16T18:55:01.530822Z","iopub.status.idle":"2022-07-16T18:55:01.538307Z","shell.execute_reply.started":"2022-07-16T18:55:01.530766Z","shell.execute_reply":"2022-07-16T18:55:01.536673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n... BASIC DATA SETUP STARTING ...\\n\\n\")\n\neval_df = pd.DataFrame({\"img_path\": glob.glob(os.path.join(EVAL_DIR, \"**\", \"*.jpg\"))})\n\nADD_SHAPE_INFO=False\nif ADD_SHAPE_INFO:\n    eval_df[\"img_shape\"] = eval_df[\"img_path\"].progress_apply(lambda x: Image.open(x).size)\n\neval_df[\"label\"] = eval_df.img_path.apply(lambda x: x.rsplit(\"/\", 2)[1])\neval_df[\"label_str\"] = eval_df[\"label\"].apply(lambda x: x.split(\".\")[1])\neval_df[\"label_int\"] = eval_df[\"label\"].apply(lambda x: x.split(\".\")[0]).astype(int)\neval_df[\"label_int_0_offset\"] = eval_df.label_int-1\n\ndisplay(eval_df)\n\nfig = px.histogram(eval_df, \"label\", color=\"label\", title=\"<b>CalTech256 Class Distribution</b>\")\nfig.update_layout(showlegend=False)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:55:01.542465Z","iopub.execute_input":"2022-07-16T18:55:01.543439Z","iopub.status.idle":"2022-07-16T18:55:10.174227Z","shell.execute_reply.started":"2022-07-16T18:55:01.543392Z","shell.execute_reply":"2022-07-16T18:55:10.172965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_CLASSES = eval_df.label.nunique()\nS2I_CLASS_MAP = eval_df[[\"label_str\", \"label_int_0_offset\"]].groupby(\"label_str\").first()[\"label_int_0_offset\"].to_dict()\nI2S_CLASS_MAP = {v:k for k,v in S2I_CLASS_MAP.items()}","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:55:10.175540Z","iopub.execute_input":"2022-07-16T18:55:10.176032Z","iopub.status.idle":"2022-07-16T18:55:10.207435Z","shell.execute_reply.started":"2022-07-16T18:55:10.175979Z","shell.execute_reply":"2022-07-16T18:55:10.206279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def flatten_l_o_l(nested_list):\n    \"\"\" Flatten a list of lists \"\"\"\n    return [item for sublist in nested_list for item in sublist]\n\ndef print_ln(symbol=\"-\", line_len=110):\n    print(symbol*line_len)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:55:10.209410Z","iopub.execute_input":"2022-07-16T18:55:10.210212Z","iopub.status.idle":"2022-07-16T18:55:10.216935Z","shell.execute_reply.started":"2022-07-16T18:55:10.210168Z","shell.execute_reply":"2022-07-16T18:55:10.215752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class CaltechDataset(Dataset):\n#     def __init__(self, img_paths, labels):\n#         self.img_paths = img_paths\n#         self.labels = labels\n#         self.to_tensor = T.ToTensor()\n        \n#     def __len__(self):\n#         return len(self.img_paths)\n    \n#     def __getitem__(self, idx):\n#         img_path, label = self.img_paths[idx], self.labels[idx]\n#         img = Image.open(img_path).convert(\"RGB\").resize((224, 224))\n#         return self.to_tensor(img), label\n    \n# eval_ds = CaltechDataset(eval_df.img_path.values, eval_df.label_int.values)\n# eval_loader = DataLoader(\n#     eval_ds,\n#     batch_size=1,\n#     num_workers=os.cpu_count(),\n#     pin_memory=True,\n#     shuffle=False\n# )\n# len(eval_ds)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:55:10.219366Z","iopub.execute_input":"2022-07-16T18:55:10.220383Z","iopub.status.idle":"2022-07-16T18:55:10.241287Z","shell.execute_reply.started":"2022-07-16T18:55:10.220191Z","shell.execute_reply":"2022-07-16T18:55:10.239703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# eval_embeddings = []\n# model = model.eval().to(\"cuda\")\n# with torch.inference_mode():\n#     for data, label in tqdm(eval_loader):\n#         pred_embedding = model(data.to(\"cuda\"))\n#         eval_embeddings.append(pred_embedding.cpu().numpy().squeeze(0))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T18:55:10.243572Z","iopub.execute_input":"2022-07-16T18:55:10.244557Z","iopub.status.idle":"2022-07-16T19:08:24.997352Z","shell.execute_reply.started":"2022-07-16T18:55:10.244510Z","shell.execute_reply":"2022-07-16T19:08:24.995823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# eval_embeddings = np.array(eval_embeddings)\n# eval_embeddings.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-16T19:08:25.002166Z","iopub.execute_input":"2022-07-16T19:08:25.002510Z","iopub.status.idle":"2022-07-16T19:08:25.060195Z","shell.execute_reply.started":"2022-07-16T19:08:25.002476Z","shell.execute_reply":"2022-07-16T19:08:25.058304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# n_neighbors = 10\n# knn_model = NearestNeighbors(n_neighbors=n_neighbors)\n# knn_model.fit(eval_embeddings)\n# distances, indices = knn_model.kneighbors(eval_embeddings)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T19:08:25.062197Z","iopub.execute_input":"2022-07-16T19:08:25.062994Z","iopub.status.idle":"2022-07-16T19:08:26.248610Z","shell.execute_reply.started":"2022-07-16T19:08:25.062949Z","shell.execute_reply":"2022-07-16T19:08:26.246937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def cv2_load_image(img_path):\n#     img = cv2.resize(cv2.imread(img_path), (224, 224))\n#     img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n#     return img\n\n# def visualize_embedding_similarity(df, idx, _distances, _indices, plot_images=True):\n#     print(\"\\n... VISUALIZING SIMILAR EMBEDDING PREDICTIONS FOR THIS EXAMPLE ROW ...\\n\")\n#     display(df.iloc[idx:(idx+1)])\n    \n#     print(\"\\n... RETRIEVING DISTANCES, CLOSEST INDICES, LABELS AND GROUND TRUTH LABEL ...\\n\")\n#     demo_dist, demo_ids = _distances[idx], _indices[idx]\n#     demo_img_paths = df.iloc[demo_ids].img_path.values\n#     demo_int_lbls, demo_str_lbls = df.iloc[demo_ids].label_int.values, df.iloc[demo_ids].label_str.values\n#     gt_demo_int_lbl, gt_demo_str_lbl = demo_int_lbls[0], demo_str_lbls[0]\n    \n#     print(\"\\n\\n... PLOTTING LINE/SCATTER TO SHOW NEIGHBOR DISTANCES RELATIVE TO EXAMPLE ROW ...\\n\")\n#     fig = px.line(x=[str(x) for x in demo_ids], y=demo_dist, markers=True,\n#                  labels={\"x\":\"<b>Row ID of Neighbor</b>\", \"y\":\"<b>Distance From Ground Truth Image</b>\"},\n#                  hover_data={\"SAME LABEL AS GT\":[demo_lbl==gt_demo_int_lbl for demo_lbl in demo_int_lbls], \n#                              \"ACTUAL INTEGER LABEL\":demo_int_lbls,\n#                              \"ACTUAL STRING LABEL\":demo_str_lbls},\n#                  title=f\"<b>Example For Row {idx} in the Eval Dataframe</b>\")\n#     fig.show()\n    \n#     if plot_images:\n#         print(\"\\n\\n... PLOTTING ORIGINAL AND 9 CLOSEST IMAGES ...\\n\")\n#         plt.figure(figsize=(20,10))\n#         for i, img_path in enumerate(demo_img_paths):\n#             plt.subplot(2,5,i+1)\n#             plt.axis(False)\n#             plt.title(\"Example/Seed Image\" if i==0 else f\"#{i} Closest Neighbor (Δ={demo_dist[i]:.2f})\", fontweight=\"bold\")\n#             plt.imshow(cv2_load_image(img_path)/255.)\n#         plt.tight_layout()\n#         plt.show()\n        \n#     print(\"\\n\\n... DATAFRAME SHOWING THE EXAMPLE/SEED ROW AND 9 CLOSEST NEIGHBORS ...\\n\")\n#     display(df.iloc[demo_ids])\n    \n# n_examples = 10\n\n# random_ids = random.sample(range(len(eval_df)), n_examples)\n# for _i in random_ids:\n#     print(\"\\n\\n\")\n#     print_ln()\n#     print(f\"\\n... EXAMPLE #{_i+1} ...\\n\")\n#     print_ln()\n#     print(\"\\n\")\n#     visualize_embedding_similarity(eval_df, idx=_i, _distances=distances, _indices=indices)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T19:08:26.250556Z","iopub.execute_input":"2022-07-16T19:08:26.251082Z","iopub.status.idle":"2022-07-16T19:08:42.510948Z","shell.execute_reply.started":"2022-07-16T19:08:26.251033Z","shell.execute_reply":"2022-07-16T19:08:42.509764Z"},"trusted":true},"execution_count":null,"outputs":[]}]}