{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from kornia.feature import LoFTR","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:17.996398Z","iopub.execute_input":"2023-09-16T17:09:17.996688Z","iopub.status.idle":"2023-09-16T17:09:31.293508Z","shell.execute_reply.started":"2023-09-16T17:09:17.996663Z","shell.execute_reply":"2023-09-16T17:09:31.292583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib\n\nimport kornia\nimport kornia as K\nimport kornia.feature as KF","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:19:13.030058Z","iopub.execute_input":"2023-09-16T17:19:13.03041Z","iopub.status.idle":"2023-09-16T17:19:13.035474Z","shell.execute_reply.started":"2023-09-16T17:19:13.030381Z","shell.execute_reply":"2023-09-16T17:19:13.034472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nmatcher = LoFTR(pretrained=None)\nmatcher.load_state_dict(torch.load(\"../input/loftr/pytorch/outdoor/1/loftr_outdoor.ckpt\")['state_dict'])\nmatcher = matcher.to(device).eval()","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:19:13.304147Z","iopub.execute_input":"2023-09-16T17:19:13.304862Z","iopub.status.idle":"2023-09-16T17:19:13.932838Z","shell.execute_reply.started":"2023-09-16T17:19:13.304825Z","shell.execute_reply":"2023-09-16T17:19:13.931815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file1 = \"../input/image-matching-challenge-2023/train/phototourism/sagrada_familia/images/05907640_5576489463.jpg\"\nfile2 = \"../input/image-matching-challenge-2023/train/phototourism/sagrada_familia/images/12447860_7511693888.jpg\"","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:19:14.136315Z","iopub.execute_input":"2023-09-16T17:19:14.137061Z","iopub.status.idle":"2023-09-16T17:19:14.141282Z","shell.execute_reply.started":"2023-09-16T17:19:14.137033Z","shell.execute_reply":"2023-09-16T17:19:14.140309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_torch_image(fname, device):\n    img = cv2.imread(fname)\n    original_shape = img.shape\n    #scale = 920 / ((img.shape[0]+ img.shape[1]) //2)\n    scale = 1.0\n    round_unit = 8\n    w = int(img.shape[1] * scale)//round_unit*round_unit\n    h = int(img.shape[0] * scale)//round_unit*round_unit\n    img_resized = cv2.resize(img, (w, h))\n    img = K.image_to_tensor(img_resized, False).float() /255.\n    img = K.color.bgr_to_rgb(img)\n    return img.to(device), original_shape, img_resized\n\ndef get_mkpts_loftr(matcher, file1, file2, th_conf=0.5, device=\"cuda\"):    \n    image_1, ori_shape_1, img_resized1 = load_torch_image(file1, device)\n    image_2, ori_shape_2, img_resized2 = load_torch_image(file2, device)\n    input_dict = {\"image0\": K.color.rgb_to_grayscale(image_1), \n              \"image1\": K.color.rgb_to_grayscale(image_2)}\n\n    with torch.no_grad():\n        correspondences = matcher(input_dict)\n\n    print(correspondences.keys())\n    mkpts0 = correspondences['keypoints0'].cpu().numpy()\n    mkpts1 = correspondences['keypoints1'].cpu().numpy()\n    mconf  = correspondences['confidence'].cpu().numpy()\n    \n    mkpts0 = mkpts0[ mconf >= th_conf, : ]\n    mkpts1 = mkpts1[ mconf >= th_conf, : ]\n    mconf  = mconf[ mconf >= th_conf ]\n\n    # Scaling coords to same pixel size of LoFTR\n    mkpts0[:,0] = mkpts0[:,0] * ori_shape_1[1] / image_1.shape[3]   # X\n    mkpts0[:,1] = mkpts0[:,1] * ori_shape_1[0] / image_1.shape[2]   # Y\n    mkpts1[:,0] = mkpts1[:,0] * ori_shape_2[1] / image_2.shape[3]   # X\n    mkpts1[:,1] = mkpts1[:,1] * ori_shape_2[0] / image_2.shape[2]   # Y\n    \n    return mkpts0, mkpts1, mconf","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:22:50.101725Z","iopub.execute_input":"2023-09-16T17:22:50.102092Z","iopub.status.idle":"2023-09-16T17:22:50.115527Z","shell.execute_reply.started":"2023-09-16T17:22:50.102063Z","shell.execute_reply":"2023-09-16T17:22:50.114532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmkpts1, mkpts2, mconf = get_mkpts_loftr(matcher, file1, file2)\nprint(mkpts1.shape, mkpts2.shape, mconf.shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:22:50.717661Z","iopub.execute_input":"2023-09-16T17:22:50.718025Z","iopub.status.idle":"2023-09-16T17:22:51.128231Z","shell.execute_reply.started":"2023-09-16T17:22:50.717997Z","shell.execute_reply":"2023-09-16T17:22:51.126558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cmap = matplotlib.cm.get_cmap('rainbow')\n\nimg1 = np.array(Image.open(file1))\nimg2 = np.array(Image.open(file2))\n\n# tiled images\noverlay = np.zeros( (max(img1.shape[0], img2.shape[0]), img1.shape[1]+img2.shape[1], 3), dtype=np.uint8)\noverlay[:, 0:img1.shape[1], :] = np.array(Image.fromarray(img1).resize((img1.shape[1], overlay.shape[0])))\noverlay[:, img1.shape[1]:,  :] = np.array(Image.fromarray(img2).resize((img2.shape[1], overlay.shape[0])))\ndisplay(Image.fromarray(overlay))\n\n# confidence map\nmkpts1 = mkpts1.astype(np.int64)\nmkpts2 = mkpts2.astype(np.int64)\n\nimg_conf1 = np.zeros(img1.shape, dtype=np.uint8)\nimg_conf1[mkpts1[:,1]-1, mkpts1[:,0]-1, :] = (cmap(mconf)[:, :3] * 255).astype(np.uint8)\nimg_conf2 = np.zeros(img2.shape, dtype=np.uint8)\nimg_conf2[mkpts2[:,1]-1, mkpts2[:,0]-1, :] = (cmap(mconf)[:, :3] * 255).astype(np.uint8)\noverlay[:, 0:img1.shape[1], :] = np.array(Image.fromarray(img_conf1).resize((img1.shape[1], overlay.shape[0])))\noverlay[:, img1.shape[1]:,  :] = np.array(Image.fromarray(img_conf2).resize((img2.shape[1], overlay.shape[0])))\ndisplay(Image.fromarray(overlay))\n","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:22:51.51262Z","iopub.execute_input":"2023-09-16T17:22:51.512971Z","iopub.status.idle":"2023-09-16T17:22:52.296693Z","shell.execute_reply.started":"2023-09-16T17:22:51.512942Z","shell.execute_reply":"2023-09-16T17:22:52.295614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}