{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":49349,"databundleVersionId":5447706,"sourceType":"competition"},{"sourceId":3840,"sourceType":"modelInstanceVersion","modelInstanceId":2742}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from kornia.feature import LoFTR","metadata":{"execution":{"iopub.status.busy":"2023-11-18T02:15:22.538440Z","iopub.execute_input":"2023-11-18T02:15:22.539204Z","iopub.status.idle":"2023-11-18T02:15:22.543479Z","shell.execute_reply.started":"2023-11-18T02:15:22.539166Z","shell.execute_reply":"2023-11-18T02:15:22.542510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport torch\nimport torch.nn.functional as F\nimport torchvision\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib\n\nimport kornia\nimport kornia as K\nimport kornia.feature as KF","metadata":{"execution":{"iopub.status.busy":"2023-11-18T02:15:22.545143Z","iopub.execute_input":"2023-11-18T02:15:22.545403Z","iopub.status.idle":"2023-11-18T02:15:22.558247Z","shell.execute_reply.started":"2023-11-18T02:15:22.545380Z","shell.execute_reply":"2023-11-18T02:15:22.557421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nmatcher = LoFTR(pretrained=None)\nmatcher.load_state_dict(torch.load(\"../input/loftr/pytorch/outdoor/1/loftr_outdoor.ckpt\")['state_dict'])\nmatcher = matcher.to(device).eval()","metadata":{"execution":{"iopub.status.busy":"2023-11-18T02:15:22.559484Z","iopub.execute_input":"2023-11-18T02:15:22.559900Z","iopub.status.idle":"2023-11-18T02:15:22.894158Z","shell.execute_reply.started":"2023-11-18T02:15:22.559864Z","shell.execute_reply":"2023-11-18T02:15:22.893360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file1 = \"/kaggle/input/image-matching-challenge-2023/train/phototourism/taj_mahal/images/01910465_50753376.jpg\"\nfile2 = \"/kaggle/input/image-matching-challenge-2023/train/phototourism/taj_mahal/images/02698907_6486192155.jpg\"","metadata":{"execution":{"iopub.status.busy":"2023-11-18T02:15:22.895416Z","iopub.execute_input":"2023-11-18T02:15:22.895787Z","iopub.status.idle":"2023-11-18T02:15:22.900379Z","shell.execute_reply.started":"2023-11-18T02:15:22.895755Z","shell.execute_reply":"2023-11-18T02:15:22.899356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_torch_image(fname, device):\n    img = cv2.imread(fname)\n    original_shape = img.shape\n    #scale = 920 / ((img.shape[0]+ img.shape[1]) //2)\n    scale = 1\n    round_unit = 8\n    w = int(img.shape[1] * scale)//round_unit*round_unit\n    h = int(img.shape[0] * scale)//round_unit*round_unit\n    img_resized = cv2.resize(img, (w, h))\n    img = K.image_to_tensor(img_resized, False).float() /255.\n    img = K.color.bgr_to_rgb(img)\n    return img.to(device), original_shape, img_resized\n\ndef get_mkpts_loftr(matcher, file1, file2, th_conf=0.5, device=\"cuda\"):    \n    image_1, ori_shape_1, img_resized1 = load_torch_image(file1, device)\n    image_2, ori_shape_2, img_resized2 = load_torch_image(file2, device)\n    input_dict = {\"image0\": K.color.rgb_to_grayscale(image_1), \n              \"image1\": K.color.rgb_to_grayscale(image_2)}\n    #input_dict['image1'] = torch.nn.functional.upsample(input_dict['image1'],(1536,1152))\n    #input_dict['image0'] = torch.nn.functional.upsample(input_dict['image0'],(1536,1152))\n    print(input_dict['image1'].shape)\n\n    with torch.no_grad():\n        correspondences = matcher(input_dict)\n\n    print(correspondences.keys())\n    mkpts0 = correspondences['keypoints0'].cpu().numpy()\n    mkpts1 = correspondences['keypoints1'].cpu().numpy()\n    mconf  = correspondences['confidence'].cpu().numpy()\n    \n    mkpts0 = mkpts0[ mconf >= th_conf, : ]\n    mkpts1 = mkpts1[ mconf >= th_conf, : ]\n    mconf  = mconf[ mconf >= th_conf ]\n\n    # Scaling coords to same pixel size of LoFTR\n    mkpts0[:,0] = mkpts0[:,0] * ori_shape_1[1] / image_1.shape[3]   # X\n    mkpts0[:,1] = mkpts0[:,1] * ori_shape_1[0] / image_1.shape[2]   # Y\n    mkpts1[:,0] = mkpts1[:,0] * ori_shape_2[1] / image_2.shape[3]   # X\n    mkpts1[:,1] = mkpts1[:,1] * ori_shape_2[0] / image_2.shape[2]   # Y\n    \n    return mkpts0, mkpts1, mconf","metadata":{"execution":{"iopub.status.busy":"2023-11-18T02:15:22.902586Z","iopub.execute_input":"2023-11-18T02:15:22.902873Z","iopub.status.idle":"2023-11-18T02:15:22.915453Z","shell.execute_reply.started":"2023-11-18T02:15:22.902828Z","shell.execute_reply":"2023-11-18T02:15:22.914697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\nmkpts1, mkpts2, mconf = get_mkpts_loftr(matcher, file1, file2)\nprint(mkpts1.shape, mkpts2.shape, mconf.shape)","metadata":{"execution":{"iopub.status.busy":"2023-11-18T02:15:22.916500Z","iopub.execute_input":"2023-11-18T02:15:22.916781Z","iopub.status.idle":"2023-11-18T02:15:23.270712Z","shell.execute_reply.started":"2023-11-18T02:15:22.916758Z","shell.execute_reply":"2023-11-18T02:15:23.269725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cmap = matplotlib.cm.get_cmap('rainbow')\n\nimg1 = np.array(Image.open(file1))\nimg2 = np.array(Image.open(file2))\n\n# tiled images\noverlay = np.zeros( (max(img1.shape[0], img2.shape[0]), img1.shape[1]+img2.shape[1], 3), dtype=np.uint8)\noverlay[:, 0:img1.shape[1], :] = np.array(Image.fromarray(img1).resize((img1.shape[1], overlay.shape[0])))\noverlay[:, img1.shape[1]:,  :] = np.array(Image.fromarray(img2).resize((img2.shape[1], overlay.shape[0])))\ndisplay(Image.fromarray(overlay))\nImage.fromarray(overlay).save('taj.jpg')\n\n# confidence map\nmkpts1 = mkpts1.astype(np.int64)\nmkpts2 = mkpts2.astype(np.int64)\n\nimg_conf1 = np.zeros(img1.shape, dtype=np.uint8)\nimg_conf1[mkpts1[:,1]-1, mkpts1[:,0]-1, :] = (cmap(mconf)[:, :3] * 255).astype(np.uint8)\nimg_conf2 = np.zeros(img2.shape, dtype=np.uint8)\nimg_conf2[mkpts2[:,1]-1, mkpts2[:,0]-1, :] = (cmap(mconf)[:, :3] * 255).astype(np.uint8)\n# Adjust intensity of confidence map points\nintensity_scale = 128.0  # Adjust the scale factor as needed\nimg_conf1 = np.clip(img_conf1 * intensity_scale, 0, 255).astype(np.uint8)\nimg_conf2 = np.clip(img_conf2 * intensity_scale, 0, 255).astype(np.uint8)\n\noverlay[:, 0:img1.shape[1], :] = np.array(Image.fromarray(img_conf1).resize((img1.shape[1], overlay.shape[0])))\noverlay[:, img1.shape[1]:,  :] = np.array(Image.fromarray(img_conf2).resize((img2.shape[1], overlay.shape[0])))\ndisplay(Image.fromarray(overlay))\nImage.fromarray(overlay).save('taj_loftr.jpg')\n\n\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-11-18T02:15:51.474299Z","iopub.execute_input":"2023-11-18T02:15:51.474705Z","iopub.status.idle":"2023-11-18T02:15:52.057616Z","shell.execute_reply.started":"2023-11-18T02:15:51.474673Z","shell.execute_reply":"2023-11-18T02:15:52.056685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install kornia_moons","metadata":{"execution":{"iopub.status.busy":"2023-11-18T02:16:42.337622Z","iopub.execute_input":"2023-11-18T02:16:42.338460Z","iopub.status.idle":"2023-11-18T02:16:55.299351Z","shell.execute_reply.started":"2023-11-18T02:16:42.338426Z","shell.execute_reply":"2023-11-18T02:16:55.298082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kornia_moons.viz import draw_LAF_matches\n\ndef drawMatches(mkpts1, mkpts2, file1, file2, device='cuda'):\n    img1, ori_shape_1, img_resized1 = load_torch_image(file1, device)\n    img2, ori_shape_2, img_resized2 = load_torch_image(file2, device)\n    Fm, inliers = cv2.findFundamentalMat(mkpts1, mkpts2, cv2.USAC_MAGSAC, 0.5, 0.999, 100000)\n    inliers = inliers > 0\n    draw_LAF_matches(\n        KF.laf_from_center_scale_ori(\n            torch.from_numpy(mkpts1).view(1, -1, 2),\n            torch.ones(mkpts1.shape[0]).view(1, -1, 1, 1),\n            torch.ones(mkpts1.shape[0]).view(1, -1, 1),\n        ),\n        KF.laf_from_center_scale_ori(\n            torch.from_numpy(mkpts2).view(1, -1, 2),\n            torch.ones(mkpts2.shape[0]).view(1, -1, 1, 1),\n            torch.ones(mkpts2.shape[0]).view(1, -1, 1),\n        ),\n        torch.arange(mkpts1.shape[0]).view(-1, 1).repeat(1, 2),\n        K.tensor_to_image(img1),\n        K.tensor_to_image(img2),\n        inliers,\n        draw_dict={\"inlier_color\": (0.2, 1, 0.2), \"tentative_color\": None, \"feature_color\": (0.2, 0.5, 1), \"vertical\": False},\n    )\n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-18T02:29:26.246442Z","iopub.execute_input":"2023-11-18T02:29:26.247345Z","iopub.status.idle":"2023-11-18T02:29:26.257223Z","shell.execute_reply.started":"2023-11-18T02:29:26.247305Z","shell.execute_reply":"2023-11-18T02:29:26.256126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drawMatches(mkpts1, mkpts2, file1, file2)","metadata":{"execution":{"iopub.status.busy":"2023-11-18T02:29:30.559617Z","iopub.execute_input":"2023-11-18T02:29:30.560473Z","iopub.status.idle":"2023-11-18T02:29:38.330510Z","shell.execute_reply.started":"2023-11-18T02:29:30.560439Z","shell.execute_reply":"2023-11-18T02:29:38.329607Z"},"trusted":true},"execution_count":null,"outputs":[]}]}