{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install kornia_moons","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-13T23:26:54.924192Z","iopub.execute_input":"2023-06-13T23:26:54.924945Z","iopub.status.idle":"2023-06-13T23:27:07.732306Z","shell.execute_reply.started":"2023-06-13T23:26:54.924897Z","shell.execute_reply":"2023-06-13T23:27:07.731086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import kornia as K\nimport kornia.feature as KF\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\nimport cv2\nfrom kornia_moons.viz import draw_LAF_matches, visualize_LAF","metadata":{"execution":{"iopub.status.busy":"2023-06-13T23:27:07.735847Z","iopub.execute_input":"2023-06-13T23:27:07.736969Z","iopub.status.idle":"2023-06-13T23:27:19.761835Z","shell.execute_reply.started":"2023-06-13T23:27:07.736929Z","shell.execute_reply":"2023-06-13T23:27:19.760673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T23:27:19.763996Z","iopub.execute_input":"2023-06-13T23:27:19.764291Z","iopub.status.idle":"2023-06-13T23:27:19.798933Z","shell.execute_reply.started":"2023-06-13T23:27:19.764260Z","shell.execute_reply":"2023-06-13T23:27:19.797172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_torch_image(fname, device=torch.device('cpu'), size = 840):\n    img = cv2.imread(fname)\n    scale = size / max(img.shape[0], img.shape[1]) \n    w = int(img.shape[1] * scale)\n    h = int(img.shape[0] * scale)\n    img = cv2.resize(img, (w, h))\n    img = K.image_to_tensor(img, False).float() /255.\n    img = K.color.bgr_to_rgb(img)\n    return img.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T23:27:19.802461Z","iopub.execute_input":"2023-06-13T23:27:19.802750Z","iopub.status.idle":"2023-06-13T23:27:19.813044Z","shell.execute_reply.started":"2023-06-13T23:27:19.802724Z","shell.execute_reply":"2023-06-13T23:27:19.812084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def detect_and_show_keynet(fname1,fname2, model):\n    resize = 1280\n    img1 = load_torch_image(fname1, device, size = resize)\n    img2 = load_torch_image(fname2, device, size = resize)\n\n    input_dict = {\n        \"image0\": K.color.rgb_to_grayscale(img1),\n        \"image1\": K.color.rgb_to_grayscale(img2),\n    }\n\n    hw1 = torch.tensor(img1.shape[2:])\n    hw2 = torch.tensor(img1.shape[2:])\n\n    adalam_config = KF.adalam.get_adalam_default_config()\n    adalam_config[\"force_seed_mnn\"] = False\n    adalam_config[\"search_expansion\"] = 16\n    adalam_config[\"ransac_iters\"] = 256\n    with torch.inference_mode():\n        lafs1, resps1, descs1 = model(K.color.rgb_to_grayscale(img1))\n        lafs2, resps2, descs2 = model(K.color.rgb_to_grayscale(img2))\n        print(\"Img 1 number of kpts = \", lafs1.shape)\n        print(\"Img 2 number of kpts = \", lafs2.shape)\n\n        dists, idxs = KF.match_adalam(\n            descs1.squeeze(0),\n            descs2.squeeze(0),\n            lafs1,\n            lafs2,  # Adalam takes into account also geometric information\n            config=adalam_config,\n            hw1=hw1,\n            hw2=hw2,  # Adalam also benefits from knowing image size\n        )\n    print(f\"{idxs.shape[0]} tentative matches with AdaLAM\")\n\n    print(\"Mean dist = :\", dists.mean().cpu().numpy())\n    bins = np.histogram_bin_edges(np.linspace(0, 1, num=11))\n    plt.hist(dists.flatten(), bins)\n    plt.title(\"Matching distance distribution\")\n    plt.show()\n    mkpts1 = KF.get_laf_center(lafs1).squeeze()[idxs[:, 0]].detach().cpu().numpy()\n    mkpts2 = KF.get_laf_center(lafs2).squeeze()[idxs[:, 1]].detach().cpu().numpy()\n\n    Fm, inliers = cv2.findFundamentalMat(mkpts1, mkpts2, cv2.USAC_MAGSAC, 5, 0.999, 100000)\n    inliers = inliers > 0\n    \n \n    draw_LAF_matches(\n    lafs1.cpu(),\n    lafs2.cpu(),\n    idxs.cpu(),\n    K.tensor_to_image(img1.cpu()),\n    K.tensor_to_image(img2.cpu()),\n    inliers,\n    draw_dict={\"inlier_color\": (0.2, 1, 0.2, 0.5), \"tentative_color\": (1, 1, 0.2, 0.3), \"feature_color\": None, \"vertical\": False},\n    )","metadata":{"execution":{"iopub.status.busy":"2023-06-13T23:27:19.816301Z","iopub.execute_input":"2023-06-13T23:27:19.816649Z","iopub.status.idle":"2023-06-13T23:27:19.830631Z","shell.execute_reply.started":"2023-06-13T23:27:19.816625Z","shell.execute_reply":"2023-06-13T23:27:19.829624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fname1 = \"/kaggle/input/image-matching-challenge-2023/train/heritage/cyprus/images/DSC_6480.JPG\"\nfname2 = \"/kaggle/input/image-matching-challenge-2023/train/heritage/cyprus/images/DSC_6488.JPG\"\n\nkeynet = KF.KeyNetAffNetHardNet(8000, False).eval().to(device)\ndetect_and_show_keynet(fname1,fname2, keynet)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T23:27:19.831969Z","iopub.execute_input":"2023-06-13T23:27:19.833889Z","iopub.status.idle":"2023-06-13T23:27:38.171185Z","shell.execute_reply.started":"2023-06-13T23:27:19.833863Z","shell.execute_reply":"2023-06-13T23:27:38.169752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Above images show that KeyNetAffNetHardNet + Adalam can properly handle rotated features matching with low mean distance.","metadata":{}},{"cell_type":"code","source":"fname1 = \"/kaggle/input/image-matching-challenge-2023/train/heritage/cyprus/images/DSC_6480.JPG\"\nfname2 = \"/kaggle/input/image-matching-challenge-2023/train/heritage/cyprus/images/DSC_6488.JPG\"\ngftt = KF.GFTTAffNetHardNet(8000, False).eval().to(device)\n\ndetect_and_show_keynet(fname1,fname2, gftt)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T23:27:38.172536Z","iopub.execute_input":"2023-06-13T23:27:38.173581Z","iopub.status.idle":"2023-06-13T23:27:44.596596Z","shell.execute_reply.started":"2023-06-13T23:27:38.173539Z","shell.execute_reply":"2023-06-13T23:27:44.595292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Above images show that GFTTAffNetHardNet and KeyNetAffNetHardNet give different keypoints.","metadata":{}},{"cell_type":"code","source":"fname1 = \"/kaggle/input/image-matching-challenge-2023/train/heritage/cyprus/images/DSC_6508.JPG\"\nfname2 = \"/kaggle/input/image-matching-challenge-2023/train/heritage/cyprus/images/DSC_6480.JPG\"\n\nkeynet = KF.KeyNetAffNetHardNet(8000, False).eval().to(device)\ndetect_and_show_keynet(fname1,fname2, keynet)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T23:27:44.598010Z","iopub.execute_input":"2023-06-13T23:27:44.598956Z","iopub.status.idle":"2023-06-13T23:27:56.830916Z","shell.execute_reply.started":"2023-06-13T23:27:44.598922Z","shell.execute_reply":"2023-06-13T23:27:56.830055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Above images show that the mean matching distance would be close to 1 when the two images don't overlap.","metadata":{}},{"cell_type":"code","source":"fname1 = \"/kaggle/input/image-matching-challenge-2023/train/heritage/wall/images/DSC_4946_acr.jpg\"\nfname2 = \"/kaggle/input/image-matching-challenge-2023/train/heritage/wall/images/DSC_5042_acr.jpg\"\n\nkeynet = KF.KeyNetAffNetHardNet(8000, False).eval().to(device)\ndetect_and_show_keynet(fname1,fname2, keynet)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T23:27:56.832331Z","iopub.execute_input":"2023-06-13T23:27:56.832866Z","iopub.status.idle":"2023-06-13T23:28:02.748789Z","shell.execute_reply.started":"2023-06-13T23:27:56.832834Z","shell.execute_reply":"2023-06-13T23:28:02.747714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Above images show the weakness of using KeyNetAffNetHardNet + Adalam distance threshold for pairing: False positive pairing caused by repetitive patterns.","metadata":{}}]}