{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71885,"databundleVersionId":8015523,"sourceType":"competition"},{"sourceId":3414836,"sourceType":"datasetVersion","datasetId":2058261}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from PIL import Image, ImageDraw\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib\n\nimport kornia\nimport kornia as K\nimport kornia.feature as KF\n\n#---------------\n# For SuperGlue\nimport sys\nsys.path.append(\"../input/super-glue-pretrained-network\")\nfrom models.matching import Matching\nfrom models.utils import (compute_pose_error, compute_epipolar_error,\n                          estimate_pose, make_matching_plot,\n                          error_colormap, AverageTimer, pose_auc, read_image,\n                          process_resize, frame2tensor,\n                          rotate_intrinsics, rotate_pose_inplane,\n                          scale_intrinsics)\n\nfrom torch.nn import functional as torchF  # For resizing tensor","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:43:41.263901Z","iopub.execute_input":"2024-03-26T16:43:41.264831Z","iopub.status.idle":"2024-03-26T16:43:49.480885Z","shell.execute_reply.started":"2024-03-26T16:43:41.264797Z","shell.execute_reply":"2024-03-26T16:43:49.479932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SuperGlue model\nhttps://github.com/magicleap/SuperGluePretrainedNetwork?tab=readme-ov-file\n![superglue](https://github.com/magicleap/SuperGluePretrainedNetwork/blob/master/assets/teaser.png?raw=true)","metadata":{}},{"cell_type":"code","source":"sg_config = {\n    \"superpoint\": {\n        \"nms_radius\": 4, \n        \"keypoint_threshold\": 0.005,\n        \"max_keypoints\": 2048\n    },\n    \"superglue\": {\n        \"weights\": \"outdoor\",\n        \"sinkhorn_iterations\": 10,\n        \"match_threshold\": 0.2,\n    }\n}","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:43:49.482815Z","iopub.execute_input":"2024-03-26T16:43:49.483402Z","iopub.status.idle":"2024-03-26T16:43:49.488611Z","shell.execute_reply.started":"2024-03-26T16:43:49.483345Z","shell.execute_reply":"2024-03-26T16:43:49.487553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nmatcher = Matching(sg_config).eval().to(device)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:43:49.489878Z","iopub.execute_input":"2024-03-26T16:43:49.490215Z","iopub.status.idle":"2024-03-26T16:43:50.525757Z","shell.execute_reply.started":"2024-03-26T16:43:49.490148Z","shell.execute_reply":"2024-03-26T16:43:50.524660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Demo","metadata":{}},{"cell_type":"code","source":"file1 = \"/kaggle/input/image-matching-challenge-2024/test/church/images/00001.png\"\nfile2 = \"/kaggle/input/image-matching-challenge-2024/test/church/images/00006.png\"","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:43:50.527843Z","iopub.execute_input":"2024-03-26T16:43:50.528138Z","iopub.status.idle":"2024-03-26T16:43:50.533137Z","shell.execute_reply.started":"2024-03-26T16:43:50.528112Z","shell.execute_reply":"2024-03-26T16:43:50.531790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def f_preprocess(image, device, resize, rotation, resize_float, w_new=None, h_new=None):\n    w, h = image.shape[1], image.shape[0]\n    w_new, h_new = process_resize(w, h, resize)\n    \n    scales = (float(w) / float(w_new), float(h) / float(h_new))\n\n    if resize_float:\n        image = cv2.resize(image.astype('float32'), (w_new, h_new))\n    else:\n        image = cv2.resize(image, (w_new, h_new)).astype('float32')\n\n    inp = frame2tensor(image, device)\n    return image, inp, scales, (h, w)\n\ndef get_mkpts_superglue(matcher, file1, file2, device=torch.device('cuda'), resize=[512,], resize_float=True):\n    image1 = cv2.imread(file1, cv2.IMREAD_GRAYSCALE)\n    image2 = cv2.imread(file2, cv2.IMREAD_GRAYSCALE)\n    \n    _image_1, inp_1, scales_1, ori_shape_1 = f_preprocess(image1, device, resize, 0, resize_float)\n    _image_2, inp_2, scales_2, ori_shape_2 = f_preprocess(image2, device, resize, 0, resize_float)\n    image1 = torchF.interpolate(inp_1, (_image_1.shape[0], _image_1.shape[1]), mode='bilinear', align_corners=False)\n    image2 = torchF.interpolate(inp_2, (_image_2.shape[0], _image_2.shape[1]), mode='bilinear', align_corners=False)\n\n    pred = matcher({\"image0\": image1.to(device), \"image1\": image2.to(device)})\n    pred = {k: v[0].detach().cpu().numpy().copy() for k, v in pred.items()}\n    mkpts0, mkpts1 = pred[\"keypoints0\"], pred[\"keypoints1\"]\n    matches, conf = pred[\"matches0\"], pred[\"matching_scores0\"]\n\n    valid = matches > -1\n    mkpts0 = mkpts0[valid]\n    mkpts1 = mkpts1[matches[valid]]\n    mconf = conf[valid]\n\n    # Scaling coords\n    mkpts0[:,0] = mkpts0[:,0] * ori_shape_1[1] / image1.shape[3]   # X\n    mkpts0[:,1] = mkpts0[:,1] * ori_shape_1[0] / image1.shape[2]   # Y\n    mkpts1[:,0] = mkpts1[:,0] * ori_shape_2[1] / image2.shape[3]   # X\n    mkpts1[:,1] = mkpts1[:,1] * ori_shape_2[0] / image2.shape[2]   # Y \n    return mkpts0, mkpts1, mconf","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:47:05.913411Z","iopub.execute_input":"2024-03-26T16:47:05.913802Z","iopub.status.idle":"2024-03-26T16:47:05.928298Z","shell.execute_reply.started":"2024-03-26T16:47:05.913771Z","shell.execute_reply":"2024-03-26T16:47:05.927214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmkpts1, mkpts2, mconf = get_mkpts_superglue(matcher, file1, file2)\nprint(mkpts1.shape, mkpts2.shape, mconf.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:47:06.516309Z","iopub.execute_input":"2024-03-26T16:47:06.516972Z","iopub.status.idle":"2024-03-26T16:47:08.047810Z","shell.execute_reply.started":"2024-03-26T16:47:06.516938Z","shell.execute_reply":"2024-03-26T16:47:08.046743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Viewer","metadata":{}},{"cell_type":"code","source":"def get_concat_h(im1, im2):\n    dst = Image.new('RGB', (im1.width + im2.width, im1.height))\n    dst.paste(im1, (0, 0))\n    dst.paste(im2, (im1.width, 0))\n    return dst\n\ndef veiw_keypoints(file1, file2, mkpts1, mkpts2, mconf, cmap_name=\"rainbow\", height=480, drops=5):\n    cmap = matplotlib.cm.get_cmap(cmap_name)\n    img1 = Image.open(file1)\n    img2 = Image.open(file2)\n\n    w1, h1 = img1.size\n    w2, h2 = img2.size\n    \n    h = height # max(h1, h2)\n    ratio1 = h / h1\n    ratio2 = h / h2\n    \n    _w1 = int(w1 * ratio1)\n    _w2 = int(w2 * ratio2)\n    \n    mkpts1 = mkpts1 * ratio1\n    mkpts2 = mkpts2 * ratio2\n    mkpts2[:, 0] = mkpts2[:, 0] + _w1\n    \n    overlay = get_concat_h(img1.resize((_w1, h)), img2.resize((_w2, h)))\n    \n    display(overlay)\n\n    draw = ImageDraw.Draw(overlay)\n    for i in range(0, mkpts1.shape[0], drops):\n        color = tuple((cmap([mconf[i]])[:, :3] * 255).astype(np.uint8)[0, :])\n        draw.line((mkpts1[i, 0], mkpts1[i, 1], mkpts2[i, 0], mkpts2[i, 1]), fill=color, width=2)\n    display(overlay)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:47:11.760883Z","iopub.execute_input":"2024-03-26T16:47:11.762054Z","iopub.status.idle":"2024-03-26T16:47:11.772669Z","shell.execute_reply.started":"2024-03-26T16:47:11.762009Z","shell.execute_reply":"2024-03-26T16:47:11.771537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"file1, file2 : \", file1, file2)\nveiw_keypoints(file1, file2, mkpts1, mkpts2, mconf, cmap_name=\"jet\", height=480, drops=1)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:47:25.921394Z","iopub.execute_input":"2024-03-26T16:47:25.921767Z","iopub.status.idle":"2024-03-26T16:47:26.341356Z","shell.execute_reply.started":"2024-03-26T16:47:25.921741Z","shell.execute_reply":"2024-03-26T16:47:26.340368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fundamental matrix\nref: https://slideplayer.com/slide/5279455/  \n![fmatrix](https://slideplayer.com/slide/5279455/17/images/4/Fundamental+matrix+epipolar+plane.+epipolar+line.+epipolar+line.+%28projection+of+ray%29+Image+1.+Image+2..jpg)","metadata":{}},{"cell_type":"code","source":"import cv2","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:47:31.250303Z","iopub.execute_input":"2024-03-26T16:47:31.250705Z","iopub.status.idle":"2024-03-26T16:47:31.255074Z","shell.execute_reply.started":"2024-03-26T16:47:31.250676Z","shell.execute_reply":"2024-03-26T16:47:31.254133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calc Fundamental matrix from keypoints\nF, inliers = cv2.findFundamentalMat(mkpts1, mkpts2, cv2.USAC_MAGSAC, 0.200, 0.9999, 30000)\ninliers = inliers > 0           \nsampled_mkpts1 = mkpts1[inliers[:,0]]\nsampled_mkpts2 = mkpts2[inliers[:,0]]\nsampled_mconf  = mconf[inliers[:,0]]\n\nprint(f\"F.shape = \", F.shape)\nprint(f\"F = \", F)\nprint(\"keypoints :\", mkpts1.shape, \"-->\", sampled_mkpts1.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:47:33.937347Z","iopub.execute_input":"2024-03-26T16:47:33.938200Z","iopub.status.idle":"2024-03-26T16:47:34.055639Z","shell.execute_reply.started":"2024-03-26T16:47:33.938169Z","shell.execute_reply":"2024-03-26T16:47:34.054671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"veiw_keypoints(file1, file2, sampled_mkpts1, sampled_mkpts2, sampled_mconf, cmap_name=\"jet\", height=480, drops=1)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:47:38.954748Z","iopub.execute_input":"2024-03-26T16:47:38.955125Z","iopub.status.idle":"2024-03-26T16:47:39.371396Z","shell.execute_reply.started":"2024-03-26T16:47:38.955095Z","shell.execute_reply":"2024-03-26T16:47:39.370503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}