{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71885,"databundleVersionId":8015523,"sourceType":"competition"},{"sourceId":3840,"sourceType":"modelInstanceVersion","modelInstanceId":2742}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from PIL import Image, ImageDraw\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib\n\nimport kornia\nimport kornia as K\nimport kornia.feature as KF\n\nfrom kornia.feature import LoFTR","metadata":{"execution":{"iopub.status.busy":"2024-03-26T15:54:44.155801Z","iopub.execute_input":"2024-03-26T15:54:44.156088Z","iopub.status.idle":"2024-03-26T15:54:52.478651Z","shell.execute_reply.started":"2024-03-26T15:54:44.156063Z","shell.execute_reply":"2024-03-26T15:54:52.477883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LoFTR model\nhttps://zju3dv.github.io/loftr/\n![loftr](https://zju3dv.github.io/loftr/images/loftr-arch.png)","metadata":{}},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nmatcher = LoFTR(pretrained=None)\nmatcher.load_state_dict(torch.load(\"../input/loftr/pytorch/outdoor/1/loftr_outdoor.ckpt\")['state_dict'])\nmatcher = matcher.to(device).eval()","metadata":{"execution":{"iopub.status.busy":"2024-03-26T15:54:52.480128Z","iopub.execute_input":"2024-03-26T15:54:52.480530Z","iopub.status.idle":"2024-03-26T15:54:53.807698Z","shell.execute_reply.started":"2024-03-26T15:54:52.480504Z","shell.execute_reply":"2024-03-26T15:54:53.806880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Demo","metadata":{}},{"cell_type":"code","source":"file1 = \"/kaggle/input/image-matching-challenge-2024/test/church/images/00001.png\"\nfile2 = \"/kaggle/input/image-matching-challenge-2024/test/church/images/00006.png\"","metadata":{"execution":{"iopub.status.busy":"2024-03-26T15:54:53.809086Z","iopub.execute_input":"2024-03-26T15:54:53.809455Z","iopub.status.idle":"2024-03-26T15:54:53.814200Z","shell.execute_reply.started":"2024-03-26T15:54:53.809419Z","shell.execute_reply":"2024-03-26T15:54:53.813225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_torch_image(fname, device):\n    img = cv2.imread(fname)\n    original_shape = img.shape\n    w = img.shape[1]\n    h = img.shape[0]\n    img_resized = cv2.resize(img, (w, h))\n    img = K.image_to_tensor(img_resized, False).float() /255.\n    img = K.color.bgr_to_rgb(img)\n    return img.to(device), original_shape, img_resized\n\ndef get_mkpts_loftr(matcher, file1, file2, th_conf=0.5, device=\"cuda\"):    \n    image_1, ori_shape_1, img_resized1 = load_torch_image(file1, device)\n    image_2, ori_shape_2, img_resized2 = load_torch_image(file2, device)\n    input_dict = {\"image0\": K.color.rgb_to_grayscale(image_1), \n              \"image1\": K.color.rgb_to_grayscale(image_2)}\n\n    with torch.no_grad():\n        correspondences = matcher(input_dict)\n\n    print(correspondences.keys())\n    mkpts0 = correspondences['keypoints0'].cpu().numpy()\n    mkpts1 = correspondences['keypoints1'].cpu().numpy()\n    mconf  = correspondences['confidence'].cpu().numpy()\n    \n    mkpts0 = mkpts0[ mconf >= th_conf, : ]\n    mkpts1 = mkpts1[ mconf >= th_conf, : ]\n    mconf  = mconf[ mconf >= th_conf ]\n\n    # Scaling coords to same pixel size of LoFTR\n    mkpts0[:,0] = mkpts0[:,0] * ori_shape_1[1] / image_1.shape[3]   # X\n    mkpts0[:,1] = mkpts0[:,1] * ori_shape_1[0] / image_1.shape[2]   # Y\n    mkpts1[:,0] = mkpts1[:,0] * ori_shape_2[1] / image_2.shape[3]   # X\n    mkpts1[:,1] = mkpts1[:,1] * ori_shape_2[0] / image_2.shape[2]   # Y\n    \n    return mkpts0, mkpts1, mconf","metadata":{"execution":{"iopub.status.busy":"2024-03-26T15:54:53.816682Z","iopub.execute_input":"2024-03-26T15:54:53.817123Z","iopub.status.idle":"2024-03-26T15:54:53.828566Z","shell.execute_reply.started":"2024-03-26T15:54:53.817090Z","shell.execute_reply":"2024-03-26T15:54:53.827655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmkpts1, mkpts2, mconf = get_mkpts_loftr(matcher, file1, file2)\nprint(mkpts1.shape, mkpts2.shape, mconf.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T15:54:53.829716Z","iopub.execute_input":"2024-03-26T15:54:53.830056Z","iopub.status.idle":"2024-03-26T15:54:55.207968Z","shell.execute_reply.started":"2024-03-26T15:54:53.830026Z","shell.execute_reply":"2024-03-26T15:54:55.207064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Viewer","metadata":{}},{"cell_type":"code","source":"def get_concat_h(im1, im2):\n    dst = Image.new('RGB', (im1.width + im2.width, im1.height))\n    dst.paste(im1, (0, 0))\n    dst.paste(im2, (im1.width, 0))\n    return dst\n\ndef veiw_keypoints(file1, file2, mkpts1, mkpts2, mconf, cmap_name=\"rainbow\", height=480, drops=5):\n    cmap = matplotlib.cm.get_cmap(cmap_name)\n    img1 = Image.open(file1)\n    img2 = Image.open(file2)\n\n    w1, h1 = img1.size\n    w2, h2 = img2.size\n    \n    h = height # max(h1, h2)\n    ratio1 = h / h1\n    ratio2 = h / h2\n    \n    _w1 = int(w1 * ratio1)\n    _w2 = int(w2 * ratio2)\n    \n    mkpts1 = mkpts1 * ratio1\n    mkpts2 = mkpts2 * ratio2\n    mkpts2[:, 0] = mkpts2[:, 0] + _w1\n    \n    overlay = get_concat_h(img1.resize((_w1, h)), img2.resize((_w2, h)))\n    \n    display(overlay)\n\n    draw = ImageDraw.Draw(overlay)\n    for i in range(0, mkpts1.shape[0], drops):\n        color = tuple((cmap([mconf[i]])[:, :3] * 255).astype(np.uint8)[0, :])\n        draw.line((mkpts1[i, 0], mkpts1[i, 1], mkpts2[i, 0], mkpts2[i, 1]), fill=color, width=2)\n    display(overlay)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:05:12.424642Z","iopub.execute_input":"2024-03-26T16:05:12.425368Z","iopub.status.idle":"2024-03-26T16:05:12.435584Z","shell.execute_reply.started":"2024-03-26T16:05:12.425334Z","shell.execute_reply":"2024-03-26T16:05:12.434473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"file1, file2 : \", file1, file2)\nveiw_keypoints(file1, file2, mkpts1, mkpts2, mconf, cmap_name=\"jet\", height=480)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:05:12.995662Z","iopub.execute_input":"2024-03-26T16:05:12.996006Z","iopub.status.idle":"2024-03-26T16:05:13.398506Z","shell.execute_reply.started":"2024-03-26T16:05:12.995978Z","shell.execute_reply":"2024-03-26T16:05:13.397697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fundamental matrix\nref: https://slideplayer.com/slide/5279455/  \n![fmatrix](https://slideplayer.com/slide/5279455/17/images/4/Fundamental+matrix+epipolar+plane.+epipolar+line.+epipolar+line.+%28projection+of+ray%29+Image+1.+Image+2..jpg)","metadata":{}},{"cell_type":"code","source":"import cv2","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:07:05.208592Z","iopub.execute_input":"2024-03-26T16:07:05.209074Z","iopub.status.idle":"2024-03-26T16:07:05.213304Z","shell.execute_reply.started":"2024-03-26T16:07:05.209041Z","shell.execute_reply":"2024-03-26T16:07:05.212389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calc Fundamental matrix from keypoints\nF, inliers = cv2.findFundamentalMat(mkpts1, mkpts2, cv2.USAC_MAGSAC, 0.200, 0.9999, 30000)\ninliers = inliers > 0           \nsampled_mkpts1 = mkpts1[inliers[:,0]]\nsampled_mkpts2 = mkpts2[inliers[:,0]]\nsampled_mconf  = mconf[inliers[:,0]]\n\nprint(f\"F.shape = \", F.shape)\nprint(f\"F = \", F)\nprint(\"keypoints :\", mkpts1.shape, \"-->\", sampled_mkpts1.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:12:26.307344Z","iopub.execute_input":"2024-03-26T16:12:26.307683Z","iopub.status.idle":"2024-03-26T16:12:26.333604Z","shell.execute_reply.started":"2024-03-26T16:12:26.307660Z","shell.execute_reply":"2024-03-26T16:12:26.332704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"veiw_keypoints(file1, file2, sampled_mkpts1, sampled_mkpts2, sampled_mconf, cmap_name=\"jet\", height=480)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:12:58.181730Z","iopub.execute_input":"2024-03-26T16:12:58.182372Z","iopub.status.idle":"2024-03-26T16:12:58.608177Z","shell.execute_reply.started":"2024-03-26T16:12:58.182341Z","shell.execute_reply":"2024-03-26T16:12:58.607306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}