{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71885,"databundleVersionId":8015523,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!git clone https://github.com/Parskatt/DKM.git","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-26T16:25:59.055244Z","iopub.execute_input":"2024-03-26T16:25:59.055585Z","iopub.status.idle":"2024-03-26T16:26:01.093965Z","shell.execute_reply.started":"2024-03-26T16:25:59.055552Z","shell.execute_reply":"2024-03-26T16:26:01.092871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install einops","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:26:01.095818Z","iopub.execute_input":"2024-03-26T16:26:01.096145Z","iopub.status.idle":"2024-03-26T16:26:14.266669Z","shell.execute_reply.started":"2024-03-26T16:26:01.096119Z","shell.execute_reply":"2024-03-26T16:26:14.265483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('./DKM/')\n\nfrom PIL import Image, ImageDraw\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\nfrom dkm.utils.utils import tensor_to_pil\nimport cv2\nfrom dkm import DKMv3_outdoor\nimport matplotlib","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:26:14.268267Z","iopub.execute_input":"2024-03-26T16:26:14.268644Z","iopub.status.idle":"2024-03-26T16:26:21.672808Z","shell.execute_reply.started":"2024-03-26T16:26:14.268613Z","shell.execute_reply":"2024-03-26T16:26:21.672008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DKM model\nDKM: Dense Kernelized Feature Matching for Geometry Estimation  \nhttps://parskatt.github.io/DKM/","metadata":{}},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndkm_model = DKMv3_outdoor(device=device)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:26:21.675268Z","iopub.execute_input":"2024-03-26T16:26:21.676140Z","iopub.status.idle":"2024-03-26T16:26:30.866843Z","shell.execute_reply.started":"2024-03-26T16:26:21.676105Z","shell.execute_reply":"2024-03-26T16:26:30.866041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Demo","metadata":{}},{"cell_type":"code","source":"file1 = \"/kaggle/input/image-matching-challenge-2024/test/church/images/00001.png\"\nfile2 = \"/kaggle/input/image-matching-challenge-2024/test/church/images/00006.png\"","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:26:30.868063Z","iopub.execute_input":"2024-03-26T16:26:30.868347Z","iopub.status.idle":"2024-03-26T16:26:30.872490Z","shell.execute_reply.started":"2024-03-26T16:26:30.868323Z","shell.execute_reply":"2024-03-26T16:26:30.871729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mkpts(dkm_model, file1, file2, shape=(384, 512), th_conf=0.5):\n    img1 = cv2.imread(file1) \n    img2 = cv2.imread(file2)\n\n    img1PIL = Image.fromarray(cv2.cvtColor(img1, cv2.COLOR_BGR2RGB)).resize(shape)\n    img2PIL = Image.fromarray(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)).resize(shape)\n\n    dense_matches, dense_certainty = dkm_model.match(img1PIL, img2PIL)\n    dense_certainty = dense_certainty.sqrt()\n    dense_matches = dense_matches.reshape((-1, 4))\n    dense_certainty = dense_certainty.reshape((-1,))\n\n    # drop low confidence pairs\n    dense_matches = dense_matches[ dense_certainty >= th_conf, :]\n    dense_certainty = dense_certainty[ dense_certainty >= th_conf]\n    \n    mkpts1 = dense_matches[:, :2]\n    mkpts2 = dense_matches[:, 2:]\n\n    h, w, c = img1.shape\n    mkpts1[:, 0] = ((mkpts1[:, 0] + 1)/2) * w\n    mkpts1[:, 1] = ((mkpts1[:, 1] + 1)/2) * h\n\n    h, w, c = img2.shape\n    mkpts2[:, 0] = ((mkpts2[:, 0] + 1)/2) * w\n    mkpts2[:, 1] = ((mkpts2[:, 1] + 1)/2) * h\n\n    # drop same pairs\n    df = pd.DataFrame()\n    df[\"mkpts0x\"] = mkpts1[:, 0].cpu().detach().numpy().astype(np.int64)\n    df[\"mkpts0y\"] = mkpts1[:, 1].cpu().detach().numpy().astype(np.int64)\n    df[\"mkpts1x\"] = mkpts2[:, 0].cpu().detach().numpy().astype(np.int64)\n    df[\"mkpts1y\"] = mkpts2[:, 1].cpu().detach().numpy().astype(np.int64)\n    df[\"mconf\"]   = dense_certainty.cpu().detach().numpy()\n    df = df.drop_duplicates(subset=[\"mkpts0x\", \"mkpts0y\"])\n\n    mkpts1 = df[[\"mkpts0x\", \"mkpts0y\"]].values\n    mkpts2 = df[[\"mkpts1x\", \"mkpts1y\"]].values\n    mconf  = df[[\"mconf\"]].values[:, 0]\n    return mkpts1, mkpts2, mconf","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:26:30.873586Z","iopub.execute_input":"2024-03-26T16:26:30.873838Z","iopub.status.idle":"2024-03-26T16:26:30.888883Z","shell.execute_reply.started":"2024-03-26T16:26:30.873816Z","shell.execute_reply":"2024-03-26T16:26:30.888035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmkpts1, mkpts2, mconf = get_mkpts(dkm_model, file1, file2, shape=(384, 512))\nprint(mkpts1.shape, mkpts2.shape, mconf.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:26:30.889937Z","iopub.execute_input":"2024-03-26T16:26:30.890546Z","iopub.status.idle":"2024-03-26T16:26:33.838929Z","shell.execute_reply.started":"2024-03-26T16:26:30.890491Z","shell.execute_reply":"2024-03-26T16:26:33.837967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Viewer","metadata":{}},{"cell_type":"code","source":"def get_concat_h(im1, im2):\n    dst = Image.new('RGB', (im1.width + im2.width, im1.height))\n    dst.paste(im1, (0, 0))\n    dst.paste(im2, (im1.width, 0))\n    return dst\n\ndef veiw_keypoints(file1, file2, mkpts1, mkpts2, mconf, cmap_name=\"rainbow\", height=480, drops=5):\n    cmap = matplotlib.cm.get_cmap(cmap_name)\n    img1 = Image.open(file1)\n    img2 = Image.open(file2)\n\n    w1, h1 = img1.size\n    w2, h2 = img2.size\n    \n    h = height # max(h1, h2)\n    ratio1 = h / h1\n    ratio2 = h / h2\n    \n    _w1 = int(w1 * ratio1)\n    _w2 = int(w2 * ratio2)\n    \n    mkpts1 = mkpts1 * ratio1\n    mkpts2 = mkpts2 * ratio2\n    mkpts2[:, 0] = mkpts2[:, 0] + _w1\n    \n    overlay = get_concat_h(img1.resize((_w1, h)), img2.resize((_w2, h)))\n    \n    display(overlay)\n\n    draw = ImageDraw.Draw(overlay)\n    for i in range(0, mkpts1.shape[0], drops):\n        color = tuple((cmap([mconf[i]])[:, :3] * 255).astype(np.uint8)[0, :])\n        draw.line((mkpts1[i, 0], mkpts1[i, 1], mkpts2[i, 0], mkpts2[i, 1]), fill=color, width=2)\n    display(overlay)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:26:33.840024Z","iopub.execute_input":"2024-03-26T16:26:33.840311Z","iopub.status.idle":"2024-03-26T16:26:33.850723Z","shell.execute_reply.started":"2024-03-26T16:26:33.840287Z","shell.execute_reply":"2024-03-26T16:26:33.849832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"file1, file2 : \", file1, file2)\nveiw_keypoints(file1, file2, mkpts1, mkpts2, mconf, cmap_name=\"jet\", height=480, drops=100)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:26:33.851842Z","iopub.execute_input":"2024-03-26T16:26:33.852102Z","iopub.status.idle":"2024-03-26T16:26:34.327470Z","shell.execute_reply.started":"2024-03-26T16:26:33.852080Z","shell.execute_reply":"2024-03-26T16:26:34.326569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fundamental matrix\nref: https://slideplayer.com/slide/5279455/  \n![fmatrix](https://slideplayer.com/slide/5279455/17/images/4/Fundamental+matrix+epipolar+plane.+epipolar+line.+epipolar+line.+%28projection+of+ray%29+Image+1.+Image+2..jpg)","metadata":{}},{"cell_type":"code","source":"import cv2","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:26:34.330568Z","iopub.execute_input":"2024-03-26T16:26:34.330903Z","iopub.status.idle":"2024-03-26T16:26:34.335219Z","shell.execute_reply.started":"2024-03-26T16:26:34.330875Z","shell.execute_reply":"2024-03-26T16:26:34.334285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calc Fundamental matrix from keypoints\nF, inliers = cv2.findFundamentalMat(mkpts1, mkpts2, cv2.USAC_MAGSAC, 0.200, 0.9999, 30000)\ninliers = inliers > 0           \nsampled_mkpts1 = mkpts1[inliers[:,0]]\nsampled_mkpts2 = mkpts2[inliers[:,0]]\nsampled_mconf  = mconf[inliers[:,0]]\n\nprint(f\"F.shape = \", F.shape)\nprint(f\"F = \", F)\nprint(\"keypoints :\", mkpts1.shape, \"-->\", sampled_mkpts1.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T16:26:34.336170Z","iopub.execute_input":"2024-03-26T16:26:34.336433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"veiw_keypoints(file1, file2, sampled_mkpts1, sampled_mkpts2, sampled_mconf, cmap_name=\"jet\", height=480, drops=100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}