{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!git clone https://github.com/Parskatt/DKM.git","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-16T16:18:23.945807Z","iopub.execute_input":"2023-09-16T16:18:23.947745Z","iopub.status.idle":"2023-09-16T16:18:26.675923Z","shell.execute_reply.started":"2023-09-16T16:18:23.947714Z","shell.execute_reply":"2023-09-16T16:18:26.674574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install einops","metadata":{"execution":{"iopub.status.busy":"2023-09-16T16:18:26.682087Z","iopub.execute_input":"2023-09-16T16:18:26.684244Z","iopub.status.idle":"2023-09-16T16:18:40.913197Z","shell.execute_reply.started":"2023-09-16T16:18:26.684203Z","shell.execute_reply":"2023-09-16T16:18:40.911906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('./DKM/')\n\nfrom PIL import Image\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\nfrom dkm.utils.utils import tensor_to_pil\nimport cv2\nfrom dkm import DKMv3_outdoor\nimport matplotlib","metadata":{"execution":{"iopub.status.busy":"2023-09-16T16:44:17.625197Z","iopub.execute_input":"2023-09-16T16:44:17.625614Z","iopub.status.idle":"2023-09-16T16:44:17.631843Z","shell.execute_reply.started":"2023-09-16T16:44:17.625577Z","shell.execute_reply":"2023-09-16T16:44:17.630299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndkm_model = DKMv3_outdoor(device=device)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T16:30:35.659493Z","iopub.execute_input":"2023-09-16T16:30:35.660035Z","iopub.status.idle":"2023-09-16T16:30:36.831768Z","shell.execute_reply.started":"2023-09-16T16:30:35.660004Z","shell.execute_reply":"2023-09-16T16:30:36.830733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file1 = \"../input/image-matching-challenge-2023/train/phototourism/sagrada_familia/images/05907640_5576489463.jpg\"\nfile2 = \"../input/image-matching-challenge-2023/train/phototourism/sagrada_familia/images/12447860_7511693888.jpg\"","metadata":{"execution":{"iopub.status.busy":"2023-09-16T16:50:39.845935Z","iopub.execute_input":"2023-09-16T16:50:39.846379Z","iopub.status.idle":"2023-09-16T16:50:39.851844Z","shell.execute_reply.started":"2023-09-16T16:50:39.846338Z","shell.execute_reply":"2023-09-16T16:50:39.850882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mkpts(dkm_model, file1, file2, shape=(384, 512), th_conf=0.5):\n    img1 = cv2.imread(file1) \n    img2 = cv2.imread(file2)\n\n    img1PIL = Image.fromarray(cv2.cvtColor(img1, cv2.COLOR_BGR2RGB)).resize(shape)\n    img2PIL = Image.fromarray(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)).resize(shape)\n\n    dense_matches, dense_certainty = dkm_model.match(img1PIL, img2PIL)\n    dense_certainty = dense_certainty.sqrt()\n    dense_matches = dense_matches.reshape((-1, 4))\n    dense_certainty = dense_certainty.reshape((-1,))\n\n    # drop low confidence pairs\n    dense_matches = dense_matches[ dense_certainty >= th_conf, :]\n    dense_certainty = dense_certainty[ dense_certainty >= th_conf]\n    \n    mkpts1 = dense_matches[:, :2]\n    mkpts2 = dense_matches[:, 2:]\n\n    h, w, c = img1.shape\n    mkpts1[:, 0] = ((mkpts1[:, 0] + 1)/2) * w\n    mkpts1[:, 1] = ((mkpts1[:, 1] + 1)/2) * h\n\n    h, w, c = img2.shape\n    mkpts2[:, 0] = ((mkpts2[:, 0] + 1)/2) * w\n    mkpts2[:, 1] = ((mkpts2[:, 1] + 1)/2) * h\n\n    # drop same pairs\n    df = pd.DataFrame()\n    df[\"mkpts0x\"] = mkpts1[:, 0].cpu().detach().numpy().astype(np.int64)\n    df[\"mkpts0y\"] = mkpts1[:, 1].cpu().detach().numpy().astype(np.int64)\n    df[\"mkpts1x\"] = mkpts2[:, 0].cpu().detach().numpy().astype(np.int64)\n    df[\"mkpts1y\"] = mkpts2[:, 1].cpu().detach().numpy().astype(np.int64)\n    df[\"mconf\"]   = dense_certainty.cpu().detach().numpy()\n    df = df.drop_duplicates(subset=[\"mkpts0x\", \"mkpts0y\"])\n\n    mkpts1 = df[[\"mkpts0x\", \"mkpts0y\"]].values\n    mkpts2 = df[[\"mkpts1x\", \"mkpts1y\"]].values\n    mconf  = df[[\"mconf\"]].values\n    return mkpts1, mkpts2, mconf","metadata":{"execution":{"iopub.status.busy":"2023-09-16T16:50:40.57145Z","iopub.execute_input":"2023-09-16T16:50:40.571834Z","iopub.status.idle":"2023-09-16T16:50:40.586142Z","shell.execute_reply.started":"2023-09-16T16:50:40.571803Z","shell.execute_reply":"2023-09-16T16:50:40.585044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmkpts1, mkpts2, mconf = get_mkpts(dkm_model, file1, file2, shape=(384, 512))\nprint(mkpts1.shape, mkpts2.shape, mconf.shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T16:50:41.897922Z","iopub.execute_input":"2023-09-16T16:50:41.898274Z","iopub.status.idle":"2023-09-16T16:50:43.423081Z","shell.execute_reply.started":"2023-09-16T16:50:41.898244Z","shell.execute_reply":"2023-09-16T16:50:43.421967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cmap = matplotlib.cm.get_cmap('rainbow')\n\nimg1 = np.array(Image.open(file1))\nimg2 = np.array(Image.open(file2))\n\n# tiled images\noverlay = np.zeros( (max(img1.shape[0], img2.shape[0]), img1.shape[1]+img2.shape[1], 3), dtype=np.uint8)\noverlay[:, 0:img1.shape[1], :] = np.array(Image.fromarray(img1).resize((img1.shape[1], overlay.shape[0])))\noverlay[:, img1.shape[1]:,  :] = np.array(Image.fromarray(img2).resize((img2.shape[1], overlay.shape[0])))\ndisplay(Image.fromarray(overlay))\n\n# confidence map\nimg_conf1 = np.zeros(img1.shape, dtype=np.uint8)\nimg_conf1[mkpts1[:,1]-1, mkpts1[:,0]-1, :] = (cmap(mconf[:,0])[:, :3] * 255).astype(np.uint8)\nimg_conf2 = np.zeros(img2.shape, dtype=np.uint8)\nimg_conf2[mkpts2[:,1]-1, mkpts2[:,0]-1, :] = (cmap(mconf[:,0])[:, :3] * 255).astype(np.uint8)\noverlay[:, 0:img1.shape[1], :] = np.array(Image.fromarray(img_conf1).resize((img1.shape[1], overlay.shape[0])))\noverlay[:, img1.shape[1]:,  :] = np.array(Image.fromarray(img_conf2).resize((img2.shape[1], overlay.shape[0])))\ndisplay(Image.fromarray(overlay))\n","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:00:34.25046Z","iopub.execute_input":"2023-09-16T17:00:34.250876Z","iopub.status.idle":"2023-09-16T17:00:35.281799Z","shell.execute_reply.started":"2023-09-16T17:00:34.250845Z","shell.execute_reply":"2023-09-16T17:00:35.280951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}