{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71885,"databundleVersionId":8143495,"sourceType":"competition"},{"sourceId":3799555,"sourceType":"datasetVersion","datasetId":2265359},{"sourceId":3462364,"sourceType":"datasetVersion","datasetId":2069214},{"sourceId":3414836,"sourceType":"datasetVersion","datasetId":2058261},{"sourceId":99073927,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-20T10:14:34.810781Z","iopub.execute_input":"2024-05-20T10:14:34.811698Z","iopub.status.idle":"2024-05-20T10:14:36.754888Z","shell.execute_reply.started":"2024-05-20T10:14:34.811644Z","shell.execute_reply":"2024-05-20T10:14:36.753547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport csv\nimport random\nfrom glob import glob\nfrom tqdm import tqdm\nfrom collections import namedtuple\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nimport torch\n\n# import sys\n# # sys.path.append(\"../input/super-glue-pretrained-network\")\n# # from models.matching import Matching\n# # from models.utils import (compute_pose_error, compute_epipolar_error,\n# #                           estimate_pose, make_matching_plot,\n# #                           error_colormap, AverageTimer, pose_auc, read_image,\n# #                           rotate_intrinsics, rotate_pose_inplane,\n# #                           scale_intrinsics)\nfrom os import listdir\n# import kornia as K\n# import kornia.feature as KF\n# import gc","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:20:59.997057Z","iopub.execute_input":"2024-05-20T10:20:59.997621Z","iopub.status.idle":"2024-05-20T10:21:00.007331Z","shell.execute_reply.started":"2024-05-20T10:20:59.997578Z","shell.execute_reply":"2024-05-20T10:21:00.005648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:21:20.830097Z","iopub.execute_input":"2024-05-20T10:21:20.830587Z","iopub.status.idle":"2024-05-20T10:21:20.837699Z","shell.execute_reply.started":"2024-05-20T10:21:20.83055Z","shell.execute_reply":"2024-05-20T10:21:20.836038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dry_run = False\n!pip install ../input/kornia-loftr/kornia-0.6.4-py2.py3-none-any.whl\n!pip install ../input/kornia-loftr/kornia_moons-0.1.9-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:21:31.815254Z","iopub.execute_input":"2024-05-20T10:21:31.815736Z","iopub.status.idle":"2024-05-20T10:22:03.860099Z","shell.execute_reply.started":"2024-05-20T10:21:31.815703Z","shell.execute_reply":"2024-05-20T10:22:03.858427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport cv2\nimport csv\nfrom glob import glob\nimport torch\nimport matplotlib.pyplot as plt\nimport kornia\nfrom kornia_moons.feature import *\nimport kornia as K\nimport kornia.feature as KF\nimport gc","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:22:03.863632Z","iopub.execute_input":"2024-05-20T10:22:03.864255Z","iopub.status.idle":"2024-05-20T10:22:03.877835Z","shell.execute_reply.started":"2024-05-20T10:22:03.864199Z","shell.execute_reply":"2024-05-20T10:22:03.876321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport kornia.feature as KF\n\n# Check if CUDA is available and set the device accordingly\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Load the LoFTR model\nmatcher = KF.LoFTR(pretrained=None)\n\n# Load the state dictionary and move the model to the appropriate device\ncheckpoint = torch.load(\"../input/kornia-loftr/loftr_outdoor.ckpt\", map_location=device)\nmatcher.load_state_dict(checkpoint['state_dict'])\n\n# Move the model to the device and set it to evaluation mode\nmatcher = matcher.to(device).eval()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:23:25.924049Z","iopub.execute_input":"2024-05-20T10:23:25.925588Z","iopub.status.idle":"2024-05-20T10:23:26.339612Z","shell.execute_reply.started":"2024-05-20T10:23:25.925532Z","shell.execute_reply":"2024-05-20T10:23:26.338462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/super-glue-pretrained-network\")\nfrom models.matching import Matching\nfrom models.utils import (compute_pose_error, compute_epipolar_error,\n                          estimate_pose, make_matching_plot,\n                          error_colormap, AverageTimer, pose_auc, read_image,\n                          rotate_intrinsics, rotate_pose_inplane,\n                          scale_intrinsics)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:23:42.417401Z","iopub.execute_input":"2024-05-20T10:23:42.417865Z","iopub.status.idle":"2024-05-20T10:23:42.482Z","shell.execute_reply.started":"2024-05-20T10:23:42.417834Z","shell.execute_reply":"2024-05-20T10:23:42.481121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nresize = [-1, ]\nresize_float = True\n\nconfig = {\n    \"superpoint\": {\n        \"nms_radius\": 3,\n        \"keypoint_threshold\": 0.005,\n        \"max_keypoints\": 2048\n    },\n    \"superglue\": {\n        \"weights\": \"outdoor\",\n        \"sinkhorn_iterations\": 100,\n        \"match_threshold\": 0.2,\n    }\n}\nmatching = Matching(config).eval().to(device)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:23:54.965957Z","iopub.execute_input":"2024-05-20T10:23:54.966359Z","iopub.status.idle":"2024-05-20T10:23:55.7303Z","shell.execute_reply.started":"2024-05-20T10:23:54.966328Z","shell.execute_reply":"2024-05-20T10:23:55.728626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"src = '/kaggle/input/image-matching-challenge-2024'\n\ntest_samples = []\nwith open(f'{src}/test/categories.csv') as f:\n    reader = csv.reader(f, delimiter=',')\n    for i, row in enumerate(reader):\n        # Skip header.\n        if i == 0:\n            continue\n        test_samples += [row]\n\n\ndef FlattenMatrix(M, num_digits=8):\n    '''Convenience function to write CSV files.'''\n    \n    return ' '.join([f'{v:.{num_digits}e}' for v in M.flatten()])\n\n\ndef load_torch_image(fname, device):\n    img = cv2.imread(fname)\n    scale = 840 / max(img.shape[0], img.shape[1]) \n    w = int(img.shape[1] * scale)\n    h = int(img.shape[0] * scale)\n    img = cv2.resize(img, (w, h))\n    img = K.image_to_tensor(img, False).float() /255.\n    img = K.color.bgr_to_rgb(img)\n    return img.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:26:51.369326Z","iopub.execute_input":"2024-05-20T10:26:51.370125Z","iopub.status.idle":"2024-05-20T10:26:51.39011Z","shell.execute_reply.started":"2024-05-20T10:26:51.370081Z","shell.execute_reply":"2024-05-20T10:26:51.388511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_resized_image(fname, resize): # function t read an image at a particular scale(when filename is given)\n    img = cv2.imread(fname)\n    h, w, _ = img.shape\n    scale = resize / max(h, w) \n    w = int(w * scale)\n    h = int(h * scale)\n    \n    img = cv2.resize(img, (w, h))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    img = torch.from_numpy(img)[None][None]/ 255.\n    \n    return img.to(device), scale\n\ndef load_image1(fname):\n    img = cv2.imread(fname)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    img = K.image_to_tensor(img, False).float() /255.\n    return img.to(device)\n\ndef load_resized_image1(img, resize): # function t read an image at a particular scale(when image itself is given)\n    img=img[0,0,:,:]\n    h, w = img.shape\n    img=img.cpu().numpy()\n    scale = resize / max(h, w) \n    w_new = int(w * scale)\n    h_new = int(h * scale)\n    \n    img = cv2.resize(img, (w_new, h_new))\n#     img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    img = torch.from_numpy(img)[None][None]\n    \n    return img.to(device), [h_new/h,w_new/w]\n\ndef scale_to_resized(mkpts0, mkpts1, scale1,scale2):\n    ### scale to original im size because we used max_image_size\n    # first point\n    mkpts0[:, 0] = mkpts0[:, 0] / scale1[0]\n    mkpts0[:, 1] = mkpts0[:, 1] / scale1[1]    \n    # second point\n    mkpts1[:, 0] = mkpts1[:, 0] / scale2[0]\n    mkpts1[:, 1] = mkpts1[:, 1] / scale2[1]\n    \n    return mkpts0, mkpts1\n\n\ndef get_loftr_match(img1,img2):\n    input_dict = {\"image0\": (img1), \n              \"image1\": (img2)}\n\n    with torch.no_grad():\n        correspondences = matcher(input_dict)\n        \n    a1 = correspondences['keypoints0'].cpu().numpy()\n    a2 = correspondences['keypoints1'].cpu().numpy()\n    return a1,a2\n\n\nfrom sklearn.cluster import DBSCAN\n\ndef crop_image(image, matching_points, eps, min_samples=5):\n    # Convert matching points to numpy array\n    matching_points = np.array(matching_points)\n\n    # Apply DBSCAN clustering\n    clustering = DBSCAN(eps=eps, min_samples=min_samples).fit(matching_points)\n\n    # Find the most dense cluster\n    unique_labels, label_counts = np.unique(clustering.labels_, return_counts=True)\n#     print(unique_labels)\n#     print(label_counts)\n    most_dense_cluster_label = unique_labels[np.argmax(label_counts)]\n#     print(unique_labels)\n    # Get the matching points in the most dense cluster\n    cluster_points = matching_points[clustering.labels_ == most_dense_cluster_label]\n#     print(cluster_points)\n    # Calculate the minimum and maximum coordinates in the cluster\n    min_x = int(np.min(cluster_points[:, 0]))\n    max_x = int(np.max(cluster_points[:, 0]))\n    min_y = int(np.min(cluster_points[:, 1]))\n    max_y = int(np.max(cluster_points[:, 1]))\n    # Crop the image based on the cluster coordinates\n    cropped_image = image[0,0,int(min_y):int(max_y), int(min_x):int(max_x)]\n\n    return cropped_image,min_x,min_y\n\ndef get_superglue(img1,img2):\n    pred = matching({\"image0\": img1, \"image1\": img2})\n    pred = {k: v[0].detach().cpu().numpy() for k, v in pred.items()}\n    # nd1 = time.time()\n    kpts1, kpts2 = pred[\"keypoints0\"], pred[\"keypoints1\"]\n    matches, conf = pred[\"matches0\"], pred[\"matching_scores0\"]\n\n    valid = matches > -1\n    a1 = kpts1[valid]\n    a2 = kpts2[matches[valid]]\n    mconf = conf[valid]\n    return a1,a2\n","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:27:13.585995Z","iopub.execute_input":"2024-05-20T10:27:13.586678Z","iopub.status.idle":"2024-05-20T10:27:14.672263Z","shell.execute_reply.started":"2024-05-20T10:27:13.586626Z","shell.execute_reply":"2024-05-20T10:27:14.67062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"F_dict = {}\nimport time\nfor i, row in enumerate(test_samples):\n    sample_id, batch_id, image_1_id, image_2_id = row\n    # Load the images. # sizes of 840,1040,1280\n    st = time.time()\n    img11=load_image1(f'{src}/test_images/{batch_id}/{image_1_id}.png')\n    img22 =load_image1(f'{src}/test_images/{batch_id}/{image_2_id}.png')\n    img1,scale1 = load_resized_image1(img11, 840)\n    img2,scale2 = load_resized_image1(img22, 840)\n    mkpts11,mkpts22=get_superglue(img1,img2)\n    mkpts11,mkpts22=scale_to_resized(mkpts11, mkpts22, scale1,scale2)\n    mkpts1=mkpts11\n    mkpts2=mkpts22\n    \n#     img1,scale1 = load_resized_image1(img11, 840)\n#     img2,scale2 = load_resized_image1(img22, 840)\n#     mkpts11,mkpts22=get_superglue(img1,img2)\n#     mkpts11,mkpts22=scale_to_resized(mkpts11, mkpts22, scale1,scale2)\n#     mkpts1=np.vstack((mkpts1,mkpts11))\n#     mkpts2=np.vstack((mkpts2,mkpts22))\n    img1,scale1 = load_resized_image1(img11, 1040)\n    img2,scale2 = load_resized_image1(img22, 1040)\n    mkpts11,mkpts22=get_superglue(img1,img2)\n    mkpts11,mkpts22=scale_to_resized(mkpts11, mkpts22, scale1,scale2)\n    mkpts1=np.vstack((mkpts1,mkpts11))\n    mkpts2=np.vstack((mkpts2,mkpts22))\n    img1,scale1 = load_resized_image1(img11, 1240)\n    img2,scale2 = load_resized_image1(img22, 1240)\n    mkpts11,mkpts22=get_superglue(img1,img2)\n    mkpts11,mkpts22=scale_to_resized(mkpts11, mkpts22, scale1,scale2)\n    mkpts1=np.vstack((mkpts1,mkpts11))\n    mkpts2=np.vstack((mkpts2,mkpts22))\n    \n    \n    c1,minx1,miny1=crop_image(img11, mkpts1, 20, min_samples=5)\n    c2,minx2,miny2=crop_image(img22, mkpts2, 20, min_samples=5)\n    c1=c1.reshape(1,1,c1.shape[0],c1.shape[1])\n    c2=c2.reshape(1,1,c2.shape[0],c2.shape[1])\n    img1,scale1=load_resized_image1(c1,840)\n    img2,scale2=load_resized_image1(c2,840)\n    mkpts11,mkpts22=get_superglue(img1,img2)\n    mkpts11,mkpts22=scale_to_resized(mkpts11, mkpts22, scale1,scale2)\n    mkpts11+=np.array([minx1,miny1])\n    mkpts22+=np.array([minx2,miny2])\n    mkpts1=mkpts11\n    mkpts2=mkpts22\n    \n#     img1,scale1=load_resized_image1(c1,840)\n#     img2,scale2=load_resized_image1(c2,840)\n#     mkpts11,mkpts22=get_superglue(img1,img2)\n#     mkpts11,mkpts22=scale_to_resized(mkpts11, mkpts22, scale1,scale2)\n#     mkpts11+=np.array([minx1,miny1])\n#     mkpts22+=np.array([minx2,miny2])\n#     mkpts1=np.vstack((mkpts1,mkpts11))\n#     mkpts2=np.vstack((mkpts2,mkpts22))\n    \n    img1,scale1=load_resized_image1(c1,1040)\n    img2,scale2=load_resized_image1(c2,1040)\n    mkpts11,mkpts22=get_superglue(img1,img2)\n    mkpts11,mkpts22=scale_to_resized(mkpts11, mkpts22, scale1,scale2)\n    mkpts11+=np.array([minx1,miny1])\n    mkpts22+=np.array([minx2,miny2])\n    mkpts1=np.vstack((mkpts1,mkpts11))\n    mkpts2=np.vstack((mkpts2,mkpts22))\n    \n    img1,scale1=load_resized_image1(c1,1280)\n    img2,scale2=load_resized_image1(c2,1280)\n    mkpts11,mkpts22=get_superglue(img1,img2)\n    mkpts11,mkpts22=scale_to_resized(mkpts11, mkpts22, scale1,scale2)\n    mkpts11+=np.array([minx1,miny1])\n    mkpts22+=np.array([minx2,miny2])\n    mkpts1=np.vstack((mkpts1,mkpts11))\n    mkpts2=np.vstack((mkpts2,mkpts22))\n    \n    \n    img1,scale1 = load_resized_image1(img11, 840)\n    img2,scale2 = load_resized_image1(img22, 840)\n    mkpts11,mkpts22=get_loftr_match(img1,img2)\n    mkpts11,mkpts22=scale_to_resized(mkpts11, mkpts22, scale1,scale2)\n    mkpts1=np.vstack((mkpts1,mkpts11))\n    mkpts2=np.vstack((mkpts2,mkpts22))\n#     img1=load_image1(f'{src}/test_images/{batch_id}/{image_1_id}.png')\n#     img2=load_image1(f'{src}/test_images/{batch_id}/{image_2_id}.png')\n    img1=img11\n    img2=img22\n    \n    if len(mkpts1) > 7:\n        F, inliers = cv2.findFundamentalMat(mkpts1, mkpts2, cv2.USAC_MAGSAC, 0.25, 0.9999, 10000)\n#         F, inlier_mask = cv2.findFundamentalMat(mkpts1, mkpts2, cv2.USAC_MAGSAC, ransacReprojThreshold=0.25, confidence=0.99999, maxIters=10000)\n        inliers = inliers > 0\n        assert F.shape == (3, 3), 'Malformed F?'\n        F_dict[sample_id] = F\n    else:\n        F_dict[sample_id] = np.zeros((3, 3))\n        continue\n    \n    nd = time.time() \n    print(\"Running time: \", nd - st, \" s\")\n    if (i < 3):\n#         print(\"Running time: \", nd - st, \" s\")\n        gc.collect()\n        draw_LAF_matches(\n        KF.laf_from_center_scale_ori(torch.from_numpy(mkpts1).view(1,-1, 2),\n                                    torch.ones(mkpts1.shape[0]).view(1,-1, 1, 1),\n                                    torch.ones(mkpts1.shape[0]).view(1,-1, 1)),\n        KF.laf_from_center_scale_ori(torch.from_numpy(mkpts2).view(1,-1, 2),\n                                    torch.ones(mkpts2.shape[0]).view(1,-1, 1, 1),\n                                    torch.ones(mkpts2.shape[0]).view(1,-1, 1)),\n        torch.arange(mkpts1.shape[0]).view(-1,1).repeat(1,2),\n        K.tensor_to_image(img1),\n        K.tensor_to_image(img2),\n        inliers,\n        draw_dict={'inlier_color': (0.2, 1, 0.2),\n                   'tentative_color': None, \n                   'feature_color': (0.2, 0.5, 1), 'vertical': False})\n       \n    \nwith open('submission.csv', 'w') as f:\n    f.write('sample_id,fundamental_matrix\\n')\n    for sample_id, F in F_dict.items():\n        f.write(f'{sample_id},{FlattenMatrix(F)}\\n')","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:28:14.247796Z","iopub.execute_input":"2024-05-20T10:28:14.24824Z","iopub.status.idle":"2024-05-20T10:28:14.455871Z","shell.execute_reply.started":"2024-05-20T10:28:14.248206Z","shell.execute_reply":"2024-05-20T10:28:14.454344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}