{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71885,"databundleVersionId":8015523,"sourceType":"competition"},{"sourceId":7884725,"sourceType":"datasetVersion","datasetId":4628331},{"sourceId":7884485,"sourceType":"datasetVersion","datasetId":4628051}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Published on March 26, 2024. Copied from OldUfo by Marília Prata, mpwolke","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Maybe on Next time I'll try that below\n\n![](https://saashistory.com/wp-content/uploads/2023/06/Opensfm-VS-Colmap.jpg)https://saashistory.com/opensfm-vs-colmap/","metadata":{}},{"cell_type":"markdown","source":"#COLMAP\n\n\"COLMAP is a general-purpose Structure-from-Motion (SfM) and Multi-View Stereo (MVS) pipeline with a graphical and command-line interface. It offers a wide range of features for reconstruction of ordered and unordered image collections. The software is licensed under the new BSD license.\"\n\nhttps://colmap.github.io/\n\nCOLMAP CITATION\n\n@inproceedings{schoenberger2016sfm,\n    author={Sch\\\"{o}nberger, Johannes Lutz and Frahm, Jan-Michael},\n    title={Structure-from-Motion Revisited},\n    booktitle={Conference on Computer Vision and Pattern Recognition (CVPR)},\n    year={2016},\n}\n\n@inproceedings{schoenberger2016mvs,\n    author={Sch\\\"{o}nberger, Johannes Lutz and Zheng, Enliang and Pollefeys, Marc and Frahm, Jan-Michael},\n    title={Pixelwise View Selection for Unstructured Multi-View Stereo},\n    booktitle={European Conference on Computer Vision (ECCV)},\n    year={2016},\n}\n\n@inproceedings{schoenberger2016vote,\n    author={Sch\\\"{o}nberger, Johannes Lutz and Price, True and Sattler, Torsten and Frahm, Jan-Michael and Pollefeys, Marc},\n    title={A Vote-and-Verify Strategy for Fast Spatial Verification in Image Retrieval},\n    booktitle={Asian Conference on Computer Vision (ACCV)},\n    year={2016},\n}","metadata":{}},{"cell_type":"markdown","source":"#Those 3 snippets didn't help","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input/imc2024-packages-lightglue-rerun-kornia'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index /kaggle/input/imc2024-packages-lightglue-rerun-kornia/* --no-deps","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install rerun-sdk","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#That rerun was a pain in my ass: D\n\nRun commented import rerun. Got 'rr' is not defined Uncomment import rerun as rr(running a 2nd time.","metadata":{}},{"cell_type":"code","source":"import rerun as rr  # pip install rerun-sdk #No module name\nrr.init(\"IMC lizard\") #name 'rr' is not defined\n\nimport io\nimport os\nimport re\nimport zipfile\nfrom argparse import ArgumentParser\nfrom pathlib import Path\nfrom typing import Final\n\nimport cv2\nimport numpy as np\nimport numpy.typing as npt\nimport requests\nimport pycolmap\n\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-03-26T23:39:07.030866Z","iopub.execute_input":"2024-03-26T23:39:07.031323Z","iopub.status.idle":"2024-03-26T23:39:07.373872Z","shell.execute_reply.started":"2024-03-26T23:39:07.031282Z","shell.execute_reply":"2024-03-26T23:39:07.372792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#I changed just Directory church to lizard","metadata":{}},{"cell_type":"code","source":"!ls /kaggle/input/image-matching-challenge-2024/train/lizard/smf\npath_to_lizard = '/kaggle/input/image-matching-challenge-2024/train/lizard/smf'\npath_to_lizard_images = '/kaggle/input/image-matching-challenge-2024/train/lizard/images'","metadata":{"execution":{"iopub.status.busy":"2024-03-26T23:44:30.204853Z","iopub.execute_input":"2024-03-26T23:44:30.205909Z","iopub.status.idle":"2024-03-26T23:44:31.312771Z","shell.execute_reply.started":"2024-03-26T23:44:30.205871Z","shell.execute_reply":"2024-03-26T23:44:31.310997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/input/image-matching-challenge-2024/train/church/smf\npath_to_church = '/kaggle/input/image-matching-challenge-2024/train/church/smf'\npath_to_church_images = '/kaggle/input/image-matching-challenge-2024/train/church/images'","metadata":{"execution":{"iopub.status.busy":"2024-03-27T00:04:18.079376Z","iopub.execute_input":"2024-03-27T00:04:18.080100Z","iopub.status.idle":"2024-03-27T00:04:19.298368Z","shell.execute_reply.started":"2024-03-27T00:04:18.080046Z","shell.execute_reply":"2024-03-27T00:04:19.296685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#By OldLufo https://www.kaggle.com/code/oldufo/colmap-3d-visualization-with-rerun-io\n\nDESCRIPTION = \"\"\"\n# Sparse Reconstruction by COLMAP\nThis example was generated from the output of a sparse reconstruction done with COLMAP.\n\n[COLMAP](https://colmap.github.io/index.html) is a general-purpose Structure-from-Motion (SfM) and Multi-View Stereo\n(MVS) pipeline with a graphical and command-line interface.\n\nIn this example a short video clip has been processed offline by the COLMAP pipeline, and we use Rerun to visualize the\nindividual camera frames, estimated camera poses, and resulting point clouds over time.\n\n## How it was made\nThe full source code for this example is available\n[on GitHub](https://github.com/rerun-io/rerun/blob/latest/examples/python/structure_from_motion/main.py).\n\n### Images\nThe images are logged through the [rr.Image archetype](https://www.rerun.io/docs/reference/types/archetypes/image)\nto the [camera/image entity](recording://camera/image).\n\n### Cameras\nThe images stem from pinhole cameras located in the 3D world. To visualize the images in 3D, the pinhole projection has\nto be logged and the camera pose (this is often referred to as the intrinsics and extrinsics of the camera,\nrespectively).\n\nThe [rr.Pinhole archetype](https://www.rerun.io/docs/reference/types/archetypes/pinhole) is logged to\nthe [camera/image entity](recording://camera/image) and defines the intrinsics of the camera. This defines how to go\nfrom the 3D camera frame to the 2D image plane. The extrinsics are logged as an\n[rr.Transform3D archetype](https://www.rerun.io/docs/reference/types/archetypes/transform3d) to the\n[camera entity](recording://camera).\n\n### Reprojection error\nFor each image a [rr.Scalar archetype](https://www.rerun.io/docs/reference/types/archetypes/scalar)\ncontaining the average reprojection error of the keypoints is logged to the\n[plot/avg_reproj_err entity](recording://plot/avg_reproj_err).\n\n### 2D points\nThe 2D image points that are used to triangulate the 3D points are visualized by logging\n[rr.Points3D archetype](https://www.rerun.io/docs/reference/types/archetypes/points2d)\nto the [camera/image/keypoints entity](recording://camera/image/keypoints). Note that these keypoints are a child of the\n[camera/image entity](recording://camera/image), since the points should show in the image plane.\n\n### Colored 3D points\nThe colored 3D points were added to the scene by logging the\n[rr.Points3D archetype](https://www.rerun.io/docs/reference/types/archetypes/points3d)\nto the [points entity](recording://points):\n```python\nrr.log(\"points\", rr.Points3D(points, colors=point_colors), rr.AnyValues(error=point_errors))\n```\n**Note:** we added some [custom per-point errors](recording://points) that you can see when you\nhover over the points in the 3D view.\n\"\"\".strip()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-03-27T00:04:30.639687Z","iopub.execute_input":"2024-03-27T00:04:30.640169Z","iopub.status.idle":"2024-03-27T00:04:30.649532Z","shell.execute_reply.started":"2024-03-27T00:04:30.640134Z","shell.execute_reply":"2024-03-27T00:04:30.648621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Colmap Reconstruction\n\nThat took from 20:45 till 21:01 just to complete 100% without any image yet!\n\nNow preparing visualization engine.... from 21:01 till 21:03 No 3DImage at all! ","metadata":{}},{"cell_type":"code","source":"#By OldLufo https://www.kaggle.com/code/oldufo/colmap-3d-visualization-with-rerun-io\n\nFILTER_MIN_VISIBLE = 50\ndef scale_camera(camera, resize: tuple[int, int]) -> tuple[pycolmap.Camera, npt.NDArray[np.float_]]:\n    \"\"\"Scale the camera intrinsics to match the resized image.\"\"\"\n    assert camera.model == \"PINHOLE\"\n    new_width = resize[0]\n    new_height = resize[1]\n    scale_factor = np.array([new_width / camera.width, new_height / camera.height])\n\n    # For PINHOLE camera model, params are: [focal_length_x, focal_length_y, principal_point_x, principal_point_y]\n    new_params = np.append(camera.params[:2] * scale_factor, camera.params[2:] * scale_factor)\n\n    return (pycolmap.Camera(camera.id, camera.model, new_width, new_height, new_params), scale_factor)\n\n\n\ndef read_and_log_sparse_reconstruction(rec_path: Path, img_path: Path, filter_output: bool, resize: tuple[int, int] | None) -> None:\n    print(\"Reading sparse COLMAP reconstruction\")\n    reconstruction = pycolmap.Reconstruction(rec_path)\n    cameras = reconstruction.cameras\n    images = reconstruction.images\n    points3D = reconstruction.points3D\n    print(\"Building visualization by logging to Rerun\")\n\n    if filter_output:\n        # Filter out noisy points\n        points3D = {id: point for id, point in points3D.items() if point.color.any() and len(point.image_ids) > 2}\n\n    rr.log(\"description\", rr.TextDocument(DESCRIPTION, media_type=rr.MediaType.MARKDOWN), timeless=True)\n    rr.log(\"/\", rr.ViewCoordinates.RIGHT_HAND_Y_DOWN, timeless=True)\n    rr.log(\"plot/avg_reproj_err\", rr.SeriesLine(color=[240, 45, 58]), timeless=True)\n\n    # Iterate through images (video frames) logging data related to each frame.\n    ii=0\n    for image in tqdm(sorted(images.values(), key=lambda im: im.name)):  # type: ignore[no-any-return]\n        image_file = img_path / image.name.replace('.jpg', '.png')\n        if not os.path.exists(image_file):\n            continue\n        #print (image_file)\n\n        # COLMAP sets image ids that don't match the original video frame\n        idx_match = re.search(r\"\\d+\", image.name)\n        assert idx_match is not None\n        frame_idx = int(idx_match.group(0))\n\n\n        camera = cameras[image.camera_id]\n        if resize:\n            camera, scale_factor = scale_camera(camera, resize)\n        else:\n            scale_factor = np.array([1.0, 1.0])\n\n        visible_ids = [id_ for id_ in points3D.keys() if image.has_point3D(id_) ]\n        \n\n        if filter_output and len(visible_ids) < FILTER_MIN_VISIBLE:\n            continue\n\n        visible_xyzs = [points3D[idx] for idx in visible_ids]\n        visible_xys = np.array([x.xy for x in image.get_valid_points2D()])\n        if resize:\n            visible_xys *= scale_factor\n\n        rr.set_time_sequence(\"frame\", frame_idx)\n        try:\n            points = [point.xyz for point in visible_xyzs]\n        except Exception as e:\n            print (e)\n            continue\n        point_colors = [point.color for point in visible_xyzs]\n        point_errors = [point.error for point in visible_xyzs]\n\n        rr.log(\"plot/avg_reproj_err\", rr.Scalar(np.mean(point_errors)))\n\n        rr.log(\"points\", rr.Points3D(points, colors=point_colors), rr.AnyValues(error=point_errors))\n\n        # COLMAP's camera transform is \"camera from world\"\n        rr.log(\n            \"camera\", rr.Transform3D(translation=image.cam_from_world.translation,\n                                     rotation=rr.Quaternion(xyzw=image.cam_from_world.rotation.quat), from_parent=True)\n        )\n        rr.log(\"camera\", rr.ViewCoordinates.RDF, timeless=True)  # X=Right, Y=Down, Z=Forward\n\n        # Log camera intrinsics\n        assert str(camera.model) in  [\"CameraModelId.SIMPLE_PINHOLE\", \"CameraModelId.PINHOLE\"]\n        if str(camera.model) == \"CameraModelId.SIMPLE_PINHOLE\":\n            rr.log(\n                \"camera/image\",\n                rr.Pinhole(\n                    resolution=[camera.width, camera.height],\n                    focal_length=[camera.params[0], camera.params[0]],\n                    principal_point=camera.params[1:],\n                ),\n            )\n        else:\n            rr.log(\n                \"camera/image\",\n                rr.Pinhole(\n                    resolution=[camera.width, camera.height],\n                    focal_length=camera.params[:2],\n                    principal_point=camera.params[2:],\n                ),\n            )\n        if resize:\n            bgr = cv2.imread(str(image_file))\n            bgr = cv2.resize(bgr, resize)\n            rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)\n            rr.log(\"camera/image\", rr.Image(rgb).compress(jpeg_quality=75))\n        else:\n            rr.log(\"camera/image\", rr.ImageEncoded(path=img_path / image.name.replace('.jpg', '.png')))\n        rr.log(\"camera/image/keypoints\", rr.Points2D(visible_xys, colors=[34, 138, 167]))\n    print (\"Now preparing visualization engine\")\n# Whatever you want to visualize in notebook, you should start the rec = rr.memory_recording()\nrec = rr.memory_recording()\nread_and_log_sparse_reconstruction(Path(path_to_lizard), Path(path_to_lizard_images), filter_output=False, resize=None)\n# And after it finishes - show it by just calling it  rec\nrec","metadata":{"execution":{"iopub.status.busy":"2024-03-26T23:45:11.162711Z","iopub.execute_input":"2024-03-26T23:45:11.163126Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Church (It's faster than the lizard) started 21:06 till 21:07","metadata":{}},{"cell_type":"code","source":"FILTER_MIN_VISIBLE = 50\ndef scale_camera(camera, resize: tuple[int, int]) -> tuple[pycolmap.Camera, npt.NDArray[np.float_]]:\n    \"\"\"Scale the camera intrinsics to match the resized image.\"\"\"\n    assert camera.model == \"PINHOLE\"\n    new_width = resize[0]\n    new_height = resize[1]\n    scale_factor = np.array([new_width / camera.width, new_height / camera.height])\n\n    # For PINHOLE camera model, params are: [focal_length_x, focal_length_y, principal_point_x, principal_point_y]\n    new_params = np.append(camera.params[:2] * scale_factor, camera.params[2:] * scale_factor)\n\n    return (pycolmap.Camera(camera.id, camera.model, new_width, new_height, new_params), scale_factor)\n\n\n\ndef read_and_log_sparse_reconstruction(rec_path: Path, img_path: Path, filter_output: bool, resize: tuple[int, int] | None) -> None:\n    print(\"Reading sparse COLMAP reconstruction\")\n    reconstruction = pycolmap.Reconstruction(rec_path)\n    cameras = reconstruction.cameras\n    images = reconstruction.images\n    points3D = reconstruction.points3D\n    print(\"Building visualization by logging to Rerun\")\n\n    if filter_output:\n        # Filter out noisy points\n        points3D = {id: point for id, point in points3D.items() if point.color.any() and len(point.image_ids) > 2}\n\n    rr.log(\"description\", rr.TextDocument(DESCRIPTION, media_type=rr.MediaType.MARKDOWN), timeless=True)\n    rr.log(\"/\", rr.ViewCoordinates.RIGHT_HAND_Y_DOWN, timeless=True)\n    rr.log(\"plot/avg_reproj_err\", rr.SeriesLine(color=[240, 45, 58]), timeless=True)\n\n    # Iterate through images (video frames) logging data related to each frame.\n    ii=0\n    for image in tqdm(sorted(images.values(), key=lambda im: im.name)):  # type: ignore[no-any-return]\n        image_file = img_path / image.name.replace('.jpg', '.png')\n        if not os.path.exists(image_file):\n            continue\n        #print (image_file)\n\n        # COLMAP sets image ids that don't match the original video frame\n        idx_match = re.search(r\"\\d+\", image.name)\n        assert idx_match is not None\n        frame_idx = int(idx_match.group(0))\n\n\n        camera = cameras[image.camera_id]\n        if resize:\n            camera, scale_factor = scale_camera(camera, resize)\n        else:\n            scale_factor = np.array([1.0, 1.0])\n\n        visible_ids = [id_ for id_ in points3D.keys() if image.has_point3D(id_) ]\n        \n\n        if filter_output and len(visible_ids) < FILTER_MIN_VISIBLE:\n            continue\n\n        visible_xyzs = [points3D[idx] for idx in visible_ids]\n        visible_xys = np.array([x.xy for x in image.get_valid_points2D()])\n        if resize:\n            visible_xys *= scale_factor\n\n        rr.set_time_sequence(\"frame\", frame_idx)\n        try:\n            points = [point.xyz for point in visible_xyzs]\n        except Exception as e:\n            print (e)\n            continue\n        point_colors = [point.color for point in visible_xyzs]\n        point_errors = [point.error for point in visible_xyzs]\n\n        rr.log(\"plot/avg_reproj_err\", rr.Scalar(np.mean(point_errors)))\n\n        rr.log(\"points\", rr.Points3D(points, colors=point_colors), rr.AnyValues(error=point_errors))\n\n        # COLMAP's camera transform is \"camera from world\"\n        rr.log(\n            \"camera\", rr.Transform3D(translation=image.cam_from_world.translation,\n                                     rotation=rr.Quaternion(xyzw=image.cam_from_world.rotation.quat), from_parent=True)\n        )\n        rr.log(\"camera\", rr.ViewCoordinates.RDF, timeless=True)  # X=Right, Y=Down, Z=Forward\n\n        # Log camera intrinsics\n        assert str(camera.model) in  [\"CameraModelId.SIMPLE_PINHOLE\", \"CameraModelId.PINHOLE\"]\n        if str(camera.model) == \"CameraModelId.SIMPLE_PINHOLE\":\n            rr.log(\n                \"camera/image\",\n                rr.Pinhole(\n                    resolution=[camera.width, camera.height],\n                    focal_length=[camera.params[0], camera.params[0]],\n                    principal_point=camera.params[1:],\n                ),\n            )\n        else:\n            rr.log(\n                \"camera/image\",\n                rr.Pinhole(\n                    resolution=[camera.width, camera.height],\n                    focal_length=camera.params[:2],\n                    principal_point=camera.params[2:],\n                ),\n            )\n        if resize:\n            bgr = cv2.imread(str(image_file))\n            bgr = cv2.resize(bgr, resize)\n            rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)\n            rr.log(\"camera/image\", rr.Image(rgb).compress(jpeg_quality=75))\n        else:\n            rr.log(\"camera/image\", rr.ImageEncoded(path=img_path / image.name.replace('.jpg', '.png')))\n        rr.log(\"camera/image/keypoints\", rr.Points2D(visible_xys, colors=[34, 138, 167]))\n    print (\"Now preparing visualization engine\")\n# Whatever you want to visualize in notebook, you should start the rec = rr.memory_recording()\nrec = rr.memory_recording()\nread_and_log_sparse_reconstruction(Path(path_to_church), Path(path_to_church_images), filter_output=False, resize=None)\n# And after it finishes - show it by just calling it  rec\nrec","metadata":{"execution":{"iopub.status.busy":"2024-03-27T00:05:49.814090Z","iopub.execute_input":"2024-03-27T00:05:49.814656Z","iopub.status.idle":"2024-03-27T00:07:40.543182Z","shell.execute_reply.started":"2024-03-27T00:05:49.814599Z","shell.execute_reply":"2024-03-27T00:07:40.540799Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Why did a get a Church  but No Lizard?????","metadata":{}},{"cell_type":"markdown","source":"#Acknowledgements:\n\nOldUfo https://www.kaggle.com/code/oldufo/colmap-3d-visualization-with-rerun-io","metadata":{}}]}