{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":91498,"databundleVersionId":11655853,"sourceType":"competition"},{"sourceId":11660458,"sourceType":"datasetVersion","datasetId":7305232},{"sourceId":12031089,"sourceType":"datasetVersion","datasetId":7329211},{"sourceId":4535,"sourceType":"modelInstanceVersion","modelInstanceId":3327,"modelId":986},{"sourceId":17191,"sourceType":"modelInstanceVersion","modelInstanceId":14317,"modelId":21716},{"sourceId":17555,"sourceType":"modelInstanceVersion","modelInstanceId":14611,"modelId":22086}],"dockerImageVersionId":31011,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --no-index /kaggle/input/imc2025-packages-python-11-new/* --no-deps\n!mkdir -p /root/.cache/torch/hub/checkpoints\n!cp /kaggle/input/aliked/pytorch/aliked-n16/1/* /root/.cache/torch/hub/checkpoints/\n!cp /kaggle/input/lightglue/pytorch/aliked/1/* /root/.cache/torch/hub/checkpoints/\n!cp /kaggle/input/lightglue/pytorch/aliked/1/aliked_lightglue.pth /root/.cache/torch/hub/checkpoints/aliked_lightglue_v0-1_arxiv-pth","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:42.547708Z","iopub.execute_input":"2025-06-02T03:14:42.548341Z","iopub.status.idle":"2025-06-02T03:14:49.729697Z","shell.execute_reply.started":"2025-06-02T03:14:42.548307Z","shell.execute_reply":"2025-06-02T03:14:49.728856Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cp -r /kaggle/input/imc-2025-scripts-group /kaggle/working/","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:49.731688Z","iopub.execute_input":"2025-06-02T03:14:49.731924Z","iopub.status.idle":"2025-06-02T03:14:50.465301Z","shell.execute_reply.started":"2025-06-02T03:14:49.731904Z","shell.execute_reply":"2025-06-02T03:14:50.464189Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# {\n#     \"task\": \"get_dino_embeddings\",\n#     \"comment\": \"\",\n#     \"params\": {\n#         \"model_name\": \"../dinov2/pytorch/large/1\",\n#         \"input\": \"\",\n#         \"output\": \"embeddings.pkl\"\n#     }\n# },","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:50.466478Z","iopub.execute_input":"2025-06-02T03:14:50.466797Z","iopub.status.idle":"2025-06-02T03:14:50.471081Z","shell.execute_reply.started":"2025-06-02T03:14:50.466745Z","shell.execute_reply":"2025-06-02T03:14:50.470329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile /kaggle/working/pipeline.json\n\n[\n    {\n        \"task\": \"get_exif\",\n        \"comment\": \"\",\n        \"params\": {\n            \"input\": \"\",\n            \"output\": \"h_w_exif.json\"\n        }\n    },\n    {\n        \"task\": \"get_dino_embeddings\",\n        \"comment\": \"\",\n        \"params\": {\n            \"model_name\": \"../dinov2/pytorch/large/1\",\n            \"input\": \"\",\n            \"output\": \"embeddings.pkl\"\n        }\n    },\n    {\n        \"task\": \"get_image_pair_exhaustive\",\n        \"comment\": \"\",\n        \"params\": {\n            \"input\": \"\",\n            \"output\": \"image_pair.csv\"\n        }\n    },\n    {\n        \"task\": \"rotate_matching_find_best\",\n        \"comment\": \"aliked_LightGlue (imsize=840)\",\n        \"params\": {\n            \"matcher\": \"LightGlue\",\n            \"extractor\": \"aliked\",\n            \"input\": {\n                \"image_pair\": \"image_pair.csv\"\n            },\n            \"output\": {\n                \"keypoints\": \"_keypoints.h5\",\n                \"descriptions\": \"_descriptions.h5\",\n                \"matches\": \"_matches.h5\",\n                \"image_pair_csv\": \"image_pair.csv\"\n            },\n            \"enable_rotation\": true,\n            \"sufficient_matching_num\": 512,\n            \"keypoint_detection_args\": {\n                \"extractor_conf\": {\n                    \"max_num_keypoints\": 1024\n                },\n                \"preprocess_conf\": {\n                    \"resize\": 840\n                },\n                \"dtype\": \"float32\"\n            },\n            \"keypoint_matching_args\": {\n                \"matcher_params\": {\n                    \"filter_threshold\": 0.1,\n                    \"width_confidence\": 0.95,\n                    \"depth_confidence\": 0.9,\n                    \"mp\": true\n                },\n                \"min_matches\": 10,\n                \"verbose\": false\n            }\n        }\n    },\n    {\n        \"task\": \"matching\",\n        \"comment\": \"aliked_LightGlue (imsize=1280)\",\n        \"params\": {\n            \"matcher\": \"LightGlue\",\n            \"extractor\": \"aliked\",\n            \"input\": {\n                \"image_pair\": \"image_pair.csv\"\n            },\n            \"output\": {\n                \"descriptions\": \"descriptions_2.h5\",\n                \"keypoints\": \"keypoints_2.h5\",\n                \"matches\": \"matches_2.h5\"\n            },\n            \"keypoint_detection_args\": {\n                \"extractor_conf\": {\n                    \"max_num_keypoints\": 8192\n                },\n                \"preprocess_conf\": {\n                    \"resize\": 1280\n                },\n                \"dtype\": \"float32\"\n            },\n            \"keypoint_matching_args\": {\n                \"matcher_params\": {\n                    \"filter_threshold\": 0.2,\n                    \"width_confidence\": -1,\n                    \"depth_confidence\": -1,\n                    \"mp\": true\n                },\n                \"min_matches\": 30,\n                \"verbose\": false\n            }\n        }\n    },\n    {\n        \"task\": \"count_matching_num\",\n        \"comment\": \"\",\n        \"params\": {\n            \"input\": {\n                \"image_pair\": \"image_pair.csv\",\n                \"matches\": \"matches_2.h5\"\n            },\n            \"output\": \"image_pair.csv\"\n        }\n    },\n    {\n        \"task\": \"matching\",\n        \"comment\": \"aliked_LightGlue (imsize=1088)\",\n        \"params\": {\n            \"matcher\": \"LightGlue\",\n            \"extractor\": \"aliked\",\n            \"input\": {\n                \"image_pair\": \"image_pair.csv\"\n            },\n            \"output\": {\n                \"keypoints\": \"keypoints_4.h5\",\n                \"descriptions\": \"descriptions_4.h5\",\n                \"matches\": \"matches_4.h5\"\n            },\n            \"keypoint_detection_args\": {\n                \"extractor_conf\": {\n                    \"max_num_keypoints\": 8192\n                },\n                \"preprocess_conf\": {\n                    \"resize\": 1088\n                },\n                \"dtype\": \"float32\"\n            },\n            \"keypoint_matching_args\": {\n                \"matcher_params\": {\n                    \"filter_threshold\": 0.2,\n                    \"width_confidence\": -1,\n                    \"depth_confidence\": -1,\n                    \"mp\": true\n                },\n                \"min_matches\": 0,\n                \"verbose\": false\n            }\n        }\n    },\n    {\n        \"task\": \"concat\",\n        \"comment\": \"\",\n        \"params\": {\n            \"input\": {\n                \"keypoints\": [\n                    \"keypoints_2.h5\",\n                    \"keypoints_4.h5\"\n                ],\n                \"matches\": [\n                    \"matches_2.h5\",\n                    \"matches_4.h5\"\n                ]\n            },\n            \"output\": {\n                \"keypoints\": \"keypoints_orig.h5\",\n                \"matches\": \"matches_orig.h5\"\n            }\n        }\n    },\n    {\n        \"task\": \"rem_less_match_pair\",\n        \"comment\": \"\",\n        \"params\": {\n            \"input\": \"matches_orig.h5\",\n            \"output\": \"matches_orig.h5\",\n            \"th_matching_num\": 100\n        }\n    },\n    {\n        \"task\": \"count_matching_num\",\n        \"comment\": \"\",\n        \"params\": {\n            \"input\": {\n                \"image_pair\": \"image_pair.csv\",\n                \"matches\": \"matches_orig.h5\"\n            },\n            \"output\": \"image_pair.csv\"\n        }\n    },\n    {\n        \"task\": \"sfm_mkpc\",\n        \"comment\": \"\",\n        \"params\": {\n            \"input\": {\n                \"keypoints\": \"keypoints_orig.h5\",\n                \"matches\": \"matches_orig.h5\"\n            },\n            \"output\": \"mkpc_rect.json\",\n            \"thresh\": 0.15,\n            \"db_scan_param\": {\n                \"eps\": 0.05,\n                \"min_samples\": 16\n            },\n            \"mkpt_rect_parms\": {\n                \"crop_scale\": [1.05, 1.05],\n                \"inliner_rate_thresh\": 0.2\n            }\n        }\n    },\n    {\n        \"task\": \"matching\",\n        \"comment\": \"aliked_LightGlue (imsize=1280)\",\n        \"params\": {\n            \"matcher\": \"LightGlue\",\n            \"extractor\": \"aliked\",\n            \"input\": {\n                \"image_pair\": \"image_pair.csv\",\n                \"rects\": \"mkpc_rect.json\"\n            },\n            \"output\": {\n                \"keypoints\": \"keypoints_crop2.h5\",\n                \"descriptions\": \"descriptions_crop2.h5\",\n                \"matches\": \"matches_crop2.h5\"\n            },\n            \"keypoint_detection_args\": {\n                \"extractor_conf\": {\n                    \"max_num_keypoints\": 8192\n                },\n                \"preprocess_conf\": {\n                    \"resize\": 1280\n                },\n                \"dtype\": \"float32\"\n            },\n            \"keypoint_matching_args\": {\n                \"matcher_params\": {\n                    \"filter_threshold\": 0.2,\n                    \"width_confidence\": -1,\n                    \"depth_confidence\": -1,\n                    \"mp\": true\n                },\n                \"min_matches\": 50,\n                \"verbose\": false\n            }\n        }\n    },\n    {\n        \"task\": \"concat\",\n        \"comment\": \"\",\n        \"params\": {\n            \"input\": {\n                \"keypoints\": [\n                    \"keypoints_orig.h5\",\n                    \"keypoints_crop2.h5\"\n                ],\n                \"matches\": [\n                    \"matches_orig.h5\",\n                    \"matches_crop2.h5\"\n                ]\n            },\n            \"output\": {\n                \"keypoints\": \"keypoints.h5\",\n                \"matches\": \"matches.h5\"\n            }\n        }\n    },\n    {\n        \"task\": \"rem_less_match_pair\",\n        \"comment\": \"\",\n        \"params\": {\n            \"input\": \"matches.h5\",\n            \"output\": \"matches.h5\",\n            \"th_matching_num\": 50\n        }\n    },\n    {\n        \"task\": \"ransac\",\n        \"comment\": \"\",\n        \"params\": {\n            \"input\": {\n                \"keypoints\": \"keypoints.h5\",\n                \"matches\": \"matches.h5\"\n            },\n            \"output\": {\n                \"matches\": \"matches.h5\",\n                \"fms\": \"fms.pkl\"\n            },\n            \"min_matches\": 0,\n            \"ransac_params\": {\n                \"param1\": 5,\n                \"param2\": 0.9999,\n                \"maxIters\": 50000\n            }\n        }\n    },\n    {\n        \"task\": \"count_matching_num\",\n        \"comment\": \"\",\n        \"params\": {\n            \"input\": {\n                \"image_pair\": \"image_pair.csv\",\n                \"matches\": \"matches.h5\"\n            },\n            \"output\": \"image_pair.csv\"\n        }\n    }\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:50.472037Z","iopub.execute_input":"2025-06-02T03:14:50.472280Z","iopub.status.idle":"2025-06-02T03:14:50.493452Z","shell.execute_reply.started":"2025-06-02T03:14:50.472257Z","shell.execute_reply":"2025-06-02T03:14:50.492823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# %%writefile /kaggle/working/pipeline.json\n\n# [\n#     {\n#         \"task\": \"get_exif\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": \"\",\n#             \"output\": \"h_w_exif.json\"\n#         }\n#     },\n#     {\n#         \"task\": \"get_dino_embeddings\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"model_name\": \"../dinov2/pytorch/large/1\",\n#             \"input\": \"\",\n#             \"output\": \"embeddings.pkl\"\n#         }\n#     },\n#     {\n#         \"task\": \"get_image_pair_exhaustive\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": \"\",\n#             \"output\": \"image_pair.csv\"\n#         }\n#     },\n#     {\n#         \"task\": \"rotate_matching_find_best\",\n#         \"comment\": \"aliked_LightGlue (imsize=840)\",\n#         \"params\": {\n#             \"matcher\": \"LightGlue\",\n#             \"extractor\": \"aliked\",\n#             \"input\": {\n#                 \"image_pair\": \"image_pair.csv\"\n#             },\n#             \"output\": {\n#                 \"keypoints\": \"_keypoints.h5\",\n#                 \"descriptions\": \"_descriptions.h5\",\n#                 \"matches\": \"_matches.h5\",\n#                 \"image_pair_csv\": \"image_pair_rot.csv\"\n#             },\n#             \"enable_rotation\": true,\n#             \"sufficient_matching_num\": 512,\n#             \"keypoint_detection_args\": {\n#                 \"extractor_conf\": {\n#                     \"max_num_keypoints\": 1024\n#                 },\n#                 \"preprocess_conf\": {\n#                     \"resize\": 840\n#                 },\n#                 \"dtype\": \"float32\"\n#             },\n#             \"keypoint_matching_args\": {\n#                 \"matcher_params\": {\n#                     \"filter_threshold\": 0.1,\n#                     \"width_confidence\": 0.95,\n#                     \"depth_confidence\": 0.9,\n#                     \"mp\": true\n#                 },\n#                 \"min_matches\": 10,\n#                 \"verbose\": false\n#             }\n#         }\n#     },\n#     {\n#         \"task\": \"matching\",\n#         \"comment\": \"aliked_LightGlue (imsize=1280)\",\n#         \"params\": {\n#             \"matcher\": \"LightGlue\",\n#             \"extractor\": \"aliked\",\n#             \"input\": {\n#                 \"image_pair\": \"image_pair_rot.csv\"\n#             },\n#             \"output\": {\n#                 \"descriptions\": \"descriptions_2.h5\",\n#                 \"keypoints\": \"keypoints_2.h5\",\n#                 \"matches\": \"matches_2.h5\"\n#             },\n#             \"keypoint_detection_args\": {\n#                 \"extractor_conf\": {\n#                     \"max_num_keypoints\": 8192\n#                 },\n#                 \"preprocess_conf\": {\n#                     \"resize\": 1280\n#                 },\n#                 \"dtype\": \"float32\"\n#             },\n#             \"keypoint_matching_args\": {\n#                 \"matcher_params\": {\n#                     \"filter_threshold\": 0.2,\n#                     \"width_confidence\": -1,\n#                     \"depth_confidence\": -1,\n#                     \"mp\": true\n#                 },\n#                 \"min_matches\": 30,\n#                 \"verbose\": false\n#             }\n#         }\n#     },\n#     {\n#         \"task\": \"count_matching_num\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"image_pair\": \"image_pair_rot.csv\",\n#                 \"matches\": \"matches_2.h5\"\n#             },\n#             \"output\": \"image_pair.csv\"\n#         }\n#     },\n#     {\n#         \"task\": \"matching\",\n#         \"comment\": \"aliked_LightGlue (imsize=1088)\",\n#         \"params\": {\n#             \"matcher\": \"LightGlue\",\n#             \"extractor\": \"aliked\",\n#             \"input\": {\n#                 \"image_pair\": \"image_pair.csv\"\n#             },\n#             \"output\": {\n#                 \"keypoints\": \"keypoints_4.h5\",\n#                 \"descriptions\": \"descriptions_4.h5\",\n#                 \"matches\": \"matches_4.h5\"\n#             },\n#             \"keypoint_detection_args\": {\n#                 \"extractor_conf\": {\n#                     \"max_num_keypoints\": 8192\n#                 },\n#                 \"preprocess_conf\": {\n#                     \"resize\": 1088\n#                 },\n#                 \"dtype\": \"float32\"\n#             },\n#             \"keypoint_matching_args\": {\n#                 \"matcher_params\": {\n#                     \"filter_threshold\": 0.2,\n#                     \"width_confidence\": -1,\n#                     \"depth_confidence\": -1,\n#                     \"mp\": true\n#                 },\n#                 \"min_matches\": 0,\n#                 \"verbose\": false\n#             }\n#         }\n#     },\n#     {\n#         \"task\": \"concat\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"keypoints\": [\n#                     \"keypoints_2.h5\",\n#                     \"keypoints_4.h5\"\n#                 ],\n#                 \"matches\": [\n#                     \"matches_2.h5\",\n#                     \"matches_4.h5\"\n#                 ]\n#             },\n#             \"output\": {\n#                 \"keypoints\": \"keypoints_orig.h5\",\n#                 \"matches\": \"matches_orig.h5\"\n#             }\n#         }\n#     },\n#     {\n#         \"task\": \"rem_less_match_pair\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": \"matches_orig.h5\",\n#             \"output\": \"matches_orig.h5\",\n#             \"th_matching_num\": 100\n#         }\n#     },\n#     {\n#         \"task\": \"count_matching_num\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"image_pair\": \"image_pair.csv\",\n#                 \"matches\": \"matches_orig.h5\"\n#             },\n#             \"output\": \"image_pair.csv\"\n#         }\n#     },\n#     {\n#         \"task\": \"sfm_mkpc\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"keypoints\": \"keypoints_orig.h5\",\n#                 \"matches\": \"matches_orig.h5\"\n#             },\n#             \"output\": \"mkpc_rect.json\",\n#             \"thresh\": 0.15,\n#             \"db_scan_param\": {\n#                 \"eps\": 0.05,\n#                 \"min_samples\": 16\n#             },\n#             \"mkpt_rect_parms\": {\n#                 \"crop_scale\": [1.05, 1.05],\n#                 \"inliner_rate_thresh\": 0.2\n#             }\n#         }\n#     },\n#     {\n#         \"task\": \"matching\",\n#         \"comment\": \"aliked_LightGlue (imsize=1280)\",\n#         \"params\": {\n#             \"matcher\": \"LightGlue\",\n#             \"extractor\": \"aliked\",\n#             \"input\": {\n#                 \"image_pair\": \"image_pair.csv\",\n#                 \"rects\": \"mkpc_rect.json\"\n#             },\n#             \"output\": {\n#                 \"keypoints\": \"keypoints_crop2.h5\",\n#                 \"descriptions\": \"descriptions_crop2.h5\",\n#                 \"matches\": \"matches_crop2.h5\"\n#             },\n#             \"keypoint_detection_args\": {\n#                 \"extractor_conf\": {\n#                     \"max_num_keypoints\": 8192\n#                 },\n#                 \"preprocess_conf\": {\n#                     \"resize\": 1280\n#                 },\n#                 \"dtype\": \"float32\"\n#             },\n#             \"keypoint_matching_args\": {\n#                 \"matcher_params\": {\n#                     \"filter_threshold\": 0.2,\n#                     \"width_confidence\": -1,\n#                     \"depth_confidence\": -1,\n#                     \"mp\": true\n#                 },\n#                 \"min_matches\": 50,\n#                 \"verbose\": false\n#             }\n#         }\n#     },\n#     {\n#         \"task\": \"concat\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"keypoints\": [\n#                     \"keypoints_orig.h5\",\n#                     \"keypoints_crop2.h5\"\n#                 ],\n#                 \"matches\": [\n#                     \"matches_orig.h5\",\n#                     \"matches_crop2.h5\"\n#                 ]\n#             },\n#             \"output\": {\n#                 \"keypoints\": \"keypoints_not_dense.h5\",\n#                 \"matches\": \"matches_not_dense.h5\"\n#             }\n#         }\n#     },\n#     {\n#         \"task\": \"rem_less_match_pair\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": \"matches_not_dense.h5\",\n#             \"output\": \"matches_not_dense.h5\",\n#             \"th_matching_num\": 50\n#         }\n#     },\n#     {\n#         \"task\": \"ransac\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"keypoints\": \"keypoints_not_dense.h5\",\n#                 \"matches\": \"matches_not_dense.h5\"\n#             },\n#             \"output\": {\n#                 \"matches\": \"matches_not_dense.h5\",\n#                 \"fms\": \"fms_not_dense.pkl\"\n#             },\n#             \"min_matches\": 0,\n#             \"ransac_params\": {\n#                 \"param1\": 5,\n#                 \"param2\": 0.9999,\n#                 \"maxIters\": 50000\n#             }\n#         }\n#     },\n#     {\n#         \"task\": \"count_matching_num\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"image_pair\": \"image_pair.csv\",\n#                 \"matches\": \"matches_not_dense.h5\"\n#             },\n#             \"output\": \"image_pair.csv\"\n#         }\n#     },\n#     {\n#         \"task\": \"get_image_pair_exhaustive_within_component\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"image_pair\": \"image_pair.csv\",\n#                 \"image_pair_rotation\": \"image_pair_rot.csv\"\n#             },\n#             \"output\": \"image_pair.csv\"\n#         }\n#     },\n#     {\n#         \"task\": \"matching\",\n#         \"comment\": \"aliked_LightGlue (imsize=1024)\",\n#         \"params\": {\n#             \"matcher\": \"LightGlue\",\n#             \"extractor\": \"aliked\",\n#             \"input\": {\n#                 \"image_pair\": \"image_pair.csv\",\n#                 \"rects\": \"mkpc_rect.json\"\n#             },\n#             \"output\": {\n#                 \"descriptions\": \"descriptions_dense.h5\",\n#                 \"keypoints\": \"keypoints_dense.h5\",\n#                 \"matches\": \"matches_dense.h5\"\n#             },\n#             \"keypoint_detection_args\": {\n#                 \"extractor_conf\": {\n#                     \"max_num_keypoints\": 4096\n#                 },\n#                 \"preprocess_conf\": {\n#                     \"resize\": 1024\n#                 },\n#                 \"dtype\": \"float32\"\n#             },\n#             \"keypoint_matching_args\": {\n#                 \"matcher_params\": {\n#                     \"filter_threshold\": 0.01,\n#                     \"width_confidence\": -1,\n#                     \"depth_confidence\": -1,\n#                     \"mp\": true\n#                 },\n#                 \"min_matches\": 50,\n#                 \"verbose\": false\n#             }\n#         }\n#     },\n#     {\n#         \"task\": \"concat\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"keypoints\": [\n#                     \"keypoints_not_dense.h5\",\n#                     \"keypoints_dense.h5\"\n#                 ],\n#                 \"matches\": [\n#                     \"matches_not_dense.h5\",\n#                     \"matches_dense.h5\"\n#                 ]\n#             },\n#             \"output\": {\n#                 \"keypoints\": \"keypoints.h5\",\n#                 \"matches\": \"matches.h5\"\n#             }\n#         }\n#     },\n#     {\n#         \"task\": \"count_matching_num\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"image_pair\": \"image_pair.csv\",\n#                 \"matches\": \"matches.h5\"\n#             },\n#             \"output\": \"image_pair.csv\"\n#         }\n#     },\n#     {\n#         \"task\": \"ransac\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"keypoints\": \"keypoints.h5\",\n#                 \"matches\": \"matches.h5\"\n#             },\n#             \"output\": {\n#                 \"matches\": \"matches.h5\",\n#                 \"fms\": \"fms.pkl\"\n#             },\n#             \"min_matches\": 0,\n#             \"ransac_params\": {\n#                 \"param1\": 5,\n#                 \"param2\": 0.9999,\n#                 \"maxIters\": 50000\n#             }\n#         }\n#     },\n#     {\n#         \"task\": \"rem_less_match_pair\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": \"matches.h5\",\n#             \"output\": \"matches.h5\",\n#             \"th_matching_num\": 70\n#         }\n#     },\n#     {\n#         \"task\": \"count_matching_num\",\n#         \"comment\": \"\",\n#         \"params\": {\n#             \"input\": {\n#                 \"image_pair\": \"image_pair.csv\",\n#                 \"matches\": \"matches.h5\"\n#             },\n#             \"output\": \"image_pair.csv\"\n#         }\n#     }\n# ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:50.495412Z","iopub.execute_input":"2025-06-02T03:14:50.495837Z","iopub.status.idle":"2025-06-02T03:14:50.517700Z","shell.execute_reply.started":"2025-06-02T03:14:50.495814Z","shell.execute_reply":"2025-06-02T03:14:50.516975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile /kaggle/working/imc-2025-scripts-group/config.py\n\nimport os\nimport os.path as osp\n\nclass Config:\n    input_dir_root = \"/kaggle/input/image-matching-challenge-2025\"\n    output_dir = \"/tmp\"\n    check_exist_dir = False\n    \n    # input_csv = osp.join(input_dir_root, \"train_labels.csv\")\n    input_csv = osp.join(input_dir_root, \"sample_submission.csv\")\n    \n    thr_config = {\n        'thr': 0.6,\n        # 'thr': 1.1,\n        'tolerance': 0.5,\n        'clustering_threshold': 0.85,\n        # 'clustering_threshold': 1.1,\n        'use_centroids': False,\n        'tries_for_second_img': 3,\n        'rem_only_registered_imgs_having_all_pairs_registered': True,\n        'fix_pair_for_iter': True, \n        'fix_pair_for_cluster': True,\n        'verbose': False\n    }\n\n    target_datasets = None\n    # target_datasets = ['imc2023_haiper', 'pt_brandenburg_british_buckingham']\n    # target_datasets = ['amy_gardens', 'ETs', 'fbk_vineyard', 'imc2023_haiper', 'imc2023_heritage', 'imc2023_theather_imc2024_church', \n    #                    'imc2024_dioscuri_baalshamin', 'imc2024_lizard_pond', 'pt_brandenburg_british_buckingham', \n    #                    'pt_piazzasanmarco_grandplace', 'pt_sacrecoeur_trevi_tajmahal', 'pt_stpeters_stpauls', 'stairs']\n\n    pipeline_json = \"/kaggle/working/pipeline.json\"\n    \n    colmap_mapper_options = {\n        \"min_model_size\": 3, # By default colmap does not generate a reconstruction if less than 10 images are registered. Lower it to 3.\n        \"max_num_models\": 2,\n        \"num_threads\": 1,\n    }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:50.518447Z","iopub.execute_input":"2025-06-02T03:14:50.518654Z","iopub.status.idle":"2025-06-02T03:14:50.538675Z","shell.execute_reply.started":"2025-06-02T03:14:50.518627Z","shell.execute_reply":"2025-06-02T03:14:50.538006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport os.path as osp\nimport pandas as pd\nimport torch\nimport shutil\n\nimport sys\nsys.path.append(\"imc-2025-scripts-group\")\nfrom config import Config as config\n\nBASE_PATH = \"/kaggle/input/image-matching-challenge-2025\"\nroot_path = f\"{BASE_PATH}/train\" if \"train\" in osp.basename(config.input_csv) else f\"{BASE_PATH}/test\"\nif config.target_datasets is None:\n    scenes = [x for x in os.listdir(root_path) if os.path.isdir(f\"{root_path}/{x}\")]\nelse:\n    scenes = config.target_datasets\ndata_num_list = [sum(len(files) for _, _, files in os.walk(f\"{root_path}/{scene}\")) for scene in scenes]\ndata_num_list","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:50.539462Z","iopub.execute_input":"2025-06-02T03:14:50.539649Z","iopub.status.idle":"2025-06-02T03:14:55.359274Z","shell.execute_reply.started":"2025-06-02T03:14:50.539634Z","shell.execute_reply":"2025-06-02T03:14:55.358437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def partition_with_index(lst, lst2):\n    indexed_lst = sorted(enumerate(lst), key=lambda x: x[1], reverse=True)\n    groupA = []\n    groupB = []\n    groupA2 = []\n    groupB2 = []\n    \n    for index, item in indexed_lst:\n        if sum([lst[i] for i in groupA]) <= sum([lst[i] for i in groupB]):\n            groupA.append(index)\n            groupA2.append(lst2[index])\n        else:\n            groupB.append(index)\n            groupB2.append(lst2[index])\n    \n    return groupA2, groupB2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:55.360269Z","iopub.execute_input":"2025-06-02T03:14:55.360672Z","iopub.status.idle":"2025-06-02T03:14:55.365931Z","shell.execute_reply.started":"2025-06-02T03:14:55.360647Z","shell.execute_reply":"2025-06-02T03:14:55.365191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scenes0, scenes1 = partition_with_index(data_num_list, scenes)\nprint(scenes0)\nprint(scenes1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:55.366743Z","iopub.execute_input":"2025-06-02T03:14:55.367036Z","iopub.status.idle":"2025-06-02T03:14:55.390290Z","shell.execute_reply.started":"2025-06-02T03:14:55.367011Z","shell.execute_reply":"2025-06-02T03:14:55.389607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.cuda.empty_cache()\nimport gc\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:55.391271Z","iopub.execute_input":"2025-06-02T03:14:55.391587Z","iopub.status.idle":"2025-06-02T03:14:55.504937Z","shell.execute_reply.started":"2025-06-02T03:14:55.391558Z","shell.execute_reply":"2025-06-02T03:14:55.504178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_command(device_id, target_datasets):\n    cmd = f\"python imc-2025-scripts-group/inference_mp.py --device_id {device_id} --output_dir /tmp/tmp{device_id}\"\n    if len(target_datasets) > 0:\n        cmd += \" --target_datasets\"\n        for scene in target_datasets:\n            cmd += f\" {scene}\"\n    return cmd.split(\" \")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:55.505662Z","iopub.execute_input":"2025-06-02T03:14:55.505965Z","iopub.status.idle":"2025-06-02T03:14:55.519776Z","shell.execute_reply.started":"2025-06-02T03:14:55.505947Z","shell.execute_reply":"2025-06-02T03:14:55.519180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import subprocess\nif scenes0:\n    proc0 = subprocess.Popen(make_command(0, scenes0))\nif scenes1:\n    proc1 = subprocess.Popen(make_command(1, scenes1))\nif scenes0:\n    proc0.wait()\nif scenes1:\n    proc1.wait()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:14:55.520615Z","iopub.execute_input":"2025-06-02T03:14:55.520868Z","iopub.status.idle":"2025-06-02T03:19:30.321665Z","shell.execute_reply.started":"2025-06-02T03:14:55.520851Z","shell.execute_reply":"2025-06-02T03:19:30.321031Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if scenes0:\n    df1 = pd.read_csv(\"/tmp/tmp0/submission.csv\")\nif scenes1:\n    df2 = pd.read_csv(\"/tmp/tmp1/submission.csv\")\nif scenes0 and scenes1:\n    df = pd.concat([df1, df2]).reset_index(drop=True)\nelse:\n    if scenes0:\n        df = df1.copy(deep=True)\n    if scenes1:\n        df = df2.copy(deep=True)\nFINAL_SUB_TMP_PATH = \"/tmp/final_submission.csv\"\ndf.to_csv(FINAL_SUB_TMP_PATH, index=False)\n\nif osp.exists(\"/tmp/tmp0/\"):\n    shutil.rmtree(\"/tmp/tmp0/\")\nif osp.exists(\"/tmp/tmp1/\"):\n    shutil.rmtree(\"/tmp/tmp1/\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:19:30.322634Z","iopub.execute_input":"2025-06-02T03:19:30.323302Z","iopub.status.idle":"2025-06-02T03:19:30.346349Z","shell.execute_reply.started":"2025-06-02T03:19:30.323273Z","shell.execute_reply":"2025-06-02T03:19:30.345581Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:19:30.348743Z","iopub.execute_input":"2025-06-02T03:19:30.349085Z","iopub.status.idle":"2025-06-02T03:19:30.369675Z","shell.execute_reply.started":"2025-06-02T03:19:30.349067Z","shell.execute_reply":"2025-06-02T03:19:30.369098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"OUTPUT_ROOT = '/kaggle/working'\nFINAL_SUB_PATH = f\"{OUTPUT_ROOT}/submission.csv\"\n!cp {FINAL_SUB_TMP_PATH} {FINAL_SUB_PATH}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:19:30.370423Z","iopub.execute_input":"2025-06-02T03:19:30.370658Z","iopub.status.idle":"2025-06-02T03:19:30.499355Z","shell.execute_reply.started":"2025-06-02T03:19:30.370641Z","shell.execute_reply":"2025-06-02T03:19:30.498480Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Compute Metric\nif \"train\" in config.input_csv:\n    from metric import score\n    from time import time\n    t = time()\n    final_score, dataset_scores = score(\n        gt_csv=\"/kaggle/input/image-matching-challenge-2025/train_labels.csv\",\n        # gt_csv=config.input_csv,\n        user_csv=FINAL_SUB_PATH,\n        thresholds_csv=osp.join(config.input_dir_root, 'train_thresholds.csv'),\n        mask_csv=None,\n        target_datasets=scenes,\n        # target_datasets=config.target_datasets,\n        inl_cf=0,\n        strict_cf=-1,\n        verbose=True,\n    )\n    print(f'Computed metric in: {time() - t:.02f} sec.')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:19:30.500341Z","iopub.execute_input":"2025-06-02T03:19:30.500544Z","iopub.status.idle":"2025-06-02T03:19:30.830030Z","shell.execute_reply.started":"2025-06-02T03:19:30.500524Z","shell.execute_reply":"2025-06-02T03:19:30.829198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !zip -qr /kaggle/working/ETs.zip /tmp/tmp1/feature_outputs/ETs\n# scene ET: mAA=50.00%\n# scene another_ET: mAA=46.43%\n# ETs: score=64.94% (mAA=48.08%, clusterness=100.00%)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T03:19:30.830946Z","iopub.execute_input":"2025-06-02T03:19:30.831816Z","iopub.status.idle":"2025-06-02T03:19:30.834841Z","shell.execute_reply.started":"2025-06-02T03:19:30.831754Z","shell.execute_reply":"2025-06-02T03:19:30.834072Z"}},"outputs":[],"execution_count":null}]}