{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":49349,"databundleVersionId":5447706,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":5627.472002,"end_time":"2025-10-11T06:46:11.553380","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-10-11T05:12:24.081378","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Fountain Photos: Gaussian Splat wo/Camera Info**\n\n","metadata":{"papermill":{"duration":0.002703,"end_time":"2025-10-11T05:12:29.869591","exception":false,"start_time":"2025-10-11T05:12:29.866888","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### **You can see the result in 3D gaussian splat viewer**\n\nhttps://splat-three.vercel.app/?url=fountain_photo.splat","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nimport subprocess\nimport shutil\nfrom pathlib import Path\nimport cv2\n\n# Configuration\n# IMAGE_PATH: Path to the image folder\n# WORK_DIR: Working directory for Gaussian Splatting repository\n# OUTPUT_DIR: Directory for the final video output\n# COLMAP_DIR: Directory for COLMAP data\n\nIMAGE_PATH = \"/kaggle/input/image-matching-challenge-2023/train/haiper/fountain/images_full\"\nWORK_DIR = '/kaggle/working/gaussian_splatting'\nOUTPUT_DIR = '/kaggle/working/output'\nCOLMAP_DIR = '/kaggle/working/colmap_data'","metadata":{"execution":{"iopub.execute_input":"2025-10-11T05:12:29.875009Z","iopub.status.busy":"2025-10-11T05:12:29.874773Z","iopub.status.idle":"2025-10-11T05:12:30.341829Z","shell.execute_reply":"2025-10-11T05:12:30.341197Z"},"papermill":{"duration":0.471162,"end_time":"2025-10-11T05:12:30.343205","exception":false,"start_time":"2025-10-11T05:12:29.872043","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def setup_environment():\n    \"\"\"Install necessary packages and clone the repository\"\"\"\n    print(\"Setting up environment...\")\n    \n    # Virtual display setup\n    print(\"Setting up virtual display...\")\n    subprocess.run(['apt-get', 'update', '-qq'], check=True)\n    subprocess.run(['apt-get', 'install', '-y', '-qq', 'xvfb'], check=True)\n    \n    # Set DISPLAY environment variable\n    os.environ['QT_QPA_PLATFORM'] = 'offscreen'\n    os.environ['DISPLAY'] = ':99'\n    \n    # Start Xvfb\n    subprocess.Popen(['Xvfb', ':99', '-screen', '0', '1024x768x24'])\n    \n    # Install COLMAP\n    print(\"Installing COLMAP...\")\n    subprocess.run(['apt-get', 'install', '-y', '-qq', 'colmap'], check=True)\n    \n    # Install build dependencies for submodules\n    print(\"Installing build dependencies...\")\n    subprocess.run([\n        'apt-get', 'install', '-y', '-qq',\n        'build-essential', 'cmake', 'git'\n    ], check=True)\n    \n    # Clone Gaussian Splatting repository\n    if not os.path.exists(WORK_DIR):\n        print(\"Cloning Gaussian Splatting repository...\")\n        subprocess.run([\n            'git', 'clone', '--recursive',\n            'https://github.com/tztechno/gaussian-splatting.git',\n            WORK_DIR\n        ], check=True)\n    \n    os.chdir(WORK_DIR)\n    \n    # Install Python packages (including build tools)\n    print(\"Installing Python packages...\")\n    subprocess.run([\n        sys.executable, '-m', 'pip', 'install', '-q', '--upgrade',\n        'pip', 'setuptools', 'wheel', 'ninja'\n    ], check=True)\n    \n    subprocess.run([\n        sys.executable, '-m', 'pip', 'install', '-q',\n        'torch', 'torchvision', 'torchaudio',\n        'plyfile', 'tqdm', 'opencv-python', 'pillow'\n    ], check=True)\n    \n    # Build submodules\n    print(\"Building diff-gaussian-rasterization...\")\n    subprocess.run([\n        sys.executable, '-m', 'pip', 'install',\n        './submodules/diff-gaussian-rasterization'\n    ], check=True, cwd=WORK_DIR)\n    \n    print(\"Building simple-knn...\")\n    subprocess.run([\n        sys.executable, '-m', 'pip', 'install',\n        './submodules/simple-knn'\n    ], check=True, cwd=WORK_DIR)\n    \n    print(\"Environment setup complete!\")","metadata":{"execution":{"iopub.execute_input":"2025-10-11T05:12:30.349081Z","iopub.status.busy":"2025-10-11T05:12:30.348844Z","iopub.status.idle":"2025-10-11T05:12:30.355834Z","shell.execute_reply":"2025-10-11T05:12:30.355319Z"},"papermill":{"duration":0.011272,"end_time":"2025-10-11T05:12:30.356903","exception":false,"start_time":"2025-10-11T05:12:30.345631","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nfrom PIL import Image\nimport glob\nimport numpy as np \n\n\ndef process_frames_from_folder(image_folder, output_dir, max_frames=300, supported_formats=['jpg', 'jpeg', 'png', 'bmp']):\n    \"\"\"Process frames from image folder instead of video\"\"\"\n    print(f\"Processing frames from folder: {image_folder}\")\n    \n    os.makedirs(output_dir, exist_ok=True)\n    \n    # Search for supported image files\n    image_files = []\n    for fmt in supported_formats:\n        pattern = os.path.join(image_folder, f\"*.{fmt}\")\n        image_files.extend(glob.glob(pattern))\n        pattern = os.path.join(image_folder, f\"*.{fmt.upper()}\")\n        image_files.extend(glob.glob(pattern))\n    \n    # Sort by filename\n    image_files.sort()\n    \n    if not image_files:\n        raise ValueError(f\"No image files found in: {image_folder}\")\n    \n    print(f\"Found {len(image_files)} image files\")\n    \n    # Limit to maximum number of frames\n    image_files = image_files[:max_frames]\n    \n    saved_count = 0\n    \n    for i, image_path in enumerate(image_files):\n        try:\n            # Read the image\n            if image_path.lower().endswith(('.png','.jpeg','.bmp')):\n                # Read with OpenCV (color space conversion may be needed)\n                img = cv2.imread(image_path)\n                if img is None:\n                    print(f\"Warning: Could not read {image_path} with OpenCV, trying PIL\")\n                    pil_img = Image.open(image_path).convert('RGB') # Ensure RGB for conversion\n                    img = cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR)\n            else:\n                img = cv2.imread(image_path)\n            \n            if img is None:\n                print(f\"Warning: Could not read {image_path}, skipping\")\n                continue\n            \n            # Generate output path\n            output_path = os.path.join(output_dir, f\"frame_{saved_count:05d}.jpg\")\n            \n            # Save the image\n            cv2.imwrite(output_path, img, [cv2.IMWRITE_JPEG_QUALITY, 95])\n            saved_count += 1\n            \n        except Exception as e:\n            print(f\"Error processing {image_path}: {e}\")\n            continue\n    \n    print(f\"Processed {saved_count} frames from folder\")\n    \n    return saved_count\n\n","metadata":{"execution":{"iopub.execute_input":"2025-10-11T05:12:30.362078Z","iopub.status.busy":"2025-10-11T05:12:30.361873Z","iopub.status.idle":"2025-10-11T05:12:30.368832Z","shell.execute_reply":"2025-10-11T05:12:30.368169Z"},"papermill":{"duration":0.010825,"end_time":"2025-10-11T05:12:30.369878","exception":false,"start_time":"2025-10-11T05:12:30.359053","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"    def extract_frames(video_path, output_dir, fps=6, max_frames=300):\n        \"\"\"Extract frames from the video\"\"\"\n        print(f\"Extracting frames from video: {video_path}\")\n        \n        os.makedirs(output_dir, exist_ok=True)\n        \n        cap = cv2.VideoCapture(video_path)\n        if not cap.isOpened():\n            raise ValueError(f\"Cannot open video: {video_path}\")\n        \n        original_fps = cap.get(cv2.CAP_PROP_FPS)\n        print('original_fps',original_fps)\n        \n        total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n        \n        print(f\"Video Info: FPS={original_fps:.2f}, Total Frames={total_frames}\")\n        \n        # Calculate frame interval\n        frame_interval = max(1, int(original_fps / fps))\n        \n        frame_count = 0\n        saved_count = 0\n        \n        while saved_count < max_frames:\n            ret, frame = cap.read()\n            if not ret:\n                break\n            \n            if frame_count % frame_interval == 0:\n                output_path = os.path.join(output_dir, f\"frame_{saved_count:05d}.jpg\")\n                cv2.imwrite(output_path, frame, [cv2.IMWRITE_JPEG_QUALITY, 95])\n                saved_count += 1\n            \n            frame_count += 1\n        \n        cap.release()\n        print(f\"Extracted {saved_count} frames\")\n        \n        return saved_count","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.002027,"end_time":"2025-10-11T05:12:30.374090","exception":false,"start_time":"2025-10-11T05:12:30.372063","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def run_colmap_reconstruction(image_dir, colmap_dir):\n    \"\"\"Estimate camera poses and 3D point cloud with COLMAP\"\"\"\n    print(\"Running SfM reconstruction with COLMAP...\")\n    \n    database_path = os.path.join(colmap_dir, \"database.db\")\n    sparse_dir = os.path.join(colmap_dir, \"sparse\")\n    os.makedirs(sparse_dir, exist_ok=True)\n    \n    # Set environment variable\n    env = os.environ.copy()\n    env['QT_QPA_PLATFORM'] = 'offscreen'\n    \n    # Feature extraction\n    print(\"1/4: Extracting features...\")\n    subprocess.run([\n        'colmap', 'feature_extractor',\n        '--database_path', database_path,\n        '--image_path', image_dir,\n        '--ImageReader.single_camera', '1',\n        '--ImageReader.camera_model', 'OPENCV',\n        '--SiftExtraction.use_gpu', '0'  # Use CPU\n    ], check=True, env=env)\n    \n    # Feature matching\n    print(\"2/4: Matching features...\")\n    subprocess.run([\n        'colmap', 'sequential_matcher',  # Use sequential_matcher instead of exhaustive_matcher\n        '--database_path', database_path,\n        '--SiftMatching.use_gpu', '0'  # Use CPU\n    ], check=True, env=env)\n    \n    # Sparse reconstruction\n    print(\"3/4: Sparse reconstruction...\")\n    subprocess.run([\n        'colmap', 'mapper',\n        '--database_path', database_path,\n        '--image_path', image_dir,\n        '--output_path', sparse_dir,\n        '--Mapper.ba_global_max_num_iterations', '20',  # Speed up\n        '--Mapper.ba_local_max_num_iterations', '10'\n    ], check=True, env=env)\n    \n    # Export to text format\n    print(\"4/4: Exporting to text format...\")\n    model_dir = os.path.join(sparse_dir, '0')\n    if not os.path.exists(model_dir):\n        # Use the first model found\n        subdirs = [d for d in os.listdir(sparse_dir) if os.path.isdir(os.path.join(sparse_dir, d))]\n        if subdirs:\n            model_dir = os.path.join(sparse_dir, subdirs[0])\n        else:\n            raise FileNotFoundError(\"COLMAP reconstruction failed\")\n    \n    subprocess.run([\n        'colmap', 'model_converter',\n        '--input_path', model_dir,\n        '--output_path', model_dir,\n        '--output_type', 'TXT'\n    ], check=True, env=env)\n    \n    print(f\"COLMAP reconstruction complete: {model_dir}\")\n    return model_dir\n\ndef convert_cameras_to_pinhole(input_file, output_file):\n    \"\"\"Convert camera model to PINHOLE format\"\"\"\n    print(f\"Reading camera file: {input_file}\")\n    \n    with open(input_file, 'r') as f:\n        lines = f.readlines()\n    \n    converted_count = 0\n    with open(output_file, 'w') as f:\n        for line in lines:\n            if line.startswith('#') or line.strip() == '':\n                f.write(line)\n            else:\n                parts = line.strip().split()\n                if len(parts) >= 4:\n                    cam_id = parts[0]\n                    model = parts[1]\n                    width = parts[2]\n                    height = parts[3]\n                    params = parts[4:]\n                    \n                    # Convert to PINHOLE format\n                    if model == \"PINHOLE\":\n                        f.write(line)\n                    elif model == \"OPENCV\":\n                        # OPENCV: fx, fy, cx, cy, k1, k2, p1, p2\n                        fx = params[0]\n                        fy = params[1]\n                        cx = params[2]\n                        cy = params[3]\n                        f.write(f\"{cam_id} PINHOLE {width} {height} {fx} {fy} {cx} {cy}\\n\")\n                        converted_count += 1\n                    else:\n                        # Convert other models too\n                        fx = fy = max(float(width), float(height))\n                        cx = float(width) / 2\n                        cy = float(height) / 2\n                        f.write(f\"{cam_id} PINHOLE {width} {height} {fx} {fy} {cx} {cy}\\n\")\n                        converted_count += 1\n                else:\n                    f.write(line)\n    \n    print(f\"Converted {converted_count} cameras to PINHOLE format\")\n\ndef prepare_gaussian_splatting_data(image_dir, colmap_model_dir):\n    \"\"\"Prepare data for Gaussian Splatting\"\"\"\n    print(\"Preparing data for Gaussian Splatting...\")\n    \n    data_dir = f\"{WORK_DIR}/data/video\"\n    os.makedirs(f\"{data_dir}/sparse/0\", exist_ok=True)\n    os.makedirs(f\"{data_dir}/images\", exist_ok=True)\n    \n    # Copy images\n    print(\"Copying images...\")\n    img_count = 0\n    for img_file in os.listdir(image_dir):\n        if img_file.lower().endswith(('.jpg', '.jpeg', '.png')):\n            shutil.copy(\n                os.path.join(image_dir, img_file),\n                f\"{data_dir}/images/{img_file}\"\n            )\n            img_count += 1\n    print(f\"Copied {img_count} images\")\n    \n    # Convert and copy camera file to PINHOLE format\n    print(\"Converting camera model to PINHOLE format...\")\n    convert_cameras_to_pinhole(\n        os.path.join(colmap_model_dir, 'cameras.txt'),\n        f\"{data_dir}/sparse/0/cameras.txt\"\n    )\n    \n    # Copy other files\n    for filename in ['images.txt', 'points3D.txt']:\n        src = os.path.join(colmap_model_dir, filename)\n        dst = f\"{data_dir}/sparse/0/{filename}\"\n        if os.path.exists(src):\n            shutil.copy(src, dst)\n            print(f\"Copied {filename}\")\n        else:\n            print(f\"Warning: {filename} not found\")\n    \n    print(f\"Data preparation complete: {data_dir}\")\n    return data_dir\n\ndef train_gaussian_splatting(data_dir, iterations=7000):\n    \"\"\"Train the Gaussian Splatting model\"\"\"\n    print(f\"Training Gaussian Splatting model for {iterations} iterations...\")\n    \n    model_path = f\"{WORK_DIR}/output/video\"\n    \n    cmd = [\n        sys.executable, 'train.py',\n        '-s', data_dir,\n        '-m', model_path,\n        '--iterations', str(iterations),\n        '--eval'\n    ]\n    \n    subprocess.run(cmd, cwd=WORK_DIR, check=True)\n    \n    return model_path\n\ndef render_video(model_path, output_video_path, iteration=7000):\n    \"\"\"Generate video from the trained model\"\"\"\n    print(\"Rendering video...\")\n    \n    # Execute rendering\n    cmd = [\n        sys.executable, 'render.py',\n        '-m', model_path,\n        '--iteration', str(iteration)\n    ]\n    \n    subprocess.run(cmd, cwd=WORK_DIR, check=True)\n    \n    # Find the rendering directory\n    possible_dirs = [\n        f\"{model_path}/test/ours_{iteration}/renders\",\n        f\"{model_path}/train/ours_{iteration}/renders\",\n    ]\n    \n    render_dir = None\n    for test_dir in possible_dirs:\n        if os.path.exists(test_dir):\n            render_dir = test_dir\n            print(f\"Rendering directory found: {render_dir}\")\n            break\n    \n    if render_dir and os.path.exists(render_dir):\n        render_imgs = sorted([f for f in os.listdir(render_dir) if f.endswith('.png')])\n        \n        if render_imgs:\n            print(f\"Found {len(render_imgs)} rendered images\")\n            \n            # Create video with ffmpeg\n            subprocess.run([\n                'ffmpeg', '-y',\n                '-framerate', '30',\n                '-pattern_type', 'glob',\n                '-i', f\"{render_dir}/*.png\",\n                '-c:v', 'libx264',\n                '-pix_fmt', 'yuv420p',\n                '-crf', '18',\n                output_video_path\n            ], check=True)\n            \n            print(f\"Video saved: {output_video_path}\")\n            return True\n    \n    print(\"Error: Rendering directory not found\")\n    return False\n\ndef create_gif(video_path, gif_path):\n    \"\"\"Create GIF from MP4\"\"\"\n    print(\"Creating animated GIF...\")\n    \n    subprocess.run([\n        'ffmpeg', '-y',\n        '-i', video_path,\n        '-vf', 'setpts=8*PTS,fps=10,scale=720:-1:flags=lanczos',\n        '-loop', '0',\n        gif_path\n    ], check=True)\n    \n    if os.path.exists(gif_path):\n        size_mb = os.path.getsize(gif_path) / (1024 * 1024)\n        print(f\"GIF creation complete: {gif_path} ({size_mb:.2f} MB)\")\n        return True\n    \n    return False","metadata":{"execution":{"iopub.execute_input":"2025-10-11T05:12:30.379281Z","iopub.status.busy":"2025-10-11T05:12:30.379075Z","iopub.status.idle":"2025-10-11T05:12:30.397485Z","shell.execute_reply":"2025-10-11T05:12:30.396748Z"},"papermill":{"duration":0.022468,"end_time":"2025-10-11T05:12:30.398642","exception":false,"start_time":"2025-10-11T05:12:30.376174","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def main():\n    \"\"\"Main execution function\"\"\"\n    print(\"=\"*60)\n    print(\"Gaussian Splatting Generation from MP4 Video\")\n    print(\"=\"*60)\n    \n    try:\n        # Step 1: Environment Setup\n        setup_environment()\n        \n        # Step 2: Extract Frames from Video\n        frame_dir = f\"{COLMAP_DIR}/images\"\n\n        process_frames_from_folder(IMAGE_PATH, frame_dir, max_frames=300)###########\n        #extract_frames(MOVIE_PATH, frame_dir, fps=2, max_frames=100)###\n        \n        # Step 3: Estimate Camera Info with COLMAP\n        colmap_model_dir = run_colmap_reconstruction(frame_dir, COLMAP_DIR)\n        \n        # Step 4: Prepare Data for Gaussian Splatting\n        data_dir = prepare_gaussian_splatting_data(frame_dir, colmap_model_dir)\n        \n        # Step 5: Train Model\n        model_path = train_gaussian_splatting(data_dir, iterations=3000)\n        \n        # Step 6: Render Video\n        os.makedirs(OUTPUT_DIR, exist_ok=True)\n        output_video = f\"{OUTPUT_DIR}/gaussian_splatting_video.mp4\"\n        success = render_video(model_path, output_video, iteration=3000)\n        \n        if success:\n            print(\"=\"*60)\n            print(f\"Success! Video generation complete: {output_video}\")\n            print(\"=\"*60)\n            \n            # Create GIF\n            output_gif = f\"{OUTPUT_DIR}/gaussian_splatting_video.gif\"\n            create_gif(output_video, output_gif)\n            \n            # Display result\n            from IPython.display import Image\n            display(Image(open(output_gif, 'rb').read()))\n            \n        else:\n            print(\"Warning: Rendering complete, but video was not generated\")\n        \n    except Exception as e:\n        print(f\"Error: {str(e)}\")\n        import traceback\n        traceback.print_exc()\n\nif __name__ == \"__main__\":\n    main()","metadata":{"execution":{"iopub.execute_input":"2025-10-11T05:12:30.403705Z","iopub.status.busy":"2025-10-11T05:12:30.403268Z","iopub.status.idle":"2025-10-11T06:46:05.998179Z","shell.execute_reply":"2025-10-11T06:46:05.997418Z"},"papermill":{"duration":5615.773555,"end_time":"2025-10-11T06:46:06.174252","exception":false,"start_time":"2025-10-11T05:12:30.400697","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2025-10-11T06:46:06.729402Z","iopub.status.busy":"2025-10-11T06:46:06.728697Z","iopub.status.idle":"2025-10-11T06:46:06.770281Z","shell.execute_reply":"2025-10-11T06:46:06.769578Z"},"papermill":{"duration":0.326095,"end_time":"2025-10-11T06:46:06.771668","exception":false,"start_time":"2025-10-11T06:46:06.445573","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"    gif_path='/kaggle/working/output/gaussian_splatting_video.gif'\n    from IPython.display import Image\n    Image(open(gif_path, 'rb').read())","metadata":{"execution":{"iopub.execute_input":"2025-09-29T08:34:30.602727Z","iopub.status.busy":"2025-09-29T08:34:30.602485Z","iopub.status.idle":"2025-09-29T08:34:30.774166Z","shell.execute_reply":"2025-09-29T08:34:30.772912Z"},"papermill":{"duration":0.239685,"end_time":"2025-10-11T06:46:07.247711","exception":false,"start_time":"2025-10-11T06:46:07.008026","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### **The point_cloud.ply file is converted into splat file here.**\n\nhttps://www.kaggle.com/code/stpeteishii/ply-to-splat-converter\n\n### **You can see the result in 3D gaussian splat viewer**\n\nhttps://splat-three.vercel.app/?url=fountain_photo.splat","metadata":{"papermill":{"duration":0.23882,"end_time":"2025-10-11T06:46:07.731101","exception":false,"start_time":"2025-10-11T06:46:07.492281","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"","metadata":{"papermill":{"duration":0.242833,"end_time":"2025-10-11T06:46:08.222088","exception":false,"start_time":"2025-10-11T06:46:07.979255","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"","metadata":{"papermill":{"duration":0.232336,"end_time":"2025-10-11T06:46:08.687514","exception":false,"start_time":"2025-10-11T06:46:08.455178","status":"completed"},"tags":[]}}]}