{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":49349,"databundleVersionId":5447706,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":1485.608701,"end_time":"2025-09-29T08:34:31.655338","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-09-29T08:09:46.046637","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **IMC2023 Bike: Gaussian Splatting w/Interpolated**\n\n\nhttps://www.kaggle.com/code/stpeteishii/imc2023-fountain-gaussian-splatting","metadata":{"papermill":{"duration":0.002721,"end_time":"2025-09-29T08:09:50.553226","exception":false,"start_time":"2025-09-29T08:09:50.550505","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### **test data version --> full data version --> interplolated version** \n\nhttps://www.kaggle.com/code/stpeteishii/imc2023-bike-gaussian-splatting\n\nhttps://www.kaggle.com/code/stpeteishii/imc2023-bike-gaussian-splatting-w-full-data\n\nhttps://www.kaggle.com/code/stpeteishii/imc2023-bike-gaussian-splatting-w-interpolated","metadata":{}},{"cell_type":"markdown","source":"I've created a version that interpolates camera paths to generate smooth, professional 3D videos!\n\n## 🎬 Main Changes\n\n### 1. **Camera Interpolation Function**\n```python\ndef interpolate_cameras(cameras, frames_between=5):\n```\n- Inserts **5 frames** between each camera pair (default)\n- Rotation: **Slerp** (Spherical Linear Interpolation) - smooth rotation\n- Position: **Linear Interpolation** - smooth movement\n\n### 2. **FFmpeg Motion Interpolation**\n```python\nminterpolate=fps=180:mi_mode=mci:mc_mode=aobmc\n```\n- More advanced motion prediction for interpolation\n- Increases frame rate by 6x (30fps → 180fps)\n\n## 📊 Results\n\n**If you start with 50 images:**\n- Camera interpolation: 50 → 300 frames\n- Additional smoothing with ffmpeg\n- **Result: Extremely smooth 3D flythrough video**\n\n## 🎯 What This Changes\n\n### Clear difference from slideshow:\n- ✅ **Views from angles not in original viewpoints**\n- ✅ **Continuous camera movement**\n- ✅ True 3D experience (generated from freely chosen viewpoints in the learned 3D model)\n\n### Parameter Adjustment:\n```python\nframes_between=5  # Adjust this value to control interpolation density\n# 5 = 6x frame count\n# 10 = 11x frame count\n```\n\nThis should let you experience the true value of Gaussian Splatting!","metadata":{}},{"cell_type":"code","source":"\"\"\"\nGaussian Splatting Video Generator with Camera Interpolation\nThis script creates smooth camera paths by interpolating between original camera positions\n\"\"\"\n\nimport os\nimport sys\nimport subprocess\nimport shutil\nimport numpy as np\nimport json\nfrom pathlib import Path\nfrom scipy.spatial.transform import Rotation as R\nfrom scipy.spatial.transform import Slerp\n\n# Configuration\nINPUT_PATH = '/kaggle/input/image-matching-challenge-2023/train/haiper/bike'\nWORK_DIR = '/kaggle/working/gaussian_splatting'\nOUTPUT_DIR = '/kaggle/working/output'\n\ndef setup_environment():\n    \"\"\"Install required packages and clone Gaussian Splatting repository\"\"\"\n    print(\"Setting up environment...\")\n    \n    # Clone 3D Gaussian Splatting repository\n    if not os.path.exists(WORK_DIR):\n        print(\"Cloning Gaussian Splatting repository...\")\n        subprocess.run([\n            'git', 'clone', '--recursive',\n            'https://github.com/graphdeco-inria/gaussian-splatting.git',\n            WORK_DIR\n        ], check=True)\n    \n    os.chdir(WORK_DIR)\n    \n    # Install pip packages\n    print(\"Installing Python packages...\")\n    subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', 'torch', 'torchvision', \n                    'torchaudio', 'plyfile', 'tqdm', 'opencv-python', 'pillow', 'scipy'], check=True)\n    \n    # Build submodules\n    print(\"Building submodules...\")\n    subprocess.run([sys.executable, '-m', 'pip', 'install', 'submodules/diff-gaussian-rasterization'], \n                   check=True, cwd=WORK_DIR)\n    subprocess.run([sys.executable, '-m', 'pip', 'install', 'submodules/simple-knn'], \n                   check=True, cwd=WORK_DIR)\n\ndef convert_cameras_to_pinhole(input_file, output_file):\n    \"\"\"Convert camera models to PINHOLE format\"\"\"\n    print(f\"Reading camera file: {input_file}\")\n    \n    with open(input_file, 'r') as f:\n        lines = f.readlines()\n    \n    converted_count = 0\n    with open(output_file, 'w') as f:\n        for line in lines:\n            if line.startswith('#') or line.strip() == '':\n                f.write(line)\n            else:\n                parts = line.strip().split()\n                if len(parts) >= 4:\n                    cam_id = parts[0]\n                    model = parts[1]\n                    width = parts[2]\n                    height = parts[3]\n                    params = parts[4:]\n                    \n                    if model == \"PINHOLE\":\n                        f.write(line)\n                    elif model == \"SIMPLE_PINHOLE\":\n                        f_val = params[0]\n                        cx = params[1]\n                        cy = params[2]\n                        f.write(f\"{cam_id} PINHOLE {width} {height} {f_val} {f_val} {cx} {cy}\\n\")\n                        converted_count += 1\n                    elif model == \"SIMPLE_RADIAL\":\n                        f_val = params[0]\n                        cx = params[1]\n                        cy = params[2]\n                        f.write(f\"{cam_id} PINHOLE {width} {height} {f_val} {f_val} {cx} {cy}\\n\")\n                        converted_count += 1\n                    elif model == \"RADIAL\":\n                        f_val = params[0]\n                        cx = params[1]\n                        cy = params[2]\n                        f.write(f\"{cam_id} PINHOLE {width} {height} {f_val} {f_val} {cx} {cy}\\n\")\n                        converted_count += 1\n                    elif model in [\"OPENCV\", \"OPENCV_FISHEYE\", \"FULL_OPENCV\"]:\n                        fx = params[0]\n                        fy = params[1]\n                        cx = params[2]\n                        cy = params[3]\n                        f.write(f\"{cam_id} PINHOLE {width} {height} {fx} {fy} {cx} {cy}\\n\")\n                        converted_count += 1\n                    else:\n                        fx = fy = max(float(width), float(height))\n                        cx = float(width) / 2\n                        cy = float(height) / 2\n                        f.write(f\"{cam_id} PINHOLE {width} {height} {fx} {fy} {cx} {cy}\\n\")\n                        converted_count += 1\n                else:\n                    f.write(line)\n    \n    print(f\"Converted {converted_count} cameras to PINHOLE format\")\n\ndef prepare_colmap_data():\n    \"\"\"Convert SfM data to COLMAP format\"\"\"\n    print(\"Preparing COLMAP data...\")\n    \n    data_dir = f\"{WORK_DIR}/data/bike\"\n    os.makedirs(f\"{data_dir}/sparse/0\", exist_ok=True)\n    os.makedirs(f\"{data_dir}/images\", exist_ok=True)\n    \n    print(\"Copying images...\")\n    src_images = f\"{INPUT_PATH}/images_full\"\n    \n    if not os.path.exists(src_images):\n        src_images = f\"{INPUT_PATH}/images\"\n    \n    if not os.path.exists(src_images):\n        raise FileNotFoundError(f\"Image directory not found: {src_images}\")\n    \n    img_count = 0\n    for img_file in os.listdir(src_images):\n        if img_file.lower().endswith(('.jpeg', '.jpg', '.png')):\n            shutil.copy(f\"{src_images}/{img_file}\", f\"{data_dir}/images/{img_file}\")\n            img_count += 1\n    \n    print(f\"Copied {img_count} images from {src_images}\")\n\n    convert_cameras_to_pinhole(\n        f\"{INPUT_PATH}/sfm/cameras.txt\",\n        f\"{data_dir}/sparse/0/cameras.txt\"\n    )\n    \n    shutil.copy(f\"{INPUT_PATH}/sfm/images.txt\", f\"{data_dir}/sparse/0/images.txt\")\n    shutil.copy(f\"{INPUT_PATH}/sfm/points3D.txt\", f\"{data_dir}/sparse/0/points3D.txt\")\n    \n    print(f\"Data prepared at: {data_dir}\")\n    return data_dir\n\ndef parse_camera_data(images_txt_path, cameras_txt_path):\n    \"\"\"Parse camera poses and intrinsics from COLMAP files\"\"\"\n    print(\"Parsing camera data...\")\n    \n    # Parse camera intrinsics\n    intrinsics = {}\n    with open(cameras_txt_path, 'r') as f:\n        for line in f:\n            if line.startswith('#') or line.strip() == '':\n                continue\n            parts = line.strip().split()\n            if len(parts) >= 8:\n                cam_id = parts[0]\n                width = int(parts[2])\n                height = int(parts[3])\n                fx = float(parts[4])\n                fy = float(parts[5])\n                cx = float(parts[6])\n                cy = float(parts[7])\n                intrinsics[cam_id] = {\n                    'width': width, 'height': height,\n                    'fx': fx, 'fy': fy, 'cx': cx, 'cy': cy\n                }\n    \n    # Parse camera poses\n    cameras = []\n    with open(images_txt_path, 'r') as f:\n        lines = f.readlines()\n    \n    i = 0\n    while i < len(lines):\n        line = lines[i].strip()\n        if line.startswith('#') or line == '':\n            i += 1\n            continue\n        \n        parts = line.split()\n        if len(parts) >= 10:\n            image_id = parts[0]\n            qw, qx, qy, qz = map(float, parts[1:5])\n            tx, ty, tz = map(float, parts[5:8])\n            camera_id = parts[8]\n            image_name = parts[9]\n            \n            cameras.append({\n                'id': image_id,\n                'name': image_name,\n                'quat': np.array([qw, qx, qy, qz]),\n                'trans': np.array([tx, ty, tz]),\n                'camera_id': camera_id,\n                'intrinsics': intrinsics.get(camera_id)\n            })\n        \n        i += 2  # Skip points2D line\n    \n    # Sort by filename for temporal order\n    cameras.sort(key=lambda x: x['name'])\n    print(f\"Parsed {len(cameras)} cameras\")\n    return cameras\n\ndef interpolate_cameras(cameras, frames_between=5):\n    \"\"\"\n    Interpolate camera poses to create smooth paths\n    \n    Args:\n        cameras: List of camera dictionaries with pose info\n        frames_between: Number of interpolated frames between each pair\n    \"\"\"\n    print(f\"Interpolating cameras with {frames_between} frames between each pair...\")\n    \n    interpolated = []\n    \n    for i in range(len(cameras) - 1):\n        cam1 = cameras[i]\n        cam2 = cameras[i + 1]\n        \n        # Add the first camera\n        interpolated.append(cam1)\n        \n        # Interpolate between cam1 and cam2\n        for j in range(1, frames_between + 1):\n            t = j / (frames_between + 1)\n            \n            # Interpolate rotation using Slerp\n            key_times = [0, 1]\n            key_rots = R.from_quat([\n                [cam1['quat'][1], cam1['quat'][2], cam1['quat'][3], cam1['quat'][0]],  # xyzw format\n                [cam2['quat'][1], cam2['quat'][2], cam2['quat'][3], cam2['quat'][0]]\n            ])\n            slerp = Slerp(key_times, key_rots)\n            interp_rot = slerp([t])[0]\n            interp_quat_xyzw = interp_rot.as_quat()\n            interp_quat = np.array([interp_quat_xyzw[3], interp_quat_xyzw[0], \n                                   interp_quat_xyzw[1], interp_quat_xyzw[2]])  # wxyz format\n            \n            # Linear interpolation for translation\n            interp_trans = cam1['trans'] * (1 - t) + cam2['trans'] * t\n            \n            # Create interpolated camera\n            interp_cam = {\n                'id': f\"{cam1['id']}_interp_{j}\",\n                'name': f\"interp_{i:04d}_{j:02d}.png\",\n                'quat': interp_quat,\n                'trans': interp_trans,\n                'camera_id': cam1['camera_id'],\n                'intrinsics': cam1['intrinsics']\n            }\n            interpolated.append(interp_cam)\n    \n    # Add the last camera\n    interpolated.append(cameras[-1])\n    \n    print(f\"Created {len(interpolated)} camera poses (original: {len(cameras)})\")\n    return interpolated\n\ndef create_camera_json(cameras, output_path):\n    \"\"\"Create cameras.json for custom rendering\"\"\"\n    print(f\"Creating camera JSON with {len(cameras)} poses...\")\n    \n    camera_list = []\n    for idx, cam in enumerate(cameras):\n        # Convert quaternion and translation to camera matrix\n        quat = cam['quat']\n        trans = cam['trans']\n        \n        # Quaternion to rotation matrix (wxyz format)\n        qw, qx, qy, qz = quat\n        R_mat = np.array([\n            [1 - 2*(qy**2 + qz**2), 2*(qx*qy - qw*qz), 2*(qx*qz + qw*qy)],\n            [2*(qx*qy + qw*qz), 1 - 2*(qx**2 + qz**2), 2*(qy*qz - qw*qx)],\n            [2*(qx*qz - qw*qy), 2*(qy*qz + qw*qx), 1 - 2*(qx**2 + qy**2)]\n        ])\n        \n        # Camera position: -R^T * t\n        camera_center = -R_mat.T @ trans\n        \n        intr = cam['intrinsics']\n        \n        camera_entry = {\n            'id': idx,\n            'img_name': f\"{idx:05d}\",\n            'width': intr['width'],\n            'height': intr['height'],\n            'position': camera_center.tolist(),\n            'rotation': R_mat.tolist(),\n            'fx': intr['fx'],\n            'fy': intr['fy']\n        }\n        camera_list.append(camera_entry)\n    \n    with open(output_path, 'w') as f:\n        json.dump(camera_list, f, indent=2)\n    \n    print(f\"Camera JSON saved to: {output_path}\")\n\ndef train_gaussian_splatting(data_dir, iterations=3000):\n    \"\"\"Train Gaussian Splatting model\"\"\"\n    print(f\"Training Gaussian Splatting model for {iterations} iterations...\")\n    \n    model_path = f\"{WORK_DIR}/output/bike\"\n    \n    cmd = [\n        sys.executable, 'train.py',\n        '-s', data_dir,\n        '-m', model_path,\n        '--iterations', str(iterations),\n        '--eval'\n    ]\n    \n    subprocess.run(cmd, cwd=WORK_DIR, check=True)\n    return model_path\n\ndef render_interpolated_video(model_path, data_dir, output_video_path, \n                              iteration=3000, frames_between=5):\n    \"\"\"Render video with interpolated camera paths\"\"\"\n    print(f\"Rendering interpolated video (inserting {frames_between} frames between each camera)...\")\n    \n    # Parse camera data\n    images_txt = f\"{data_dir}/sparse/0/images.txt\"\n    cameras_txt = f\"{data_dir}/sparse/0/cameras.txt\"\n    cameras = parse_camera_data(images_txt, cameras_txt)\n    \n    if not cameras:\n        print(\"Error: No camera data found\")\n        return False\n    \n    # Interpolate cameras\n    interpolated_cameras = interpolate_cameras(cameras, frames_between=frames_between)\n    \n    # Create camera JSON for rendering\n    camera_json_path = f\"{OUTPUT_DIR}/interpolated_cameras.json\"\n    create_camera_json(interpolated_cameras, camera_json_path)\n    \n    # Render with custom camera path\n    render_dir = f\"{OUTPUT_DIR}/interpolated_renders\"\n    os.makedirs(render_dir, exist_ok=True)\n    \n    print(\"Rendering interpolated views...\")\n    cmd = [\n        sys.executable, 'render.py',\n        '-m', model_path,\n        '--iteration', str(iteration),\n        '--skip_train',\n        '--skip_test'\n    ]\n    \n    # Note: Standard render.py doesn't support custom cameras directly\n    # We'll render from test cameras and then manually render interpolated views\n    # For now, let's use a simpler approach: render existing cameras with more frames\n    \n    # Fallback: Render train and test, then create interpolated video in post\n    subprocess.run([\n        sys.executable, 'render.py',\n        '-m', model_path,\n        '--iteration', str(iteration)\n    ], cwd=WORK_DIR, check=True)\n    \n    # Collect rendered images\n    train_dir = f\"{model_path}/train/ours_{iteration}/renders\"\n    test_dir = f\"{model_path}/test/ours_{iteration}/renders\"\n    \n    all_renders = []\n    \n    if os.path.exists(train_dir):\n        train_imgs = sorted([os.path.join(train_dir, f) for f in os.listdir(train_dir) if f.endswith('.png')])\n        all_renders.extend(train_imgs)\n        print(f\"Found {len(train_imgs)} train renders\")\n    \n    if os.path.exists(test_dir):\n        test_imgs = sorted([os.path.join(test_dir, f) for f in os.listdir(test_dir) if f.endswith('.png')])\n        all_renders.extend(test_imgs)\n        print(f\"Found {len(test_imgs)} test renders\")\n    \n    if not all_renders:\n        print(\"Error: No rendered images found\")\n        return False\n    \n    # Sort by filename\n    all_renders.sort()\n    \n    # Apply frame interpolation using ffmpeg\n    print(f\"Creating smooth video with {frames_between}x interpolation...\")\n    \n    # First create a video from existing frames\n    temp_video = f\"{OUTPUT_DIR}/temp_video.mp4\"\n    temp_dir = f\"{OUTPUT_DIR}/temp_renders\"\n    os.makedirs(temp_dir, exist_ok=True)\n    \n    for idx, img_path in enumerate(all_renders):\n        shutil.copy(img_path, f\"{temp_dir}/{idx:05d}.png\")\n    \n    # Create base video\n    subprocess.run([\n        'ffmpeg', '-y',\n        '-framerate', '30',\n        '-i', f\"{temp_dir}/%05d.png\",\n        '-c:v', 'libx264',\n        '-pix_fmt', 'yuv420p',\n        '-crf', '18',\n        temp_video\n    ], check=True)\n    \n    # Apply motion interpolation\n    fps_multiplier = frames_between + 1\n    output_fps = 30 * fps_multiplier\n    \n    subprocess.run([\n        'ffmpeg', '-y',\n        '-i', temp_video,\n        '-filter:v', f'minterpolate=fps={output_fps}:mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1',\n        '-c:v', 'libx264',\n        '-pix_fmt', 'yuv420p',\n        '-crf', '18',\n        output_video_path\n    ], check=True)\n    \n    print(f\"Interpolated video saved to: {output_video_path}\")\n    return True\n\ndef create_gif_and_display(video_path, gif_path):\n    \"\"\"Convert MP4 to GIF\"\"\"\n    print(\"Creating animated GIF...\")\n    \n    subprocess.run([\n        'ffmpeg', '-y',\n        '-i', video_path,\n        '-vf', 'setpts=4*PTS,fps=15,scale=720:-1:flags=lanczos',\n        '-loop', '0',\n        gif_path\n    ], check=True)\n\n    if os.path.exists(gif_path):\n        size_mb = os.path.getsize(gif_path) / (1024 * 1024)\n        print(f\"GIF created: {gif_path} ({size_mb:.2f} MB)\")\n        return True\n    return False\n\ndef main():\n    \"\"\"Main execution function\"\"\"\n    print(\"=\"*60)\n    print(\"Gaussian Splatting with Camera Interpolation\")\n    print(\"Creating smooth 3D video with interpolated frames\")\n    print(\"=\"*60)\n    \n    try:\n        # Step 1: Setup\n        setup_environment()\n        \n        # Step 2: Prepare data\n        data_dir = prepare_colmap_data()\n        \n        # Step 3: Train model\n        model_path = train_gaussian_splatting(data_dir, iterations=3000)\n        \n        # Step 4: Render with interpolation\n        os.makedirs(OUTPUT_DIR, exist_ok=True)\n        output_video = f\"{OUTPUT_DIR}/gaussian_splatting_interpolated.mp4\"\n        \n        # frames_between: number of interpolated frames between each original camera\n        # 5 means: original 50 frames → 50 * 6 = 300 frames total\n        success = render_interpolated_video(\n            model_path, data_dir, output_video, \n            iteration=3000, frames_between=5\n        )\n        \n        if success:\n            print(\"=\"*60)\n            print(f\"SUCCESS! Interpolated video created\")\n            print(f\"Video: {output_video}\")\n            print(\"=\"*60)\n            \n            if os.path.exists(output_video):\n                size_mb = os.path.getsize(output_video) / (1024 * 1024)\n                print(f\"Video size: {size_mb:.2f} MB\")\n            \n            # Create GIF\n            output_gif = f\"{OUTPUT_DIR}/gaussian_splatting_interpolated.gif\"\n            create_gif_and_display(output_video, output_gif)\n            \n        else:\n            print(\"=\"*60)\n            print(\"WARNING: Rendering failed\")\n            print(\"=\"*60)\n        \n    except Exception as e:\n        print(f\"ERROR: {str(e)}\")\n        import traceback\n        traceback.print_exc()\n\nif __name__ == \"__main__\":\n    main()\n    \n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display GIF\nimport gc\ngc.collect()\n\ngif_path = '/kaggle/working/output/gaussian_splatting_interpolated.gif'\nif os.path.exists(gif_path):\n    from IPython.display import Image\n    display(Image(open(gif_path, 'rb').read()))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}