{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":15768,"databundleVersionId":700263}],"dockerImageVersionId":29271,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Phase 1 (Data Exploration and EDA)","metadata":{}},{"cell_type":"markdown","source":"![gif](https://raw.githubusercontent.com/lyft/nuscenes-devkit/master/notebooks/media/001.gif)","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\n!pip install lyft-dataset-sdk > /dev/null 2>&1\nprint(\"βœ… Lyft SDK Installed Successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:11:19.769420Z","iopub.execute_input":"2026-04-06T18:11:19.769711Z","iopub.status.idle":"2026-04-06T18:11:54.858153Z","shell.execute_reply.started":"2026-04-06T18:11:19.769645Z","shell.execute_reply":"2026-04-06T18:11:54.857244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# 1. Define where the data is, and where we want our \"fake\" standard folders to be\nKAGGLE_DIR = '/kaggle/input/3d-object-detection-for-autonomous-vehicles/'\nWORK_DIR = '/kaggle/working/lyft_data/'\n\n# Create the working directory\nos.makedirs(WORK_DIR, exist_ok=True)\n\n# 2. Map Kaggle's folder names to the SDK's expected folder names\nfolder_mapping = {\n 'train_images': 'images',\n 'train_maps': 'maps',\n 'train_lidar': 'lidar',\n 'train_data': 'train_data'\n}\n\n# 3. Create the symlinks (shortcuts)\nfor kaggle_folder, sdk_folder in folder_mapping.items():\n src = os.path.join(KAGGLE_DIR, kaggle_folder)\n dst = os.path.join(WORK_DIR, sdk_folder)\n \n # Create shortcut if it doesn't already exist\n if not os.path.exists(dst):\n os.symlink(src, dst)\n \nprint(\"βœ… Symlinks created successfully! The SDK will now be happy.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:11:54.860249Z","iopub.execute_input":"2026-04-06T18:11:54.860469Z","iopub.status.idle":"2026-04-06T18:11:54.868162Z","shell.execute_reply.started":"2026-04-06T18:11:54.860430Z","shell.execute_reply":"2026-04-06T18:11:54.867406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lyft_dataset_sdk\nfrom lyft_dataset_sdk.lyftdataset import LyftDataset\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nlyft = LyftDataset(data_path=WORK_DIR, json_path=os.path.join(WORK_DIR, 'train_data'), verbose=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:11:54.870633Z","iopub.execute_input":"2026-04-06T18:11:54.870944Z","iopub.status.idle":"2026-04-06T18:12:13.293243Z","shell.execute_reply.started":"2026-04-06T18:11:54.870890Z","shell.execute_reply":"2026-04-06T18:12:13.292509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 0.5: WHAT'S INSIDE THE JSON FILES?\n# Shows the actual fields in each SDK table so the reader\n# understands what raw data we're working with\n# ============================================================\n\ntables_to_show = {\n 'sample' : lyft.sample[0],\n 'sample_annotation': lyft.sample_annotation[0],\n 'sample_data' : lyft.sample_data[0],\n 'scene' : lyft.scene[0],\n 'instance' : lyft.instance[0],\n 'category' : lyft.category[0],\n 'ego_pose' : lyft.ego_pose[0],\n 'calibrated_sensor': lyft.calibrated_sensor[0],\n 'sensor' : lyft.sensor[0],\n 'log' : lyft.log[0],\n}\n\nprint(\"=\" * 60)\nprint(\"RAW JSON SCHEMA β€” FIELDS IN EACH TABLE\")\nprint(\"=\" * 60)\n\nfor table_name, record in tables_to_show.items():\n print(f\"\\nπŸ“„ {table_name.upper()}\")\n print(\"-\" * 40)\n rows = []\n for key, val in record.items():\n # Truncate long values for readability\n if isinstance(val, list) and len(val) > 3:\n display_val = f\"[list of {len(val)} items]\"\n elif isinstance(val, str) and len(val) > 50:\n display_val = val[:47] + \"...\"\n else:\n display_val = val\n rows.append({'field': key, 'example_value': display_val})\n display(pd.DataFrame(rows))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:13.294676Z","iopub.execute_input":"2026-04-06T18:12:13.294898Z","iopub.status.idle":"2026-04-06T18:12:13.371017Z","shell.execute_reply.started":"2026-04-06T18:12:13.294859Z","shell.execute_reply":"2026-04-06T18:12:13.370058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\"\"The highest level of our data architecture is a Scene. \nThe self-driving car doesn't just take random pictures; it records 25-second continuous driving logs. \nThe scene.json table keeps track of these mini-videos.\n\"\"\")\n\n# Grab the very first scene in the dataset\nmy_scene = lyft.scene[0]\n\nprint(\"=== RAW SCENE JSON ===\")\nprint(f\"Token (ID): {my_scene['token']}\")\nprint(f\"Name: {my_scene['name']}\")\nprint(f\"Number of Samples (Frames): {my_scene['nbr_samples']}\")\nprint(f\"First Frame ID: {my_scene['first_sample_token']}\")\nprint(f\"Last Frame ID: {my_scene['last_sample_token']}\")\n\nprint(\"\\nWe learn that the data is sequential. nbr_samples is usually 126. This means 126 frames make up one continuous scene.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:13.372704Z","iopub.execute_input":"2026-04-06T18:12:13.373027Z","iopub.status.idle":"2026-04-06T18:12:13.379905Z","shell.execute_reply.started":"2026-04-06T18:12:13.372970Z","shell.execute_reply":"2026-04-06T18:12:13.379148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\"\"\nInside a Scene, we have Samples. \nA Sample is a single snapshot in time (one frame of the video). \nBecause this is a continuous video, the dataset uses a Doubly Linked List architecture. \nEach sample knows exactly what frame came before it (prev) and what frame comes after it (next).\n\"\"\")\n# Let's look at the first frame of our scene\nmy_sample = lyft.get('sample', my_scene['first_sample_token'])\n\nprint(\"=== RAW SAMPLE JSON ===\")\nprint(f\"Token (ID): {my_sample['token']}\")\nprint(f\"Timestamp: {my_sample['timestamp']} (Microseconds since 1970)\")\nprint(f\"Previous Frame: {my_sample['prev']}\") # Will be empty if it's the first frame!\nprint(f\"Next Frame: {my_sample['next']}\")\nprint(f\"\\nSensors fired at this exact millisecond: \\n{list(my_sample['data'].keys())}\")\nprint(f\"\\nNumber of bounding boxes in this frame: {len(my_sample['anns'])}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:13.381875Z","iopub.execute_input":"2026-04-06T18:12:13.382202Z","iopub.status.idle":"2026-04-06T18:12:13.393166Z","shell.execute_reply.started":"2026-04-06T18:12:13.382146Z","shell.execute_reply":"2026-04-06T18:12:13.392063Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\"\"\nA Sample represents a moment in time, but it doesn't hold the actual images. \nInstead, the sample.json points us to the sample_data.json table. \nThis table contains the actual file paths to the .jpeg images and .bin LiDAR arrays saved on the hard drive.\n\"\"\")\n# Let's see the physical file for the Front Camera at this exact moment\nfront_cam_token = my_sample['data']['CAM_FRONT']\ncam_data = lyft.get('sample_data', front_cam_token)\n\nprint(\"=== RAW SAMPLE_DATA JSON (Front Camera) ===\")\nprint(f\"Token (ID): {cam_data['token']}\")\nprint(f\"File Format: {cam_data['fileformat']}\")\nprint(f\"Width x Height: {cam_data['width']} x {cam_data['height']} pixels\")\nprint(f\"Is this a Key Frame?: {cam_data['is_key_frame']}\")\nprint(f\"Actual Filepath on disk: {cam_data['filename']}\")\n\nprint(\"\\nWe now understand how the JSON database links to the actual 80GB of raw image files sitting in the Kaggle directory.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:13.394887Z","iopub.execute_input":"2026-04-06T18:12:13.395266Z","iopub.status.idle":"2026-04-06T18:12:13.403173Z","shell.execute_reply.started":"2026-04-06T18:12:13.395084Z","shell.execute_reply":"2026-04-06T18:12:13.402201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\"\"\nwe need to know where the cars and pedestrians are. \nThe anns list in the sample.json points us to the sample_annotation.json table. \nThis contains the exact 3D coordinates, sizes, and rotation of every object the model needs to learn to detect.\n\"\"\")\n\n# Let's look at the very first bounding box inside our sample\nfirst_box_token = my_sample['anns'][0]\nannotation = lyft.get('sample_annotation', first_box_token)\n\nprint(\"=== RAW ANNOTATION JSON ===\")\nprint(f\"Token (ID): {annotation['token']}\")\nprint(f\"Instance Token (Object Tracking ID): {annotation['instance_token']}\")\nprint(f\"Translation (X, Y, Z coordinates): {annotation['translation']}\")\nprint(f\"Size (Width, Length, Height): {annotation['size']}\")\nprint(f\"Rotation (Quaternion w,x,y,z): {annotation['rotation']}\")\nprint(f\"LiDAR points hitting this object: {annotation['num_lidar_pts']}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:13.404731Z","iopub.execute_input":"2026-04-06T18:12:13.405222Z","iopub.status.idle":"2026-04-06T18:12:13.418465Z","shell.execute_reply.started":"2026-04-06T18:12:13.405042Z","shell.execute_reply":"2026-04-06T18:12:13.417581Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\"\"\n1. instance.json (The Physical Object)\nAn 'Instance' is a specific, physical object in the real world (e.g., 'Bob's blue Honda'). Even if Bob's car appears in 100 different video frames, it only has one Instance ID. This is how the car tracks objects over time.\n\n2. category.json (The Class Label)\nThis is a simple dictionary of the 9 object classes the AI needs to learn (e.g., car, pedestrian, animal, bicycle).\n\n3. attribute.json (The Object's State)\nWhat is the object doing? This table lists states like object_action_parked, object_action_running, or object_action_standing.\n\n4. visibility.json (The Occlusion Level)\nHow much of the object can the camera actually see? This table defines the 4 levels of visibility (e.g., 0-40% visible, 80-100% visible).\n\"\"\")\n\nprint(\"=== RAW INSTANCE JSON ===\")\nmy_instance = lyft.instance[0]\nprint(f\"Token (ID): {my_instance['token']}\")\nprint(f\"Category: {lyft.get('category', my_instance['category_token'])['name']}\")\nprint(f\"Number of frames this exact car appears in: {my_instance['nbr_annotations']}\")\n\nprint(\"\\n=== RAW CATEGORY JSON ===\")\nprint(f\"Categories to detect: {[cat['name'] for cat in lyft.category]}\")\n\nprint(\"\\n=== RAW ATTRIBUTE JSON ===\")\nprint(f\"Example state: {lyft.attribute[0]['name']}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:13.420215Z","iopub.execute_input":"2026-04-06T18:12:13.420533Z","iopub.status.idle":"2026-04-06T18:12:13.428168Z","shell.execute_reply.started":"2026-04-06T18:12:13.420475Z","shell.execute_reply":"2026-04-06T18:12:13.427207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\"\"\n5. sensor.json (The Hardware List)\n\"A simple list of the 9 sensors physically bolted to the Lyft car (6 cameras, 3 LiDARs).\"\n\n6. calibrated_sensor.json (The Camera Math)\n\"This is the most mathematically important table. Every camera is mounted at a slightly different angle. This table contains the translation (where the camera is relative to the center of the car) and the camera_intrinsic matrix (how the lens curves and distorts the image).\"\n\n7. ego_pose.json (The Self-Driving Car's Location)\n\"As the Lyft car drives, it uses GPS and sensors to track its own location. The 'Ego Pose' records the exact X, Y, Z coordinates of our self-driving car on the global map at every millisecond.\"\n\n8. log.json (The Drive Details)\n\"Which specific Lyft vehicle drove this route, and on what date? This table holds the metadata (e.g., 'Vehicle a101 in Palo Alto on 2019-05-14').\"\n\n9. map.json (The HD Map)\n\"Self-driving cars don't just use Google Maps; they use High-Definition (HD) raster maps that tell them exactly where lanes, sidewalks, and intersections are. This table points to those map images.\"\n\"\"\")\n\nprint(\"=== RAW SENSOR JSON ===\")\nprint(f\"Sensor Name: {lyft.sensor[0]['channel']} ({lyft.sensor[0]['modality']})\")\n\nprint(\"\\n=== RAW CALIBRATED SENSOR JSON ===\")\nmy_calib = lyft.calibrated_sensor[0]\nprint(f\"Translation (X,Y,Z from car center): {my_calib['translation']}\")\nprint(f\"Rotation (Angle mounted): {my_calib['rotation']}\")\n\nprint(\"\\n=== RAW EGO POSE JSON ===\")\nmy_pose = lyft.ego_pose[0]\nprint(f\"Our Car's Global X,Y,Z Location: {my_pose['translation']}\")\n\nprint(\"\\n=== RAW LOG JSON ===\")\nmy_log = lyft.log[0]\nprint(f\"Date Captured: {my_log['date_captured']}\")\nprint(f\"Location: {my_log['location']} (Vehicle ID: {my_log['vehicle']})\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:13.429874Z","iopub.execute_input":"2026-04-06T18:12:13.430162Z","iopub.status.idle":"2026-04-06T18:12:13.439939Z","shell.execute_reply.started":"2026-04-06T18:12:13.430106Z","shell.execute_reply":"2026-04-06T18:12:13.439095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 0.6: CATEGORIES AND ATTRIBUTES\n# ============================================================\n\nprint(\"πŸ“¦ ALL OBJECT CATEGORIES:\\n\")\ncat_rows = [{'name': c['name'], 'token': c['token'][:12] + '...'} \n for c in lyft.category]\ndisplay(pd.DataFrame(cat_rows))\n\nprint(\"\\nπŸ“‘ SENSORS ON THE VEHICLE:\\n\")\nsensor_rows = []\nfor s in lyft.sensor:\n sensor_rows.append({'channel': s['channel'], 'modality': s['modality']})\ndisplay(pd.DataFrame(sensor_rows))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:13.440971Z","iopub.execute_input":"2026-04-06T18:12:13.441148Z","iopub.status.idle":"2026-04-06T18:12:13.463566Z","shell.execute_reply.started":"2026-04-06T18:12:13.441120Z","shell.execute_reply":"2026-04-06T18:12:13.462688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = []\nfor ann in tqdm(lyft.sample_annotation, desc=\"Parsing annotations\"):\n w, l, h = ann['size']\n num_pts = ann['num_lidar_pts'] if ann['num_lidar_pts'] >= 0 else np.nan\n data.append({\n 'class' : ann['category_name'], # SDK pre-injects this\n 'width' : w,\n 'length' : l,\n 'height' : h,\n 'num_lidar_pts': num_pts,\n 'token' : ann['token'],\n 'sample_token' : ann['sample_token'],\n })\n\ndf_eda = pd.DataFrame(data)\ndf_eda['volume'] = df_eda['width'] * df_eda['length'] * df_eda['height']\n\n# Drop token cols for cleaner display β€” they're still in df_eda if needed later\nprint(f\"Built df_eda: {len(df_eda):,} rows\")\nprint(f\"num_lidar_pts available: {df_eda['num_lidar_pts'].notna().sum():,} / {len(df_eda):,}\")\n\n# Clean display preview β€” only show meaningful columns\ndisplay(df_eda[['class', 'width', 'length', 'height', 'volume']].head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:13.464756Z","iopub.execute_input":"2026-04-06T18:12:13.465062Z","iopub.status.idle":"2026-04-06T18:12:15.654264Z","shell.execute_reply.started":"2026-04-06T18:12:13.465005Z","shell.execute_reply":"2026-04-06T18:12:15.653553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 1: DATASET STRUCTURE OVERVIEW\n# ============================================================\nprint(\"=\" * 60)\nprint(\"LYFT 3D OBJECT DETECTION β€” DATASET STRUCTURE OVERVIEW\")\nprint(\"=\" * 60)\n\n# Core SDK tables and their sizes\ntables = ['sample', 'sample_data', 'sample_annotation', 'instance', \n 'category', 'sensor', 'calibrated_sensor', 'ego_pose', 'scene', 'log']\n\nprint(f\"\\n{'Table':<25} {'# Records':>10}\")\nprint(\"-\" * 37)\nfor table in tables:\n try:\n records = lyft.get_table(table) if hasattr(lyft, 'get_table') else getattr(lyft, table)\n print(f\"{table:<25} {len(records):>10,}\")\n except:\n pass\n\n# Scene-level info\nprint(f\"\\nπŸ“½οΈ Total Scenes : {len(lyft.scene)}\")\nprint(f\"πŸ“Έ Total Samples : {len(lyft.sample)}\")\nprint(f\"🏷️ Total Annotations: {len(lyft.sample_annotation)}\")\nprint(f\"πŸ“¦ Unique Categories: {len(lyft.category)}\")\n\n# Samples per scene\nsamples_per_scene = []\nfor scene in lyft.scene:\n samples_per_scene.append(scene['nbr_samples'])\n\nprint(f\"\\n⏱️ Samples per Scene β€” Min: {min(samples_per_scene)}, Max: {max(samples_per_scene)}, Avg: {np.mean(samples_per_scene):.1f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:15.655402Z","iopub.execute_input":"2026-04-06T18:12:15.655596Z","iopub.status.idle":"2026-04-06T18:12:15.664522Z","shell.execute_reply.started":"2026-04-06T18:12:15.655562Z","shell.execute_reply":"2026-04-06T18:12:15.663690Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\"\"\nReading the Output β€” Table by Table\n\nscene: 180\nThe car went on 180 separate driving trips. Each trip is called a scene. Every scene is about 25 seconds long.\n\nsample: 22,680\nWithin those 180 trips, the system took 22,680 snapshots β€” one every ~0.5 seconds. These are your individual frames of data. 22,680 Γ· 180 = exactly 126 frames per scene, which is confirmed by the last line of output.\n\nsample_data: 189,504\nFor each of those 22,680 frames, multiple sensors were recording simultaneously β€” 6 cameras + the LiDAR. So each frame produces multiple files. 189,504 Γ· 22,680 β‰ˆ 8.35 files per frame, which roughly matches the number of sensors.\n\nsample_annotation: 638,179\nAcross all 22,680 frames, human annotators drew 638,179 bounding boxes around objects. This is the core labelled data the model learns from.\n\ninstance: 18,421\nThis is the most subtle one. An \"instance\" is a unique physical object that appears across multiple frames. If the same car is visible for 50 consecutive frames, that's 50 annotations but only 1 instance. So 638,179 annotations collapsed down to 18,421 unique real-world objects.\n\ncategory: 9\nJust the 9 object classes β€” car, pedestrian, truck, bus, bicycle, motorcycle, other_vehicle, emergency_vehicle, animal.\n\nsensor: 10\nThe 10 physical sensors on the car β€” 7 cameras and 3 LiDARs.\n\ncalibrated_sensor: 148\nEach sensor has a slightly different physical position and angle on the car body, and different cars have slightly different sensor placements. The calibration records store the exact position and rotation of each sensor on each specific vehicle. 148 because multiple vehicles were used across the 180 scenes.\n\nego_pose: 177,789\nEvery time the car's GPS/IMU system recorded its own position and orientation in the world β€” 177,789 times. This is more frequent than sample (22,680) because the ego pose is recorded at a higher rate than the annotated frames.\n\nlog: 180\nOne log per scene β€” basically metadata about each driving session (date, location, which vehicle).\n\"\"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:15.665786Z","iopub.execute_input":"2026-04-06T18:12:15.666034Z","iopub.status.idle":"2026-04-06T18:12:15.678977Z","shell.execute_reply.started":"2026-04-06T18:12:15.665998Z","shell.execute_reply":"2026-04-06T18:12:15.678138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 2: CATEGORY DEEP DIVE\n# ============================================================\n\n# --- Prepare Data ---\nclass_counts = df_eda['class'].value_counts()\ntotal = len(df_eda)\n\nsummary = pd.DataFrame({\n 'Count': class_counts,\n 'Percentage (%)': (class_counts / total * 100).round(2)\n})\n\nanchor_boxes = df_eda.groupby('class')[['width', 'length', 'height']].mean().round(2)\n\ncombined_table = summary.join(anchor_boxes)\n\n\n# --- Plot: Class Distribution ---\nfig, ax = plt.subplots(figsize=(12, 6))\n\nsns.barplot(\n x=class_counts.values,\n y=class_counts.index,\n palette='plasma',\n ax=ax\n)\nax.set_title('Object Class Distribution', fontsize=15, fontweight='bold')\n\nfor i, v in enumerate(class_counts.values):\n ax.text(v + 50, i, f'{v:,} ({v/total*100:.2f}%)', va='center', fontsize=10)\n\nplt.tight_layout()\nplt.show()\n\n\n# --- Combined Table Output ---\nprint(\"\\nπŸ“Š Combined Class Statistics Table:\")\ndisplay(combined_table)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:15.680644Z","iopub.execute_input":"2026-04-06T18:12:15.680963Z","iopub.status.idle":"2026-04-06T18:12:16.256884Z","shell.execute_reply.started":"2026-04-06T18:12:15.680905Z","shell.execute_reply":"2026-04-06T18:12:16.255847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 3: BOUNDING BOX DIMENSIONS\n# Tells the reader the physical size of each object class β€”\n# critical context for understanding what the model must detect\n# ============================================================\n\ndims = ['width', 'length', 'height']\nfig, axes = plt.subplots(1, 3, figsize=(20, 10))\n\nfor i, dim in enumerate(dims):\n clip_val = df_eda[dim].quantile(0.99)\n clipped = df_eda[df_eda[dim] <= clip_val]\n sns.boxplot(x='class', y=dim, data=clipped,\n palette='Set2', ax=axes[i], showfliers=False)\n axes[i].set_title(f'{dim.capitalize()} by class (metres)',\n fontsize=13, fontweight='bold')\n axes[i].tick_params(axis='x', rotation=45)\n axes[i].set_xlabel('')\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\nπŸ“¦ Anchor Box Reference (mean dimensions):\")\nanchor = df_eda.groupby('class')[['width','length','height','volume']].mean().round(3)\nanchor['aspect_l_w'] = (anchor['length'] / anchor['width']).round(2)\ndisplay(anchor.sort_values('volume', ascending=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:16.258790Z","iopub.execute_input":"2026-04-06T18:12:16.259211Z","iopub.status.idle":"2026-04-06T18:12:17.964596Z","shell.execute_reply.started":"2026-04-06T18:12:16.259151Z","shell.execute_reply":"2026-04-06T18:12:17.963775Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"LiDAR stands for Light Detection And Ranging. The sensor on this car is a Velodyne HDL-64E β€” a cylinder about the size of a coffee can mounted on the roof that spins 360Β° continuously, firing 64 laser beams simultaneously at different vertical angles. Each beam shoots out, hits something, bounces back, and the sensor records:\n\nWhere the reflection came from (x, y, z coordinates)\n\nHow strong the reflection was (intensity)\n\nWhich beam fired it (ring index β€” beam 0 is lowest angle, beam 63 is highest)\n\nIt does this full 360Β° rotation about 10 times per second. One full rotation = one sweep = one point cloud. Each sweep gives you roughly 60,000-100,000 individual points, each one a precise 3D measurement of whatever surface the laser hit.\nThe result is a 3D point cloud β€” a constellation of dots in space representing the physical geometry of everything around the car.","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# SECTION 4: LIDAR DEEP DIVE\n# The LiDAR fires 64 laser beams at different vertical angles.\n# Each point captured has: x, y, z, intensity, ring_index\n# We'll visualize height, intensity, and ring structure.\n# ============================================================\nfrom lyft_dataset_sdk.utils.data_classes import LidarPointCloud\n\nFRAME = 300\nsample = lyft.sample[FRAME]\nlidar_token = sample['data']['LIDAR_TOP']\npc = LidarPointCloud.from_file(lyft.get_sample_data_path(lidar_token))\n\n# pc.points shape is (4, N): rows are x, y, z, intensity\nx = pc.points[0, :]\ny = pc.points[1, :]\nz = pc.points[2, :]\nintensity = pc.points[3, :]\n\nprint(f\"Total LiDAR points in this scan : {len(x):,}\")\nprint(f\"X range : {x.min():.1f}m to {x.max():.1f}m\")\nprint(f\"Y range : {y.min():.1f}m to {y.max():.1f}m\")\nprint(f\"Z range : {z.min():.1f}m to {z.max():.1f}m\")\nprint(f\"Intensity: {intensity.min():.1f} to {intensity.max():.1f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:17.967968Z","iopub.execute_input":"2026-04-06T18:12:17.968372Z","iopub.status.idle":"2026-04-06T18:12:17.996425Z","shell.execute_reply.started":"2026-04-06T18:12:17.968207Z","shell.execute_reply":"2026-04-06T18:12:17.995597Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 4a: BEV coloured by HEIGHT (z) β€” the right way ---\n\nfig, ax = plt.subplots(figsize=(20, 9))\n\n# Clip to 60m radius for a clean view\nmask = (np.abs(x) < 60) & (np.abs(y) < 60)\nx_c, y_c, z_c = x[mask], y[mask], z[mask]\n\n# Scatter plot coloured by height\nsc1 = ax.scatter(\n -x_c, y_c,\n c=z_c,\n cmap='RdYlGn_r',\n s=0.3,\n alpha=0.8,\n vmin=-3,\n vmax=4\n)\n\n# Colorbar\nplt.colorbar(sc1, ax=ax, label='Height z (metres)')\n\n# Ego vehicle marker\nax.plot(0, 0, 'r*', markersize=14, label='Ego vehicle')\n\n# Labels\nax.set_title(\n 'BEV β€” Coloured by Height (Z)\\nRed=tall objects, Yellow=mid, Green=ground',\n fontsize=13,\n fontweight='bold'\n)\nax.set_xlabel('X forward (m)')\nax.set_ylabel('Y left (m)')\nax.set_aspect('equal')\nax.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:17.998020Z","iopub.execute_input":"2026-04-06T18:12:17.998386Z","iopub.status.idle":"2026-04-06T18:12:20.714127Z","shell.execute_reply.started":"2026-04-06T18:12:17.998221Z","shell.execute_reply":"2026-04-06T18:12:20.713208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 4b: Height histogram WITH corrected ground plane ---\n\n# Detect ground plane β€” must run before any z_rel calculations\nground_z = float(pd.Series(z).round(1).value_counts().idxmax())\nprint(f\"Detected ground plane: z β‰ˆ {ground_z:.2f}m\")\nprint(f\"LiDAR sensor is mounted ~{abs(ground_z):.1f}m above ground on the vehicle roof\")\n\nplt.figure(figsize=(12, 8))\n\n# Height histogram β€” corrected relative to ground\nz_rel_all = z - ground_z\n\nplt.hist(z_rel_all, bins=150, color='steelblue',\n edgecolor='none', density=True, range=(-1, 8))\n\n# Reference lines\nplt.axvline(0, color='red', linestyle='--', linewidth=2, label='Ground (0m)')\nplt.axvline(1.5, color='orange', linestyle='--', linewidth=2, label='Car roof (~1.5m)')\nplt.axvline(3.5, color='green', linestyle='--', linewidth=2, label='Truck roof (~3.5m)')\n\n# Formatting\nplt.title('LiDAR Height Distribution (Ground-Corrected)\\n'\n 'Peaks indicate the most common object heights in the scene',\n fontsize=15, fontweight='bold')\nplt.xlabel('Height above ground (m)', fontsize=12)\nplt.ylabel('Point density', fontsize=12)\nplt.legend(fontsize=11)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:20.715258Z","iopub.execute_input":"2026-04-06T18:12:20.715477Z","iopub.status.idle":"2026-04-06T18:12:21.554753Z","shell.execute_reply.started":"2026-04-06T18:12:20.715440Z","shell.execute_reply":"2026-04-06T18:12:21.553940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 4c: Multi-sweep (1 vs 3 vs 5 sweeps) ---\n# More sweeps = stacking consecutive frames = denser cloud\n# Useful to show because detection models often use 3-5 sweeps\n\nfig, axes = plt.subplots(1, 3, figsize=(21, 12))\n\nfor ax, nsweeps in zip(axes, [1, 3, 5]):\n pc_multi, _ = LidarPointCloud.from_file_multisweep(\n lyft, sample, 'LIDAR_TOP', 'LIDAR_TOP', num_sweeps=nsweeps\n )\n xm = pc_multi.points[0, :]\n ym = pc_multi.points[1, :]\n zm = pc_multi.points[2, :]\n zm_rel = zm - ground_z # correct for ground plane\n\n mask = (np.abs(xm) < 60) & (np.abs(ym) < 60)\n ax.scatter(-xm[mask], ym[mask], c=zm_rel[mask],\n cmap='RdYlGn_r', s=0.2, alpha=0.7,\n vmin=-0.3, vmax=4.0)\n ax.plot(0, 0, 'r*', markersize=10)\n ax.set_title(f'{nsweeps} sweep{\"s\" if nsweeps > 1 else \"\"} accumulated\\n{mask.sum():,} points',\n fontsize=13, fontweight='bold')\n ax.set_xlabel('X (m)')\n ax.set_ylabel('Y (m)')\n ax.set_aspect('equal')\n\nplt.suptitle('Effect of Accumulating Multiple LiDAR Sweeps\\n'\n 'Detection models use 3–5 sweeps for denser coverage of fast-moving objects',\n fontsize=14, fontweight='bold')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:21.556220Z","iopub.execute_input":"2026-04-06T18:12:21.556527Z","iopub.status.idle":"2026-04-06T18:12:36.839487Z","shell.execute_reply.started":"2026-04-06T18:12:21.556469Z","shell.execute_reply":"2026-04-06T18:12:36.838708Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 4d: Annotated BEV vs camera side by side ---\n\nfig, axes = plt.subplots(1, 2, figsize=(20, 15))\nlyft.render_sample_data(sample['data']['CAM_FRONT'], ax=axes[0])\naxes[0].set_title('Camera view β€” 3D boxes projected', fontsize=13, fontweight='bold')\n\nlyft.render_sample_data(sample['data']['LIDAR_TOP'], nsweeps=3, ax=axes[1])\naxes[1].set_title('LiDAR BEV β€” 3 sweeps with annotation boxes', fontsize=13, fontweight='bold')\n\nplt.suptitle('Same frame: what the camera sees vs what LiDAR sees',\n fontsize=14, fontweight='bold')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:36.840881Z","iopub.execute_input":"2026-04-06T18:12:36.841153Z","iopub.status.idle":"2026-04-06T18:12:43.576524Z","shell.execute_reply.started":"2026-04-06T18:12:36.841106Z","shell.execute_reply":"2026-04-06T18:12:43.575761Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"According to this BEV, we can conclude that the car is moving towards the right","metadata":{}},{"cell_type":"code","source":"import plotly.graph_objects as go\nimport numpy as np\nfrom lyft_dataset_sdk.utils.data_classes import LidarPointCloud\n\n# 1. Grab a sample and get the path to the actual LiDAR .bin file\nmy_sample = lyft.sample[FRAME] \nlidar_token = my_sample['data']['LIDAR_TOP']\nlidar_data = lyft.get('sample_data', lidar_token)\nlidar_filepath = lyft.data_path / lidar_data['filename']\n\n# 2. Load the Point Cloud using the SDK\npc = LidarPointCloud.from_file(lidar_filepath)\n\n# 3. Extract X, Y, Z coordinates\n# The SDK loads points as a (5, N) matrix. [X, Y, Z, Intensity, Ring index]\nx = pc.points[0, :]\ny = pc.points[1, :]\nz = pc.points[2, :]\n\n# WARNING: A single LiDAR sweep has ~60,000 to 100,000 points.\n# Rendering 100k points in a web browser will crash the Kaggle tab.\n# We randomly sample 25,000 points to keep the interactivity perfectly smooth.\nnum_points = x.shape[0]\nsample_indices = np.random.choice(num_points, size=25000, replace=False)\n\nx_sampled = x[sample_indices]\ny_sampled = y[sample_indices]\nz_sampled = z[sample_indices]\n\n# 4. Create the Plotly 3D Scatter Plot\nfig = go.Figure(data=[go.Scatter3d(\n x=x_sampled,\n y=y_sampled,\n z=z_sampled,\n mode='markers',\n marker=dict(\n size=1.5,\n color=z_sampled, # Color the points based on their height (Z-axis)\n colorscale='Viridis', # Beautiful color gradient (Yellow is high, Purple is low)\n opacity=0.8\n )\n)])\n\n# 5. Format the layout so the physical proportions remain 1:1:1\nfig.update_layout(\n title=\"Interactive 3D LiDAR Point Cloud (Drag to Rotate, Scroll to Zoom)\",\n scene=dict(\n xaxis_title='X (meters)',\n yaxis_title='Y (meters)',\n zaxis_title='Z (meters)',\n aspectmode='data', # Crucial: stops Plotly from squishing the 3D space!\n \n # Start with a nice angled top-down view\n camera=dict(\n up=dict(x=0, y=0, z=1),\n center=dict(x=0, y=0, z=0),\n eye=dict(x=0.5, y=-1.5, z=1.5)\n )\n ),\n margin=dict(l=0, r=0, b=0, t=40)\n)\n\nfig.show(renderer=\"iframe\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:43.578064Z","iopub.execute_input":"2026-04-06T18:12:43.578338Z","iopub.status.idle":"2026-04-06T18:12:45.684199Z","shell.execute_reply.started":"2026-04-06T18:12:43.578292Z","shell.execute_reply":"2026-04-06T18:12:45.683294Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Ego pose is stored separately from annotations β€” the car's position is recorded independently of what objects were annotated.","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# SECTION 5: SPATIAL DISTRIBUTION\n# Where do objects actually appear relative to the ego vehicle?\n# ============================================================\n\npositions = []\nfor ann in tqdm(lyft.sample_annotation[:5000], desc=\"Parsing positions\"):\n s = lyft.get('sample', ann['sample_token'])\n sd = lyft.get('sample_data', s['data']['LIDAR_TOP'])\n ego = lyft.get('ego_pose', sd['ego_pose_token'])\n ox, oy, _ = ann['translation']\n ex, ey, _ = ego['translation']\n positions.append({\n 'class': ann['category_name'],\n 'rel_x': ox - ex,\n 'rel_y': oy - ey,\n })\n\n#Simple subtraction. If the ego is at world position (2680m, 698m) and a car is at (2720m, 698m), \n#then rel_x = 40m, rel_y = 0m β€” that car is 40 meters directly ahead. \n#The _ throws away the z coordinate because you only care about ground plane position here.\n\ndf_pos = pd.DataFrame(positions)\ndf_pos['dist'] = np.sqrt(df_pos['rel_x']**2 + df_pos['rel_y']**2)\ntop_classes = df_pos['class'].value_counts().head(4).index.tolist()\n\nfig, axes = plt.subplots(1, len(top_classes), figsize=(20, 7))\nfor i, cls in enumerate(top_classes):\n sub = df_pos[df_pos['class'] == cls]\n axes[i].hist2d(sub['rel_x'], sub['rel_y'],\n bins=80, cmap='YlOrRd',\n range=[[-60,60],[-60,60]])\n axes[i].plot(0, 0, 'b*', markersize=12, label='Ego')\n axes[i].set_title(cls, fontsize=13, fontweight='bold')\n axes[i].set_xlabel('X forward (m)')\n axes[i].set_ylabel('Y left (m)')\n axes[i].set_aspect('equal')\n axes[i].legend()\n\nplt.suptitle('Where Do Objects Appear Relative to the Ego Vehicle?\\n'\n 'Bright = objects appear here most frequently',\n fontsize=15, fontweight='bold')\nplt.tight_layout()\nplt.show()\n\nprint(\"\\nπŸ“Š Median detection distance per class (metres):\")\ndisplay(df_pos.groupby('class')['dist'].median()\n .sort_values().round(1).rename('median_dist_m'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:45.685721Z","iopub.execute_input":"2026-04-06T18:12:45.686058Z","iopub.status.idle":"2026-04-06T18:12:47.303058Z","shell.execute_reply.started":"2026-04-06T18:12:45.685999Z","shell.execute_reply":"2026-04-06T18:12:47.302284Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Pedestrians and cyclists appear close and require high sensitivity at short range. Trucks and buses appear far away and require the model to handle very sparse point clouds","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# SECTION 6: VELOCITY ANALYSIS\n# Uses lyft.box_velocity() β€” computes centered difference\n# between prev/next annotations. Returns NaN if uncomputable.\n# ============================================================\n\nvel_data = []\nfor ann in tqdm(lyft.sample_annotation[:8000], desc=\"Computing velocities\"):\n vx, vy, vz = lyft.box_velocity(ann['token'])\n\n if np.isnan(vx): # single annotation β€” can't estimate\n continue\n\n speed = np.sqrt(vx**2 + vy**2) * 3.6\n vel_data.append({\n 'class': ann['category_name'],\n 'vx' : vx,\n 'vy' : vy,\n 'speed': speed,\n })\n\ndf_vel = pd.DataFrame(vel_data)\n\nfig, axes = plt.subplots(1, 2, figsize=(18, 6))\n\n# --- Speed distribution per class ---\nsns.boxplot(x='class', y='speed', data=df_vel,\n palette='Set2', showfliers=False, ax=axes[0])\naxes[0].set_title('Object Speed by Class (km/h)', fontsize=13, fontweight='bold')\naxes[0].tick_params(axis='x', rotation=45)\naxes[0].set_ylabel('Speed (km/h)')\naxes[0].set_xlabel('')\n\n# --- Average speed bar ---\navg_speed = df_vel.groupby('class')['speed'].mean().sort_values(ascending=False)\nsns.barplot(x=avg_speed.values, y=avg_speed.index, palette='Set2', ax=axes[1])\nfor i, v in enumerate(avg_speed.values):\n axes[1].text(v + 0.05, i, f'{v:.2f} km/h', va='center', fontsize=10)\naxes[1].set_title('Average Speed per Class', fontsize=13, fontweight='bold')\naxes[1].set_xlabel('Avg speed (km/h)')\n\nplt.suptitle('Object Velocity Analysis (SDK box_velocity)', fontsize=16, fontweight='bold')\nplt.tight_layout()\nplt.show()\n\nprint(\"\\nπŸ“Š Speed Stats (km/h):\")\ndisplay(df_vel.groupby('class')['speed'].describe().round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:47.304345Z","iopub.execute_input":"2026-04-06T18:12:47.304782Z","iopub.status.idle":"2026-04-06T18:12:48.236683Z","shell.execute_reply.started":"2026-04-06T18:12:47.304584Z","shell.execute_reply":"2026-04-06T18:12:48.235775Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Count means how many annotations of that class successfully produced a velocity estimate out of the 8000 annotations you sampled.","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# SECTION 7: EGO VEHICLE TRAJECTORY ON MAP\n# This plots every GPS position the car visited across ALL\n# scenes onto the actual Palo Alto street map.\n# Bright spots = roads driven most frequently.\n# It tells us: dataset coverage, geographic spread, and\n# whether the data has route diversity or is repetitive.\n# ============================================================\n\nlocation = lyft.log[0]['location']\nprint(f\"πŸ“ Recording location: {location}\")\nprint(f\" Total ego poses recorded: {len(lyft.ego_pose):,}\")\nprint(f\" Total scenes: {len(lyft.scene)}\")\nprint(f\" This map shows all routes driven across every scene.\\n\")\n\nplt.figure(figsize=(14, 14))\nlyft.render_egoposes_on_map(log_location=location)\nplt.suptitle(\n f'Ego Vehicle Route Coverage β€” {location}\\n'\n 'Colour = how many times that road segment was driven',\n fontsize=14, fontweight='bold', y=1.01\n)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:12:48.237941Z","iopub.execute_input":"2026-04-06T18:12:48.238148Z","iopub.status.idle":"2026-04-06T18:14:31.500729Z","shell.execute_reply.started":"2026-04-06T18:12:48.238112Z","shell.execute_reply":"2026-04-06T18:14:31.499760Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 8: CAMERA IMAGE ANALYSIS FROM ALL ANGLES\n# ============================================================\n\n# List of all 6 cameras on the car\ncamera_channels = [\n 'CAM_FRONT_LEFT', 'CAM_FRONT', 'CAM_FRONT_RIGHT',\n 'CAM_BACK_LEFT', 'CAM_BACK', 'CAM_BACK_RIGHT'\n]\nmy_sample = lyft.sample[FRAME] \n\n# Create a big 2x3 grid for the images\nfig, axes = plt.subplots(2, 3, figsize=(20, 10))\naxes = axes.flatten() # Flatten the grid so we can loop through it easily\n\n# Loop through each camera and plot it\nfor i, camera_name in enumerate(camera_channels):\n \n # Get the token for this specific camera in our sample\n cam_token = my_sample['data'][camera_name]\n \n # Use the SDK to render it on our specific subplot axis\n lyft.render_sample_data(cam_token, ax=axes[i])\n \n # Give it a title and hide the axis ticks\n axes[i].set_title(camera_name, fontsize=16, fontweight='bold')\n axes[i].axis('off')\n\nplt.tight_layout()\nplt.subplots_adjust(top=0.9)\nplt.suptitle('Self-Driving Car: Full 360Β° Camera Suite View', fontsize=24)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:14:31.502035Z","iopub.execute_input":"2026-04-06T18:14:31.502306Z","iopub.status.idle":"2026-04-06T18:14:33.295659Z","shell.execute_reply.started":"2026-04-06T18:14:31.502263Z","shell.execute_reply":"2026-04-06T18:14:33.294797Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 9: SENSOR FUSION INTO CAMERA FRAME\n# ============================================================\n\nplt.figure(figsize=(20, 12))\n\n# We are projecting the Top LiDAR points onto the Front Camera image\nlyft.render_pointcloud_in_image(\n my_sample['token'],\n pointsensor_channel='LIDAR_TOP',\n camera_channel='CAM_FRONT',\n)\n\nplt.title(\"Sensor Fusion: LiDAR Point Cloud Projected onto 2D Camera\", fontsize=20)\nplt.axis('off')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:14:33.297293Z","iopub.execute_input":"2026-04-06T18:14:33.297580Z","iopub.status.idle":"2026-04-06T18:14:34.005003Z","shell.execute_reply.started":"2026-04-06T18:14:33.297533Z","shell.execute_reply":"2026-04-06T18:14:34.004314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SUMMARY: KEY FINDINGS FROM EDA\n# ============================================================\nprint(\"=\" * 60)\nprint(\"KEY FINDINGS\")\nprint(\"=\" * 60)\n\ntop_class = df_eda['class'].value_counts().index[0]\ntop_pct = df_eda['class'].value_counts(normalize=True).iloc[0] * 100\nrarest_class = df_eda['class'].value_counts().index[-1]\nrare_pct = df_eda['class'].value_counts(normalize=True).iloc[-1] * 100\n\nprint(f\"\\n1. CLASS IMBALANCE\")\nprint(f\" '{top_class}' dominates at {top_pct:.1f}% of all annotations\")\nprint(f\" '{rarest_class}' is rarest at {rare_pct:.2f}% β€” model will struggle here\")\n\nprint(f\"\\n2. DATASET SCALE\")\nprint(f\" {len(lyft.scene)} scenes, {len(lyft.sample):,} frames, \"\n f\"{len(lyft.sample_annotation):,} annotations\")\n\nprint(f\"\\n3. LIDAR\")\nprint(f\" Ground plane at zβ‰ˆ{ground_z:.2f}m (sensor mounted on roof)\")\nprint(f\" ~{len(x):,} points per scan, 64-beam rotating sensor\")\n\nprint(f\"\\n4. GEOGRAPHY\")\nprint(f\" All data recorded in: {lyft.log[0]['location']}\")\nprint(f\" Limited geographic diversity β€” model may not generalise globally\")\n\nprint(f\"\\n5. ANCHOR BOX PRIORS (avg dimensions)\")\ndisplay(df_eda.groupby('class')[['width','length','height']]\n .mean().round(2).sort_values('length', ascending=False))\n\nprint(f\"\\n6. VELOCITY\")\nprint(f\" Cars: median {df_vel[df_vel['class']=='car']['speed'].median():.1f} km/h\")\nprint(f\" Pedestrians: consistent ~{df_vel[df_vel['class']=='pedestrian']['speed'].median():.1f} km/h walking speed\")\nprint(f\" other_vehicle nearly stationary: median ~0 km/h\")\n\nprint(f\"\\n7. DETECTION RANGE\")\nprint(f\" Small objects (motorcycle/bicycle) detected at ~20-30m median\")\nprint(f\" Large objects (truck/bus) detected at ~45-52m median\")\nprint(f\" Size directly determines how far the model can reliably see an object\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:14:34.006322Z","iopub.execute_input":"2026-04-06T18:14:34.006523Z","iopub.status.idle":"2026-04-06T18:14:34.287595Z","shell.execute_reply.started":"2026-04-06T18:14:34.006487Z","shell.execute_reply":"2026-04-06T18:14:34.286843Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Phase 2 : Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# DATA PIPELINE: POINT CLOUD β†’ BEV IMAGE\n# ============================================================\n# Switching to the reference model's voxel approach.\n# Key difference from our previous approach:\n#\n# BEFORE: max-height + density + occupancy channels\n# Problem: single max-height value per cell loses\n# vertical structure β€” a car roof and a pedestrian\n# at the same height look identical\n#\n# NOW: 3 height-bin channels via voxel counting\n# Each channel = a vertical slice of space\n# Channel 0: z from -2.0m to -0.5m (near ground)\n# Channel 1: z from -0.5m to 1.0m (low objects)\n# Channel 2: z from 1.0m to 2.5m (car tops, trucks)\n# Point count per voxel β†’ normalized 0-1\n#\n# This gives the model vertical structure, not just height.\n# A car looks like: channel 1 lit up (body), channel 2 lit up (roof)\n# A pedestrian looks like: channel 1 lit up only (shorter)\n# ============================================================\n\n# --- ALL REQUIRED SDK IMPORTS ---\nfrom pyquaternion import Quaternion\nimport cv2\nimport numpy as np\nimport torch\nfrom lyft_dataset_sdk.utils.data_classes import LidarPointCloud, Box\n\nclass BEVConfig:\n # --------------------------------------------------------\n # Voxel dimensions in meters (x, y, z)\n # 0.4m Γ— 0.4m Γ— 1.5m per voxel β€” matches reference model\n # Coarser than our previous 0.25m but much faster to compute\n # --------------------------------------------------------\n VOXEL_SIZE = (0.4, 0.4, 1.5)\n\n # Z offset shifts the vertical slices downward\n # sensor is roof-mounted so ground is at z β‰ˆ -1.7m\n # z_offset = -2.0 puts ground at the bottom of channel 0\n Z_OFFSET = -2.0\n\n # Output BEV image shape: (H, W, C) = (336, 336, 3)\n # 336 Γ— 0.4m = 134.4m total coverage each side\n BEV_SHAPE = (336, 336, 3)\n\n # Scale boxes down to 80% before drawing target mask\n # Separates touching objects β€” without this adjacent cars\n # merge into one blob in the target\n BOX_SCALE = 0.8\n\n # Derived for convenience in dataset and inference code\n GRID_H = BEV_SHAPE[0] # 336\n GRID_W = BEV_SHAPE[1] # 336\n NUM_SWEEPS = 1 # Lock to 1 for training speed\n\ncfg = BEVConfig()\nprint(f\"BEV shape : {cfg.BEV_SHAPE}\")\nprint(f\"Voxel size : {cfg.VOXEL_SIZE} m\")\nprint(f\"Z offset : {cfg.Z_OFFSET} m\")\nprint(f\"Coverage : {cfg.GRID_W * cfg.VOXEL_SIZE[0]:.1f}m Γ— {cfg.GRID_H * cfg.VOXEL_SIZE[1]:.1f}m\")\nprint(f\"Box scale : {cfg.BOX_SCALE}\")\n\n\n# ============================================================\n# VOXEL HELPER FUNCTIONS\n# ============================================================\n# These are taken directly from the reference model and are\n# the standard way to convert LiDAR points into a BEV image\n# using voxel counting rather than max-height projection.\n# ============================================================\n\ndef create_transformation_matrix_to_voxel_space(shape, voxel_size, offset):\n # --------------------------------------------------------\n # Constructs a 4Γ—4 transformation matrix that maps\n # car-frame coordinates (meters) into voxel grid indices.\n #\n # The matrix centres the grid on (0,0,0) in car space,\n # applies the voxel scale, and shifts by the z_offset.\n #\n # shape : (H, W, C) of the voxel grid\n # voxel_size: (x, y, z) size of each voxel in meters\n # offset : (0, 0, z_offset) β€” shifts vertical position\n # --------------------------------------------------------\n shape, voxel_size, offset = (np.array(shape),\n np.array(voxel_size),\n np.array(offset))\n tm = np.eye(4, dtype=np.float32)\n translation = shape / 2 + offset / voxel_size\n tm = tm * np.array(np.hstack((1 / voxel_size, [1])))\n tm[:3, 3] = np.transpose(translation)\n return tm\n\n\ndef transform_points(points, transf_matrix):\n # --------------------------------------------------------\n # Apply a 4Γ—4 homogeneous transformation to a set of 3D\n # points. Input shape must be (3, N) or (4, N).\n # Returns (3, N) transformed points.\n # --------------------------------------------------------\n if points.shape[0] not in [3, 4]:\n raise Exception(\n f\"Points shape should be (3,N) or (4,N), got {points.shape}\"\n )\n return transf_matrix.dot(\n np.vstack((points[:3, :], np.ones(points.shape[1])))\n )[:3, :]\n\n\ndef car_to_voxel_coords(points, shape, voxel_size, z_offset=0):\n # --------------------------------------------------------\n # Convert car-frame (meter) coordinates into voxel indices.\n # Combines transformation matrix creation and application.\n # --------------------------------------------------------\n tm = create_transformation_matrix_to_voxel_space(\n shape, voxel_size, (0, 0, z_offset)\n )\n return transform_points(points, tm)\n\n\ndef create_voxel_pointcloud(points, shape, voxel_size, z_offset):\n # --------------------------------------------------------\n # Convert a raw (4, N) LiDAR point cloud into a voxel BEV.\n # Each voxel cell gets the COUNT of points that landed in it.\n # Output shape: (H, W, C) where C = number of z bins.\n #\n # FIX: Using np.floor and strict int32 casting to ensure\n # pixels align perfectly with camera projections.\n # --------------------------------------------------------\n pts_voxel = car_to_voxel_coords(\n points.copy(), shape, voxel_size, z_offset\n )\n pts_voxel = pts_voxel[:3].transpose(1, 0) # (N, 3)\n pts_voxel = np.floor(pts_voxel).astype(np.int32)\n\n bev = np.zeros(shape, dtype=np.float32)\n bev_shape_arr = np.array(shape)\n\n # Keep only points that land inside the grid bounds\n within_bounds = (\n np.all(pts_voxel >= 0, axis=1) &\n np.all(pts_voxel < bev_shape_arr, axis=1)\n )\n pts_voxel = pts_voxel[within_bounds]\n\n # Count how many points hit each voxel\n # Note: X and Y axes are swapped vs array indexing convention\n if len(pts_voxel) > 0:\n coord, count = np.unique(pts_voxel, axis=0, return_counts=True)\n bev[coord[:, 1], coord[:, 0], coord[:, 2]] = count\n\n return bev\n\n\ndef normalize_voxel_intensities(bev, max_intensity=16):\n # --------------------------------------------------------\n # Normalize voxel point counts to the range [0, 1].\n # max_intensity=16 means any voxel with 16+ points = 1.0\n # This prevents a single dense cluster from dominating.\n # --------------------------------------------------------\n return (bev / max_intensity).clip(0, 1)\n\n\n# ============================================================\n# BOX HELPER FUNCTIONS\n# ============================================================\n# Used both in dataset target generation and in inference\n# when converting predicted pixel boxes back to world space.\n# ============================================================\n\ndef move_boxes_to_car_space(boxes, ego_pose):\n # --------------------------------------------------------\n # Transform boxes from world GPS coordinates into the\n # ego vehicle's local coordinate frame.\n # Mutates the boxes in place.\n # --------------------------------------------------------\n translation = -np.array(ego_pose['translation'])\n rotation = Quaternion(ego_pose['rotation']).inverse\n for box in boxes:\n box.translate(translation)\n box.rotate(rotation)\n\n\ndef scale_boxes(boxes, factor):\n # --------------------------------------------------------\n # Scale each box's width/length/height by factor.\n # Used with factor=0.8 to shrink targets slightly β€”\n # prevents adjacent objects from merging in the mask.\n # Mutates the boxes in place.\n # --------------------------------------------------------\n for box in boxes:\n box.wlh = box.wlh * factor\n\n\ndef draw_boxes(im, voxel_size, boxes, classes, z_offset=0.0):\n # --------------------------------------------------------\n # Rasterize 3D box footprints onto a 2D image canvas.\n # Each box is drawn with a colour = class index + 1\n # (background = 0, car = 1, motorcycle = 2, etc.)\n # --------------------------------------------------------\n for box in boxes:\n corners = box.bottom_corners()\n \n # FIX: Pass a 3D shape so the voxel transformation math doesn't crash\n grid_shape_3d = (im.shape[0], im.shape[1], 3)\n \n corners_voxel = car_to_voxel_coords(\n corners, grid_shape_3d, voxel_size, z_offset\n ).transpose(1, 0)\n \n corners_voxel = corners_voxel[:, :2].astype(np.int32) # drop z\n\n # class index 0 = background, so object classes start at 1\n if box.name not in classes:\n continue\n \n class_color = classes.index(box.name) + 1\n \n # FIX: Pass single scalar integer for a 1-channel image\n cv2.drawContours(\n im, [corners_voxel], 0,\n int(class_color), -1\n )\n\n\n# Quick sanity check\nprint(\"Helper functions defined.\")\nprint(f\"Voxel transform test:\")\ntest_pts = np.array([[0.0], [0.0], [0.0]])\ntest_tm = create_transformation_matrix_to_voxel_space(\n cfg.BEV_SHAPE, cfg.VOXEL_SIZE, (0, 0, cfg.Z_OFFSET)\n)\ntest_out = transform_points(test_pts, test_tm)\nprint(f\" Car origin (0,0,0) β†’ voxel coords: \"\n f\"({test_out[0,0]:.1f}, {test_out[1,0]:.1f}, {test_out[2,0]:.1f})\")\nprint(f\" (should be near centre of grid: ~168, ~168)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:14:34.289090Z","iopub.execute_input":"2026-04-06T18:14:34.289497Z","iopub.status.idle":"2026-04-06T18:14:35.378944Z","shell.execute_reply.started":"2026-04-06T18:14:34.289296Z","shell.execute_reply":"2026-04-06T18:14:35.378213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# BEV + VOXEL REPRESENTATION β€” CLEAR EXPLANATION\n# ============================================================\n\n# 1. BEV SHAPE\n# ------------------------------------------------------------\n# BEV_SHAPE = (336, 336, 3)\n#\n# This is NOT just an image β€” it is a top-down map of the world.\n#\n# - 336 x 336 β†’ spatial grid (like pixels)\n# - 3 channels β†’ height slices (Z dimension)\n#\n# So overall:\n# Each pixel = a small area in the real world\n# Each channel = a vertical slice of height\n\n\n# 2. VOXEL SIZE\n# ------------------------------------------------------------\n# VOXEL_SIZE = (0.4, 0.4, 1.5) meters\n#\n# This defines the resolution of the grid:\n#\n# - Each pixel covers:\n# 0.4m (x direction) Γ— 0.4m (y direction)\n#\n# - Each channel covers:\n# 1.5m height (z direction)\n#\n# So each voxel = small 3D cell:\n# 0.4m Γ— 0.4m Γ— 1.5m\n\n\n# 3. COVERAGE\n# ------------------------------------------------------------\n# Coverage = 134.4m Γ— 134.4m\n#\n# Computation:\n# 336 pixels Γ— 0.4m = 134.4m\n#\n# Meaning:\n# The BEV grid covers ~67m in all directions from the car\n#\n# 67m left | car | 67m right\n# 67m front | | 67m back\n#\n# So total area = 134.4m Γ— 134.4m\n\n\n# 4. HOW VOXELS ARE USED (IMPORTANT)\n# ------------------------------------------------------------\n# We do NOT store a full 3D voxel grid.\n# Instead, we COMPRESS height into 3 channels.\n#\n# Each pixel represents a vertical column:\n#\n# ↑ height\n# β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”\n# β”‚ Channel 2 β”‚ (1.0m β†’ 2.5m)\n# β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n# β”‚ Channel 1 β”‚ (-0.5m β†’ 1.0m)\n# β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n# β”‚ Channel 0 β”‚ (-2.0m β†’ -0.5m)\n# β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜\n# 0.4m Γ— 0.4m (ground area)\n#\n# For each (x, y) cell:\n# We count how many LiDAR points fall into each height bin.\n#\n# So each pixel stores:\n# [count_low, count_mid, count_high]\n\n\n# 5. Z OFFSET\n# ------------------------------------------------------------\n# Z_OFFSET = -2.0m\n#\n# LiDAR sensor is mounted on the roof.\n# So:\n# Ground β‰ˆ -1.7m\n#\n# We shift everything DOWN so:\n# Ground aligns nicely with the bottom of the voxel grid\n#\n# This makes height bins consistent and easier to learn.\n\n\n# 6. CAR POSITION IN GRID\n# ------------------------------------------------------------\n# Output:\n# Car origin (0,0,0) β†’ (168, 168, ~0)\n#\n# Explanation:\n# Grid size = 336\n# Center = 336 / 2 = 168\n#\n# So:\n# The ego vehicle is always at the CENTER of the BEV map\n\n\n# 7. FINAL PIPELINE SUMMARY\n# ------------------------------------------------------------\n# 1. Take LiDAR points (x, y, z)\n# 2. Convert meters β†’ voxel grid indices\n# 3. Place into BEV tensor of shape (336, 336, 3)\n#\n# Each pixel answers:\n# \"How many points exist in this 0.4m Γ— 0.4m area\n# at different height levels?\"\n\n\n# 8. WHY THIS REPRESENTATION IS USED\n# ------------------------------------------------------------\n# - Converts irregular point cloud β†’ structured grid\n# - Makes it compatible with CNNs\n# - Preserves vertical structure via channels\n#\n# Example:\n# Car β†’ mid + high channels active\n# Pedestrian β†’ mostly mid channel\n# Ground β†’ low channel\n#\n# This allows the model to distinguish object types.\n\n\n# 9. BOX SCALE\n# ------------------------------------------------------------\n# BOX_SCALE = 0.8\n#\n# During training:\n# Bounding boxes are shrunk to 80% size\n#\n# Why?\n# Prevent nearby objects from merging into one blob\n# Improves training stability\n\n\n# ============================================================\n# ONE-LINE SUMMARY\n# ============================================================\n# \"We convert the LiDAR point cloud into a BEV grid covering\n# 134m Γ— 134m around the vehicle. Each pixel represents a\n# 0.4m Γ— 0.4m region, and height is discretized into 3 bins.\n# Each channel stores point density for a specific height range,\n# allowing the model to capture 3D structure in a 2D format.\"\n# ============================================================","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:14:35.380211Z","iopub.execute_input":"2026-04-06T18:14:35.380414Z","iopub.status.idle":"2026-04-06T18:14:35.387401Z","shell.execute_reply.started":"2026-04-06T18:14:35.380379Z","shell.execute_reply":"2026-04-06T18:14:35.386551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom lyft_dataset_sdk.utils.geometry_utils import transform_matrix\n\n# Object classes in the same order as the reference model\n# Index in this list + 1 = pixel value in target mask\n# (0 is reserved for background)\nCLASSES = [\"car\", \"motorcycle\", \"bus\", \"bicycle\", \"truck\",\n \"pedestrian\", \"other_vehicle\", \"animal\", \"emergency_vehicle\"]\n\nclass LyftBEVDataset(Dataset):\n # ============================================================\n # Dataset class: wraps the full point cloud β†’ BEV pipeline.\n # Each item returned is a training pair:\n #\n # bev_tensor : (3, 336, 336) float32 β€” voxel BEV image\n # 3 height-bin channels, values 0-1\n # Channel 0: near ground (~-2.0m to -0.5m)\n # Channel 1: mid level (~-0.5m to 1.0m)\n # Channel 2: high objects (~ 1.0m to 2.5m)\n #\n # target : (336, 336) int64 β€” integer class label map\n # 0 = background\n # 1 = car, 2 = motorcycle, 3 = bus, ...\n # This format is what F.cross_entropy expects.\n #\n # token : str β€” sample token for debugging\n # ============================================================\n\n def __init__(self, lyft_sdk, sample_tokens, cfg):\n self.lyft = lyft_sdk\n self.tokens = sample_tokens\n self.cfg = cfg\n\n def __len__(self):\n return len(self.tokens)\n\n def __getitem__(self, idx):\n token = self.tokens[idx]\n sample = self.lyft.get('sample', token)\n\n try:\n # --------------------------------------------------------\n # STEP 1: Load raw point cloud\n # --------------------------------------------------------\n lidar_token = sample['data']['LIDAR_TOP']\n lidar_data = self.lyft.get('sample_data', lidar_token)\n lidar_filepath = self.lyft.get_sample_data_path(lidar_token)\n ego_pose = self.lyft.get('ego_pose',\n lidar_data['ego_pose_token'])\n calibrated_sensor = self.lyft.get(\n 'calibrated_sensor',\n lidar_data['calibrated_sensor_token']\n )\n\n # --------------------------------------------------------\n # STEP 2: Load point cloud CORRECTLY (Optimized for Speed)\n # --------------------------------------------------------\n # Split logic ensures we only use multi-sweep when explicitly\n # requested. Single sweep is 10x faster for training.\n # --------------------------------------------------------\n if self.cfg.NUM_SWEEPS == 1:\n # βœ… FAST MODE: Single sweep\n lidar_pointcloud = LidarPointCloud.from_file(lidar_filepath)\n \n # Manual transform sensor β†’ car frame\n car_from_sensor = transform_matrix(\n calibrated_sensor['translation'],\n Quaternion(calibrated_sensor['rotation']),\n inverse=False\n )\n lidar_pointcloud.transform(car_from_sensor)\n \n else:\n # βœ… ANALYSIS MODE: Multi-sweep (Already aligned by SDK)\n lidar_pointcloud, _ = LidarPointCloud.from_file_multisweep(\n self.lyft,\n sample,\n 'LIDAR_TOP',\n 'LIDAR_TOP',\n num_sweeps=self.cfg.NUM_SWEEPS\n )\n\n # --------------------------------------------------------\n # STEP 3: Build the 3-channel voxel BEV image\n # --------------------------------------------------------\n # create_voxel_pointcloud divides 3D space into voxels\n # and counts how many points land in each voxel.\n # --------------------------------------------------------\n bev = create_voxel_pointcloud(\n lidar_pointcloud.points,\n self.cfg.BEV_SHAPE,\n voxel_size = self.cfg.VOXEL_SIZE,\n z_offset = self.cfg.Z_OFFSET\n )\n\n # Normalize point counts to 0-1 range\n bev = normalize_voxel_intensities(bev)\n\n # (H, W, C) β†’ (C, H, W) for PyTorch convention\n bev_tensor = torch.from_numpy(\n bev.transpose(2, 0, 1).astype(np.float32)\n ) # (3, 336, 336)\n\n # --------------------------------------------------------\n # STEP 4: Build the integer class target mask\n # --------------------------------------------------------\n # Project human-labeled 3D boxes into our BEV grid.\n # --------------------------------------------------------\n boxes = self.lyft.get_boxes(lidar_token)\n target = np.zeros(self.cfg.BEV_SHAPE[:2], dtype=np.uint8)\n\n move_boxes_to_car_space(boxes, ego_pose)\n scale_boxes(boxes, self.cfg.BOX_SCALE)\n draw_boxes(\n target,\n voxel_size = self.cfg.VOXEL_SIZE,\n boxes = boxes,\n classes = CLASSES,\n z_offset = self.cfg.Z_OFFSET\n )\n\n # Convert to int64 β€” required by F.cross_entropy\n target_tensor = torch.from_numpy(target.astype(np.int64))\n\n return bev_tensor, target_tensor, token\n\n except Exception:\n # 🚨 SILENCE THE BOTTLENECK: No prints in the hot-loop.\n # Returning zeros keeps the training pipeline moving.\n bev_tensor = torch.zeros(3, self.cfg.GRID_H, self.cfg.GRID_W, dtype=torch.float32)\n target_tensor = torch.zeros(self.cfg.GRID_H, self.cfg.GRID_W, dtype=torch.long)\n return bev_tensor, target_tensor, token","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:14:35.388745Z","iopub.execute_input":"2026-04-06T18:14:35.389219Z","iopub.status.idle":"2026-04-06T18:14:35.407299Z","shell.execute_reply.started":"2026-04-06T18:14:35.389045Z","shell.execute_reply":"2026-04-06T18:14:35.406506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# LYFT BEV DATASET PIPELINE β€” STEP-BY-STEP EXPLANATION\n# ============================================================\n\n# This Dataset class converts raw LiDAR + annotations into\n# training-ready tensors for a neural network.\n#\n# Each sample returned:\n# INPUT β†’ bev_tensor : (3, 336, 336)\n# TARGET β†’ target_mask : (336, 336)\n#\n# This turns 3D object detection into a 2D segmentation problem.\n\n\n# ============================================================\n# STEP 1: LOAD RAW LIDAR POINT CLOUD\n# ============================================================\n# We load the .bin file from the /lidar/ folder.\n# Points are initially in the SENSOR frame (centered on the roof).\n\n\n# ============================================================\n# STEP 2: TRANSFORM SENSOR β†’ CAR FRAME\n# ============================================================\n# We apply a rotation and translation matrix so that (0,0,0) \n# is the center of the car, not the sensor on top.\n# This aligns the lasers with the vehicle's footprint.\n\n\n# ============================================================\n# STEP 3: VOXELIZATION β†’ BEV IMAGE\n# ============================================================\n# Instead of a flat image, we slice the world into 3 layers:\n#\n# Channel 0 β†’ Ground level (-2.0m to -0.5m)\n# Channel 1 β†’ Object bodies (-0.5m to 1.0m)\n# Channel 2 β†’ Tall object tops ( 1.0m to 2.5m)\n#\n# We count how many laser hits land in each layer for every 0.4m grid square.\n\n\n# ============================================================\n# STEP 4: BUILD TARGET MASK\n# ============================================================\n# We take the Ground Truth 3D boxes, move them to car-space,\n# shrink them to 80% to separate objects, and \"color them in\" \n# on a 2D grid with their class IDs (Car=1, Pedestrian=6, etc.).\n\n\n# ============================================================\n# KEY INSIGHT\n# ============================================================\n# This converts 3D object detection into BEV Semantic Segmentation.\n# Instead of predicting boxes directly, the model learns to\n# classify every square of the road.\n\n\n# ============================================================\n# ONE-LINE SUMMARY\n# ============================================================\n# \"We convert LiDAR point clouds into a 3-channel BEV grid\n# using voxelization, and generate a pixel-wise class map\n# by projecting 3D bounding boxes into BEV space, enabling\n# object detection as a segmentation problem.\"\n# ============================================================","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:14:35.408575Z","iopub.execute_input":"2026-04-06T18:14:35.409135Z","iopub.status.idle":"2026-04-06T18:14:35.417410Z","shell.execute_reply.started":"2026-04-06T18:14:35.408854Z","shell.execute_reply":"2026-04-06T18:14:35.416728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# TEST THE DATASET CLASS\n# ============================================================\n# We take a small subset of 100 frames to verify the pipeline.\n# ============================================================\nsample_tokens = [s['token'] for s in lyft.sample[:100]]\ndataset = LyftBEVDataset(lyft, sample_tokens, cfg)\nprint(f\"Dataset size: {len(dataset)} frames\")\n\n# Select a frame to inspect\nf = 2\nbev, target, token = dataset[f]\n\nprint(f\"\\nSingle item check:\")\nprint(f\"BEV tensor shape : {bev.shape}\")\nprint(f\"Target tensor shape: {target.shape}\")\nprint(f\"BEV dtype : {bev.dtype}\")\nprint(f\"Target dtype : {target.dtype}\")\nprint(f\"BEV value range : {bev.min():.2f} to {bev.max():.2f}\")\nprint(f\"Target unique vals : {target.unique()}\")\nprint(f\"Token : {token[:16]}...\")\n\n# --- Quick 4-plot check WITH EGO MARKER ---\nfig, axes = plt.subplots(1, 4, figsize=(28, 9))\n\n# Updated channel names to reflect the Voxel Slice strategy\nchannel_names = [\n 'Channel 0: Near Ground\\n(Green=High density, Red=Low)',\n 'Channel 1: Mid Level\\n(White/Yellow=Objects)',\n 'Channel 2: High Objects\\n(White=Tall structures)',\n 'Target Mask\\n(Class labels per pixel)'\n]\ncmaps = ['RdYlGn_r', 'hot', 'gray']\n\n# Ego position (center of our 336x336 grid)\nego_x = cfg.GRID_W // 2\nego_y = cfg.GRID_H // 2\n\nfor i in range(3):\n axes[i].imshow(bev[i].numpy(), cmap=cmaps[i], origin='upper')\n axes[i].set_title(channel_names[i], fontsize=13, fontweight='bold')\n axes[i].set_xlabel('X pixel (forward β†’)')\n axes[i].set_ylabel('Y pixel (left ↑)')\n \n # Invert X axis to match standard Bird's Eye View convention\n axes[i].invert_xaxis()\n\n # πŸ”΄ Draw ego vehicle\n axes[i].scatter(ego_x, ego_y, c='red', s=100, marker='*', label='Ego')\n axes[i].legend(loc='upper right')\n\n\n# --- Target mask ---\n# We use origin='lower' for the target mask to align with car-space coords\naxes[3].imshow(target.numpy(), cmap='hot', origin='lower')\naxes[3].set_title('Target Mask', fontsize=13, fontweight='bold')\naxes[3].set_xlabel('X pixel (forward β†’)')\naxes[3].set_ylabel('Y pixel (left ↑)')\n\n# πŸ”΄ Draw ego vehicle\naxes[3].scatter(ego_x, ego_y, c='cyan', s=100, marker='*', label='Ego')\naxes[3].legend(loc='upper right')\n\nplt.suptitle('Dataset __getitem__ Output β€” One Training Pair (Ego Marked)',\n fontsize=15, fontweight='bold')\nplt.tight_layout()\nplt.show()\n\n# --- DataLoader batch check ---\n# Verifies that batching (B, C, H, W) is working correctly\nloader = DataLoader(dataset, batch_size=4, shuffle=True, num_workers=0)\nbev_batch, target_batch, tokens_batch = next(iter(loader))\n\nprint(f\"\\nDataLoader batch check:\")\nprint(f\"BEV batch shape : {bev_batch.shape}\")\nprint(f\"Target batch shape: {target_batch.shape}\")\nprint(\"This is what the model actually receives during training β€” 4 frames simultaneously.\")\n\n# ============================================================\n# FULL SANITY CHECK (Pipeline vs Camera vs SDK)\n# ============================================================\ncheck_token = sample_tokens[f]\ncheck_sample = lyft.get('sample', check_token)\n\nfig = plt.figure(figsize=(28, 25))\n\nchannel_names_full = [\n 'Channel 0: Near Ground\\n(Green=High density, Red=Low)',\n 'Channel 1: Mid Level\\n(White/Yellow=Objects)',\n 'Channel 2: High Objects\\n(White=Tall structures)',\n 'Target Mask\\n(Class labels per pixel)'\n]\ncmaps_full = ['RdYlGn_r', 'hot', 'gray', 'hot']\n\n# Pipeline Tensors\ntensors = [bev[0].numpy(), bev[1].numpy(),\n bev[2].numpy(), target.numpy()]\n \norigins = ['upper', 'upper', 'upper', 'lower'] # Key for alignment\nflip_x = [True, True, True, False]\n\n# Ego position\nego_x = cfg.GRID_W // 2\nego_y = cfg.GRID_H // 2\n\nfor i in range(4):\n ax = fig.add_subplot(3, 4, i + 1)\n ax.imshow(tensors[i], cmap=cmaps_full[i], origin=origins[i])\n ax.set_title(channel_names_full[i], fontsize=11, fontweight='bold')\n\n if flip_x[i]:\n ax.invert_xaxis()\n\n ax.set_xlabel('X (forward β†’)')\n ax.set_ylabel('Y (left ↑)')\n\n # πŸ”΄ Mark ego\n ax.scatter(ego_x, ego_y, c='cyan', s=80, marker='*')\n\n# Camera Views for visual confirmation\ncamera_channels = [\n 'CAM_FRONT_LEFT', 'CAM_FRONT', 'CAM_FRONT_RIGHT',\n 'CAM_BACK_LEFT', 'CAM_BACK', 'CAM_BACK_RIGHT'\n]\nfor i, cam_name in enumerate(camera_channels):\n ax = fig.add_subplot(3, 4, i + 5)\n lyft.render_sample_data(check_sample['data'][cam_name], ax=ax)\n ax.set_title(cam_name, fontsize=11, fontweight='bold')\n ax.axis('off')\n\n# Final Reference: The SDK's built-in BEV renderer\nax_sdk = fig.add_subplot(3, 4, 11)\nlyft.render_sample_data(check_sample['data']['LIDAR_TOP'], nsweeps=1, ax=ax_sdk)\nax_sdk.set_title('SDK LiDAR BEV\\n(ground truth reference)',\n fontsize=11, fontweight='bold')\n\nplt.suptitle(\n f'Full Sanity Check β€” Frame token: {check_token[:16]}...\\n'\n 'Top row: our pipeline output | Middle+Bottom: camera views | '\n 'Bottom-right: SDK reference BEV',\n fontsize=13, fontweight='bold'\n)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:14:35.418705Z","iopub.execute_input":"2026-04-06T18:14:35.418996Z","iopub.status.idle":"2026-04-06T18:14:43.810469Z","shell.execute_reply.started":"2026-04-06T18:14:35.418937Z","shell.execute_reply":"2026-04-06T18:14:43.809244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# DATASET OUTPUT + SANITY CHECK β€” EXPLANATION\n# ============================================================\n\n# This cell verifies that our Dataset pipeline is working correctly.\n# It checks:\n# 1. Single sample output (shapes, values)\n# 2. Visual correctness (BEV + target alignment)\n# 3. Batch loading via DataLoader\n# 4. Full consistency with camera + SDK visualization\n\n\n# ============================================================\n# STEP 1: CREATE DATASET\n# ============================================================\n# We take first 100 frames from dataset\n# and wrap them inside our custom Dataset class\n\n# sample_tokens = [s['token'] for s in lyft.sample[:100]]\n# dataset = LyftBEVDataset(lyft, sample_tokens, cfg)\n\n# Each item in dataset returns:\n# bev_tensor β†’ (3, 336, 336)\n# target_tensor β†’ (336, 336)\n# token β†’ frame ID\n\n\n# ============================================================\n# STEP 2: SINGLE SAMPLE CHECK\n# ============================================================\n# We fetch one frame to verify:\n# - tensor shapes\n# - data types\n# - value ranges\n\n# bev, target, token = dataset[f]\n\n# Expected:\n# BEV:\n# shape = (3, 336, 336)\n# dtype = float32\n# values ∈ [0, 1]\n#\n# Target:\n# shape = (336, 336)\n# dtype = int64\n# values ∈ {0, 1, ..., 9}\n\n\n# ============================================================\n# STEP 3: VISUALIZE BEV CHANNELS\n# ============================================================\n# We plot the 3 BEV channels:\n#\n# Channel 0 β†’ near ground\n# Channel 1 β†’ mid-level objects\n# Channel 2 β†’ tall objects\n#\n# ------------------ IMPORTANT INTERPRETATION ------------------\n#\n# Each pixel represents:\n# 0.4m Γ— 0.4m area in real world\n#\n# Entire BEV:\n# 336 Γ— 0.4 = 134.4m coverage around the car\n#\n# Ego vehicle is always at center:\n# (168, 168)\n#\n# Each pixel is NOT just 2D β€” it is a vertical column:\n#\n# ↑ height\n# β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”\n# β”‚ Channel 2 β”‚ (1.0 β†’ 2.5m) β†’ tall objects\n# β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n# β”‚ Channel 1 β”‚ (-0.5 β†’ 1.0m) β†’ cars, pedestrians\n# β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n# β”‚ Channel 0 β”‚ (-2.0 β†’ -0.5m) β†’ ground\n# β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜\n#\n# ------------------ WHAT COLORS MEAN ------------------\n#\n# In ALL BEV channels:\n# Pixel value = LiDAR point density (NOT height, NOT class)\n#\n# Channel 0 (RdYlGn_r):\n# Green β†’ dense ground points\n# Yellow β†’ medium\n# Red β†’ sparse\n#\n# Channel 1 (hot):\n# Black β†’ no points\n# Red β†’ some points\n# Yellow/White β†’ dense (objects)\n#\n# Channel 2 (gray):\n# Black β†’ empty\n# White β†’ tall structures\n#\n# Interpretation:\n# Bright regions = objects\n# Dark regions = empty space\n\n\n# ============================================================\n# STEP 4: VISUALIZE TARGET MASK\n# ============================================================\n# Target is a segmentation map:\n#\n# 0 = background\n# 1 = car\n# 2 = motorcycle\n# ...\n#\n# ------------------ IMPORTANT ------------------\n#\n# Here, pixel values DO NOT represent density\n#\n# Instead:\n# Pixel value = object class at that location\n#\n# So:\n# BEV β†’ \"what geometry exists?\"\n# TARGET β†’ \"what object is here?\"\n#\n# Example:\n# Car present β†’\n# BEV Channel 1 = bright (many points)\n# TARGET = 1 (car)\n\n\n# ============================================================\n# STEP 5: DATA LOADER CHECK\n# ============================================================\n# DataLoader loads multiple samples at once (batching)\n\n# loader = DataLoader(dataset, batch_size=4, shuffle=True)\n\n# bev_batch, target_batch, tokens_batch = next(iter(loader))\n\n# Expected:\n# bev_batch β†’ (4, 3, 336, 336)\n# target_batch β†’ (4, 336, 336)\n\n# This is what the model actually receives during training\n\n\n# ============================================================\n# STEP 6: FULL SANITY CHECK (VERY IMPORTANT)\n# ============================================================\n# This step verifies alignment between:\n#\n# - BEV representation (our pipeline)\n# - Target mask (our labels)\n# - Camera images (real-world view)\n# - SDK LiDAR rendering (ground truth reference)\n#\n# If everything is correct:\n# - Objects in BEV match camera views\n# - Target mask aligns with BEV features\n# - Our BEV looks similar to SDK BEV\n\n\n# ============================================================\n# FINAL GOAL OF THIS CELL\n# ============================================================\n# Ensure:\n#\n# Input (BEV) β†’ correctly represents LiDAR geometry\n# Target β†’ correctly represents object labels\n#\n# So the model can learn:\n#\n# \"Patterns of LiDAR point density β†’ object classes\"\n\n\n# ============================================================\n# ONE-LINE SUMMARY (VIVA READY)\n# ============================================================\n# \"This step validates the dataset pipeline by checking tensor\n# shapes, visualizing BEV channels and target masks, and ensuring\n# alignment between LiDAR data and labels. Each pixel represents\n# a 0.4m Γ— 0.4m region, where BEV channels encode point density\n# at different heights and the target mask encodes object classes.\"\n# ============================================================","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:14:43.812371Z","iopub.execute_input":"2026-04-06T18:14:43.812782Z","iopub.status.idle":"2026-04-06T18:14:43.820304Z","shell.execute_reply.started":"2026-04-06T18:14:43.812606Z","shell.execute_reply":"2026-04-06T18:14:43.819387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# PHASE 2.5 β€” REPRESENTATION ANALYSIS\n# ============================================================\n# This section quantifies the quality of our Bird's Eye View (BEV).\n# We measure how different voxel sizes and sweep counts affect \n# \"Data Sparsity\" and \"Object Concentration.\"\n# ============================================================\n\nimport numpy as np\nfrom tqdm import tqdm\n\ndef analyze_bev_representation(dataset, num_samples=50):\n \"\"\"\n Computes statistical metrics for the BEV representation.\n Returns sparsity, points per voxel, and object-wise statistics.\n \"\"\"\n total_voxels = 0\n non_empty_voxels = 0\n total_points = 0\n object_pixel_counts = []\n object_point_counts = []\n\n for i in tqdm(range(min(num_samples, len(dataset))), desc=\"Analyzing BEV\"):\n bev, target, _ = dataset[i]\n bev_np = bev.numpy() # (3, H, W)\n\n # 1. Voxel Density Analysis\n # combine height channels to see total spatial occupancy\n voxel_sum = bev_np.sum(axis=0) \n non_empty = voxel_sum > 0\n non_empty_voxels += np.sum(non_empty)\n total_voxels += voxel_sum.size\n total_points += voxel_sum.sum()\n\n # 2. Object-wise Statistical Analysis\n target_np = target.numpy()\n object_ids = np.unique(target_np)\n object_ids = object_ids[object_ids != 0] # remove background\n\n for obj_id in object_ids:\n mask = (target_np == obj_id)\n # Area of the object in pixel-space\n object_pixel_counts.append(np.sum(mask))\n # Number of laser points successfully \"captured\" by this object\n object_point_counts.append(voxel_sum[mask].sum())\n\n sparsity = 1 - (non_empty_voxels / total_voxels)\n avg_points_per_voxel = total_points / (non_empty_voxels + 1e-6)\n avg_pixels_per_object = np.mean(object_pixel_counts) if object_pixel_counts else 0\n avg_points_per_object = np.mean(object_point_counts) if object_point_counts else 0\n\n print(\"\\n\" + \"=\"*50)\n print(\" REPRESENTATION ANALYSIS RESULTS\")\n print(\"=\"*50)\n print(f\"Sparsity (% empty voxels) : {sparsity*100:.2f}%\")\n print(f\"Avg points per occupied voxel : {avg_points_per_voxel:.2f}\")\n print(f\"Avg pixels per object : {avg_pixels_per_object:.2f}\")\n print(f\"Avg points per object : {avg_points_per_object:.2f}\")\n print(\"=\"*50 + \"\\n\")\n\n return {\n \"sparsity\": sparsity,\n \"points_per_voxel\": avg_points_per_voxel,\n \"pixels_per_object\": avg_pixels_per_object,\n \"points_per_object\": avg_points_per_object\n }\n\n# --- TEST 1: VOXEL SIZE SENSITIVITY ---\n# We compare high-res vs. low-res spatial grids.\nfor voxel_size in [(0.2,0.2,1.5), (0.4,0.4,1.5), (0.6,0.6,1.5)]:\n cfg.VOXEL_SIZE = voxel_size\n dataset = LyftBEVDataset(lyft, sample_tokens, cfg)\n print(f\"\\n[STRESS TEST] Testing Voxel Size: {voxel_size}\")\n analyze_bev_representation(dataset)\n\n# --- TEST 2: TEMPORAL SWEEP SENSITIVITY ---\n# We compare single-frame snapshots vs. multi-frame accumulations.\nfor sweeps in [1, 3, 5]:\n cfg.NUM_SWEEPS = sweeps\n dataset = LyftBEVDataset(lyft, sample_tokens, cfg)\n print(f\"\\n[STRESS TEST] Testing Accumulation Sweeps: {sweeps}\")\n analyze_bev_representation(dataset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:14:43.821769Z","iopub.execute_input":"2026-04-06T18:14:43.822177Z","iopub.status.idle":"2026-04-06T18:15:42.685578Z","shell.execute_reply.started":"2026-04-06T18:14:43.822132Z","shell.execute_reply":"2026-04-06T18:15:42.684795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ANALYSIS INSIGHTS: VOXELIZATION & TEMPORAL STACKING\n# ============================================================\n\n# 1. THE VOXEL SIZE TRADE-OFF (Spatial Detail vs. Cost)\n# ------------------------------------------------------------\n# Observations:\n# Smaller voxels (0.2m) increase \"Sparsity\" (>98% empty space).\n# Larger voxels (0.6m) increase \"Points per occupied voxel.\"\n#\n# Technical Insight:\n# Smaller voxels preserve the fine-grained geometry of \n# pedestrians and bicycles, which is critical for classification.\n# However, they significantly increase the spatial dimensions \n# of the input tensor, leading to higher GPU memory usage and \n# slower training times. \n# Conversely, larger voxels produce a \"compact\" representation \n# but suffer from 'Quantization Error'β€”where small objects \n# vanish into a single pixel, making them undetectable.\n\n# \n\n# 2. THE MULTI-SWEEP DILEMMA (Density vs. Misalignment)\n# ------------------------------------------------------------\n# Observations:\n# Increasing sweeps (1 -> 5) drastically improves \"Voxel Density.\"\n# However, it reduces the \"Point concentration\" within objects.\n#\n# Technical Insight:\n# While stacking multiple sweeps solves the problem of \"sparse \n# point clouds\" at long ranges, it introduces \"Temporal \n# Misalignment\" (Ghosting). \n# Because other vehicles move between the 0.05s gaps of each \n# sweep, stacking them creates a \"trail\" of points. \n# Since our target bounding boxes only represent the object's \n# position at the CURRENT millisecond, many accumulated points \n# actually fall OUTSIDE the ground-truth box, creating noise \n# that confuses the model's localization accuracy.\n\n# \n\n# ============================================================\n# FINAL CONCLUSION FOR CONFIGURATION:\n# ------------------------------------------------------------\n# Based on this analysis, we select:\n# VOXEL_SIZE = 0.4m (The balance point for speed and detail)\n# NUM_SWEEPS = 1 (To maintain zero temporal misalignment)\n# ============================================================","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:15:42.687201Z","iopub.execute_input":"2026-04-06T18:15:42.687512Z","iopub.status.idle":"2026-04-06T18:15:42.693532Z","shell.execute_reply.started":"2026-04-06T18:15:42.687453Z","shell.execute_reply":"2026-04-06T18:15:42.692412Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Smaller voxel sizes preserve spatial detail but increase computational cost,\nwhile larger voxel sizes produce more compact but less informative representations.\n\nWhile increasing the number of sweeps improves voxel density,\nit introduces temporal misalignment effects that reduce\npoint concentration within ground-truth object regions.","metadata":{}},{"cell_type":"markdown","source":"# Phase 3: Building the UNet model","metadata":{}},{"cell_type":"code","source":"# Phase 3: Building the UNet model\n# ============================================================\n# U-NET β€” UPDATED FOR MULTI-CLASS CROSS-ENTROPY\n# ============================================================\n# Input : (B, 3, 336, 336) β€” batch of voxel BEV images\n# Output : (B, 10, 336, 336) β€” raw logits per pixel\n#\n# 10 output channels = background + 9 object classes\n# Channel 0 = background probability\n# Channel 1 = car, 2 = motorcycle, ..., 9 = emergency_vehicle\n#\n# This is the same U-Net architecture as before but:\n# out_channels changed from 9 β†’ 10 (background class added)\n# No sigmoid at output β€” cross_entropy applies softmax internally\n#\n# Architecture:\n# Input (3ch)\n# β”‚\n# Encoder (going down β€” shrinking spatial size)\n# β”‚ down1: 3 β†’ 16 channels, 336Γ—336\n# β”‚ down2: 16 β†’ 32 channels, 168Γ—168\n# β”‚ down3: 32 β†’ 64 channels, 84Γ—84\n# β”‚\n# Bottleneck\n# β”‚ center: 64 β†’ 128 channels, 42Γ—42\n# β”‚\n# Decoder (going up β€” restoring spatial size)\n# β”‚ up3: 128β†’64, concat with down3 β†’ 64ch, 84Γ—84\n# β”‚ up2: 64β†’32, concat with down2 β†’ 32ch, 168Γ—168\n# β”‚ up1: 32β†’16, concat with down1 β†’ 16ch, 336Γ—336\n# β”‚\n# Output head: 16 β†’ 10 channels β†’ raw logits\n# Softmax applied by F.cross_entropy during training\n# Softmax applied manually during inference\n# ============================================================\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass ConvBlock(nn.Module):\n \"\"\"\n Two conv layers back to back β€” the basic building block.\n Each conv: (in_ch β†’ out_ch), kernel 3Γ—3, padding=1\n Padding=1 keeps spatial size the same after each conv.\n BatchNorm stabilizes training. ReLU is the activation.\n \"\"\"\n def __init__(self, in_ch, out_ch):\n super().__init__()\n self.block = nn.Sequential(\n nn.Conv2d(in_ch, out_ch, kernel_size=3, padding=1),\n nn.BatchNorm2d(out_ch),\n nn.ReLU(inplace=True),\n nn.Conv2d(out_ch, out_ch, kernel_size=3, padding=1),\n nn.BatchNorm2d(out_ch),\n nn.ReLU(inplace=True)\n )\n\n def forward(self, x):\n return self.block(x)\n\n\nclass SimpleUNet(nn.Module):\n def __init__(self, in_channels=3, out_channels=10):\n super().__init__()\n\n # --- Encoder ---\n # Gradually extracts high-level spatial features while reducing resolution\n self.down1 = ConvBlock(in_channels, 16) # 336Γ—336\n self.pool1 = nn.MaxPool2d(2) # β†’ 168Γ—168\n self.down2 = ConvBlock(16, 32) # 168Γ—168\n self.pool2 = nn.MaxPool2d(2) # β†’ 84Γ—84\n self.down3 = ConvBlock(32, 64) # 84Γ—84\n self.pool3 = nn.MaxPool2d(2) # β†’ 42Γ—42\n\n # --- Bottleneck ---\n # Deepest part of the network, captures global scene context\n self.center = ConvBlock(64, 128) # 42Γ—42\n\n # --- Decoder ---\n # Upsamples features back to original resolution using skip connections\n # to preserve fine-grained LiDAR geometry.\n self.up3 = nn.ConvTranspose2d(128, 64, kernel_size=2, stride=2)\n self.dec3 = ConvBlock(128, 64)\n self.up2 = nn.ConvTranspose2d(64, 32, kernel_size=2, stride=2)\n self.dec2 = ConvBlock(64, 32)\n self.up1 = nn.ConvTranspose2d(32, 16, kernel_size=2, stride=2)\n self.dec1 = ConvBlock(32, 16)\n\n # --- Output head ---\n # Maps the 16 feature channels to our 10 class logits.\n self.final = nn.Conv2d(16, out_channels, kernel_size=1)\n\n def forward(self, x):\n d1 = self.down1(x)\n d2 = self.down2(self.pool1(d1))\n d3 = self.down3(self.pool2(d2))\n c = self.center(self.pool3(d3))\n\n # Upsampling with Skip Connections (Concatenation)\n u3 = self.dec3(torch.cat([self.up3(c), d3], dim=1))\n u2 = self.dec2(torch.cat([self.up2(u3), d2], dim=1))\n u1 = self.dec1(torch.cat([self.up1(u2), d1], dim=1))\n\n return self.final(u1) # (B, 10, 336, 336) raw logits\n\n\n# ============================================================\n# ARCHITECTURE SANITY CHECK\n# ============================================================\n# Verifies tensor flow and parameter count.\n# ============================================================\nmodel_check = SimpleUNet(in_channels=3, out_channels=10)\ntotal_params = sum(p.numel() for p in model_check.parameters()\n if p.requires_grad)\n\n# Create a dummy BEV tensor (Batch=2, Channels=3, H=336, W=336)\ndummy_input = torch.zeros(2, 3, 336, 336)\ndummy_output = model_check(dummy_input)\n\nprint(\"=\"*60)\nprint(\"U-NET ARCHITECTURE CHECK\")\nprint(\"=\"*60)\nprint(f\"Total Trainable Parameters : {total_params:,}\")\nprint(f\"Input Shape (B, C, H, W) : {dummy_input.shape}\")\nprint(f\"Output Shape (B, C, H, W) : {dummy_output.shape}\")\nprint(f\" -> {dummy_output.shape[1]} channels = Background + 9 object classes\")\nprint(f\"Logit Range : min={dummy_output.min():.3f}, max={dummy_output.max():.3f}\")\nprint(\"=\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:23:54.070251Z","iopub.execute_input":"2026-04-06T18:23:54.071142Z","iopub.status.idle":"2026-04-06T18:23:54.690456Z","shell.execute_reply.started":"2026-04-06T18:23:54.070574Z","shell.execute_reply":"2026-04-06T18:23:54.689448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# TRAINING LOOP\n# ============================================================\n# Key changes for the Master Version:\n#\n# 1. Loss Function: Multi-class Cross-Entropy with 0.2 Background \n# weight to force the model to focus on objects.\n#\n# 2. Scene-Based Split: 160 train / 20 val to prevent data leakage.\n#\n# 3. Hardware Alignment: num_workers=4 to match Kaggle's CPU cores.\n#\n# 4. State Reset: Explicitly setting sweeps/voxels to ensure training\n# matches our Phase 2.5 conclusions.\n# ============================================================\n\nfrom tqdm import tqdm\nimport time\nimport os\n\n# --- HARDWARE & HYPERPARAMS ---\nEPOCHS = 10\nBATCH_SIZE = 32\nLR = 1e-3\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ntorch.backends.cudnn.benchmark = True\n\n# --------------------------------------------------------\n# 1. STATE LOCK: ENSURE TRAINING PARAMETERS\n# --------------------------------------------------------\n# Resetting these here prevents pollution from the \n# Representation Analysis tests in Phase 2.5.\n# --------------------------------------------------------\ncfg.NUM_SWEEPS = 1\ncfg.VOXEL_SIZE = (0.4, 0.4, 1.5)\nprint(f\"βœ… Training State Locked: Sweeps={cfg.NUM_SWEEPS}, Voxel={cfg.VOXEL_SIZE}\")\n\n# --------------------------------------------------------\n# 2. SCENE-BASED TRAIN/VAL SPLIT\n# --------------------------------------------------------\nall_scenes = lyft.scene\ntrain_scenes = all_scenes[:160]\nval_scenes = all_scenes[160:]\n\ndef collect_tokens_from_scenes(scenes, lyft_sdk):\n tokens = []\n for scene in scenes:\n token = scene['first_sample_token']\n while token:\n tokens.append(token)\n token = lyft_sdk.get('sample', token)['next']\n return tokens\n\ntrain_tokens = collect_tokens_from_scenes(train_scenes, lyft)\nval_tokens = collect_tokens_from_scenes(val_scenes, lyft)\n\n# --- DataLoaders (Optimized Workers) ---\ntrain_loader = DataLoader(\n LyftBEVDataset(lyft, train_tokens, cfg),\n batch_size = BATCH_SIZE,\n shuffle = True,\n num_workers = 4, # FIXED: Match Kaggle CPU for speed\n pin_memory = True\n)\nval_loader = DataLoader(\n LyftBEVDataset(lyft, val_tokens, cfg),\n batch_size = BATCH_SIZE,\n shuffle = False,\n num_workers = 4,\n pin_memory = True\n)\n\n# --- Model & Optimizer ---\nmodel = SimpleUNet(in_channels=3, out_channels=10)\nif torch.cuda.device_count() > 1:\n print(f\"βœ… Using {torch.cuda.device_count()} GPUs\")\n model = torch.nn.DataParallel(model)\nmodel = model.to(device)\n\n# --- Weighted Cross Entropy ---\nclass_weights = torch.from_numpy(\n np.array([0.2] + [1.0] * len(CLASSES), dtype=np.float32)\n).to(device)\n\noptimizer = torch.optim.Adam(model.parameters(), lr=LR)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n optimizer, mode='min', factor=0.5, patience=2, verbose=True\n)\n\nCHECKPOINT_DIR = '/kaggle/working/checkpoints'\nos.makedirs(CHECKPOINT_DIR, exist_ok=True)\n\nhistory = {'train_loss': [], 'val_loss': []}\n\nprint(f\"\\nStarting training on {device}...\")\nprint(f\"Train samples : {len(train_tokens):,}\")\nprint(f\"Val samples : {len(val_tokens):,}\\n\")\n\n# --------------------------------------------------------\n# 3. MAIN TRAINING ENGINE\n# --------------------------------------------------------\nfor epoch in range(1, EPOCHS + 1):\n model.train()\n epoch_loss = 0\n start_time = time.time()\n\n # Progress bar for the train batch\n pbar = tqdm(train_loader, desc=f\"Epoch {epoch}/{EPOCHS} [Train]\")\n for bev_batch, target_batch, _ in pbar:\n bev_batch = bev_batch.to(device)\n target_batch = target_batch.to(device)\n\n # Forward\n predictions = model(bev_batch)\n loss = F.cross_entropy(predictions, target_batch, weight=class_weights)\n\n # Backward\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n epoch_loss += loss.item()\n pbar.set_postfix({'loss': f'{loss.item():.4f}'})\n\n avg_train_loss = epoch_loss / len(train_loader)\n\n # ---- VALIDATION PHASE ----\n model.eval()\n val_loss = 0\n with torch.no_grad():\n for bev_batch, target_batch, _ in tqdm(val_loader, desc=f\"Epoch {epoch}/{EPOCHS} [Val]\"):\n bev_batch = bev_batch.to(device)\n target_batch = target_batch.to(device)\n preds = model(bev_batch)\n val_loss += F.cross_entropy(preds, target_batch, weight=class_weights).item()\n\n avg_val_loss = val_loss / len(val_loader)\n history['train_loss'].append(avg_train_loss)\n history['val_loss'].append(avg_val_loss)\n\n # Step Scheduler\n scheduler.step(avg_val_loss)\n \n # Save Checkpoint\n ckpt_path = os.path.join(CHECKPOINT_DIR, f'unet_epoch_{epoch}.pth')\n torch.save(model.state_dict(), ckpt_path)\n\n elapsed = time.time() - start_time\n print(f\"Summary β€” Train Loss: {avg_train_loss:.4f} | Val Loss: {avg_val_loss:.4f} | Time: {elapsed:.1f}s\")\n print(f\"Saved: {ckpt_path}\\n\")\n\nprint(\"βœ… Masterpiece Training Complete!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T18:24:41.789551Z","iopub.execute_input":"2026-04-06T18:24:41.790026Z","iopub.status.idle":"2026-04-06T19:21:44.088937Z","shell.execute_reply.started":"2026-04-06T18:24:41.789785Z","shell.execute_reply":"2026-04-06T19:21:44.087849Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# TRAINING CURVES\n# ============================================================\n# This cell visualizes the model's learning progress.\n# It plots Training vs. Validation loss to check for:\n# 1. Convergence (Is the loss actually going down?)\n# 2. Overfitting (Is the validation loss rising while train loss drops?)\n# ============================================================\n\nimport matplotlib.pyplot as plt\n\n# Check if history exists and has data\nif 'history' in locals() and len(history.get('train_loss', [])) > 0:\n \n fig, ax = plt.subplots(figsize=(10, 6))\n epochs_x = range(1, len(history['train_loss']) + 1)\n\n # 1. Plot Training Loss\n ax.plot(epochs_x, history['train_loss'], 'b-o',\n linewidth=2, markersize=5, label='Train Loss (Cross-Entropy)')\n \n # 2. Plot Validation Loss\n ax.plot(epochs_x, history['val_loss'], 'r-o',\n linewidth=2, markersize=5, label='Val Loss (Cross-Entropy)')\n\n # Styling the Report Chart\n ax.set_title('Training History β€” Multi-Class Voxel Model\\n'\n 'Gap = Overfitting Indicator | Both dropping = Healthy Generalization',\n fontsize=12, fontweight='bold')\n ax.set_xlabel('Epoch', fontsize=10)\n ax.set_ylabel('Loss (Weighted Cross-Entropy)', fontsize=10)\n ax.legend(fontsize=10)\n ax.grid(True, alpha=0.3, linestyle='--')\n\n # Add markers for the best epoch\n best_val_loss = min(history['val_loss'])\n best_epoch_idx = history['val_loss'].index(best_val_loss)\n ax.annotate(f'Best: {best_val_loss:.4f}', \n xy=(best_epoch_idx + 1, best_val_loss), \n xytext=(best_epoch_idx + 1.5, best_val_loss + 0.05),\n arrowprops=dict(facecolor='black', shrink=0.05, width=1, headwidth=5))\n\n plt.tight_layout()\n\n # 🚨 SAVING THE PLOT TO DISK 🚨\n # This file can be downloaded directly from the Kaggle /working/ folder for your report.\n curve_save_path = '/kaggle/working/training_curves.png'\n plt.savefig(curve_save_path, bbox_inches='tight', facecolor='white', dpi=300)\n print(f\"βœ… Saved high-res training curves to: {curve_save_path}\")\n\n plt.show()\n\n # Final Summary Printout\n best_epoch = best_epoch_idx + 1\n print(f\"\\n{'='*40}\")\n print(f\" FINAL TRAINING SUMMARY\")\n print(f\"{'='*40}\")\n print(f\"Final Train Loss : {history['train_loss'][-1]:.4f}\")\n print(f\"Final Val Loss : {history['val_loss'][-1]:.4f}\")\n print(f\"Best Val Loss : {best_val_loss:.4f} at Epoch {best_epoch}\")\n print(f\"{'='*40}\")\n print(f\"πŸ‘‰ Use 'unet_epoch_{best_epoch}.pth' for Phase 4 Inference.\")\n\nelse:\n print(\"⚠️ No training history found. Please run the training loop cell first.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T19:21:44.095224Z","iopub.execute_input":"2026-04-06T19:21:44.095615Z","iopub.status.idle":"2026-04-06T19:21:45.509724Z","shell.execute_reply.started":"2026-04-06T19:21:44.095447Z","shell.execute_reply":"2026-04-06T19:21:45.508588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ANALYSIS: INTERPRETING THE TRAINING CURVES\n# ============================================================\n\n# 1. LOSS CONVERGENCE\n# ------------------------------------------------------------\n# The \"Cross-Entropy\" loss measures the mathematical distance \n# between the model's predicted class probabilities and the \n# ground-truth target mask. \n# A steady decline in both blue (Train) and red (Val) lines \n# indicates that the U-Net is successfully learning the \n# geometric features of the 3-channel voxel input.\n\n\n# 2. THE OVERFITTING GAP\n# ------------------------------------------------------------\n# If the Blue line continues to drop while the Red line stays \n# flat or begins to rise, the model is \"Overfitting.\" \n# This means it is memorizing specific LiDAR frames in the \n# training set rather than learning general patterns of how \n# cars and pedestrians look. \n# By splitting our data by SCENE instead of by FRAME, we \n# minimize this risk, as the model is forced to generalize \n# to entirely new street layouts.\n\n\n# 3. EARLY STOPPING & MODEL SELECTION\n# ------------------------------------------------------------\n# In deep learning, the \"Final Epoch\" is not always the \"Best \n# Epoch.\" We use the epoch with the absolute lowest Validation \n# Loss for our final inference. \n# This ensures we utilize the version of the brain that \n# performed best on data it had never seen during backpropagation.\n\n\n# ============================================================\n# ONE-LINE SUMMARY\n# ============================================================\n# \"The training history validates the model's ability to \n# generalize, using Cross-Entropy loss as a proxy for \n# segmentation accuracy, with the optimal weights selected \n# based on minimum validation error.\"\n# ============================================================","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Phase 4 : Inference + Post Processing + GIF Demo + Testing","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# PHASE 4 β€” CELL 1: THE BRAIN LOADER\n# ============================================================\nimport os, glob, cv2, torch, numpy as np, matplotlib.pyplot as plt\nfrom scipy.spatial.transform import Rotation as R\nfrom PIL import Image as PILImage\nfrom pyquaternion import Quaternion\nfrom lyft_dataset_sdk.utils.geometry_utils import view_points, transform_matrix\nfrom lyft_dataset_sdk.utils.data_classes import Box\n\n# 1. Physics Priors\nCLASS_HEIGHTS = {'car': 1.72, 'pedestrian': 1.78, 'animal': 0.51, 'other_vehicle': 3.23, \n 'bus': 3.44, 'motorcycle': 1.59, 'truck': 3.44, 'emergency_vehicle': 2.39, 'bicycle': 1.44}\n\nCLASS_COLORS = {'car': ('orange','orange','orange'), 'pedestrian': ('cyan','cyan','cyan'), 'bus': ('magenta','magenta','magenta'),\n 'truck': ('yellow','yellow','yellow'), 'bicycle': ('lime','lime','lime'), 'motorcycle': ('red','red','red')}\n\n# 2. Hunting for Weights\nprint(\"πŸ” Searching for .pth files...\")\nckpt_path = None\nif 'history' in locals() and len(history.get('val_loss', [])) > 0:\n best_epoch = history['val_loss'].index(min(history['val_loss'])) + 1\n ckpt_path = f'/kaggle/working/checkpoints/unet_epoch_{best_epoch}.pth'\nelif glob.glob('/kaggle/working/checkpoints/*.pth'):\n ckpt_path = max(glob.glob('/kaggle/working/checkpoints/*.pth'), key=os.path.getctime)\nelse:\n input_paths = glob.glob('/kaggle/input/**/*.pth', recursive=True)\n if input_paths: ckpt_path = input_paths[0]\n\nif not ckpt_path: raise FileNotFoundError(\"🚨 NO WEIGHTS FOUND. Run training first!\")\nprint(f\"βœ… Found weights: {ckpt_path}\")\n\n# 3. Model Restore\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nif 'model' not in locals():\n model = SimpleUNet(in_channels=3, out_channels=10)\n if torch.cuda.device_count() > 1: model = torch.nn.DataParallel(model)\n model = model.to(device)\n\nstate_dict = torch.load(ckpt_path, map_location=device)\nclean_dict = {k.replace('module.', ''): v for k, v in state_dict.items()}\nif isinstance(model, torch.nn.DataParallel): model.module.load_state_dict(clean_dict)\nelse: model.load_state_dict(clean_dict)\nmodel.eval()\nprint(\"🧠 AI Brain Synced.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T20:14:17.577782Z","iopub.execute_input":"2026-04-06T20:14:17.578079Z","iopub.status.idle":"2026-04-06T20:14:17.605483Z","shell.execute_reply.started":"2026-04-06T20:14:17.578038Z","shell.execute_reply":"2026-04-06T20:14:17.604303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# PHASE 4 β€” CELL 2: THE SCIENTIFIC DASHBOARD (ANTI-GHOST)\n# ============================================================\nMORPH_KERNEL = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))\nBACKGROUND_THRESHOLD = int(0.85 * 255)\n\n# 🚨 FIX: Ensure INSPECT_TOKENS exists\nif 'val_tokens' in locals():\n INSPECT_TOKENS = val_tokens[:4]\nelse:\n print(\"βš™οΈ Fallback: Generating new inspection tokens...\")\n fallback_scene = lyft.scene[0]\n t = fallback_scene['first_sample_token']\n INSPECT_TOKENS = []\n for _ in range(4):\n if t: INSPECT_TOKENS.append(t); t = lyft.get('sample', t)['next']\n\nfor frame_idx, token in enumerate(INSPECT_TOKENS):\n temp_ds = LyftBEVDataset(lyft, [token], cfg)\n bev, target, _ = temp_ds[0]\n bev_np = bev.numpy().transpose(1, 2, 0)\n \n with torch.no_grad():\n preds = model(bev.unsqueeze(0).to(device))\n probs = torch.softmax(preds, dim=1).squeeze().cpu().numpy()\n\n # Denoising & Contour Detection\n pred_uint8 = np.round(probs * 255).astype(np.uint8)\n pred_non_bg = 255 - pred_uint8[0]\n opened = cv2.morphologyEx((pred_non_bg > BACKGROUND_THRESHOLD).astype(np.uint8), cv2.MORPH_OPEN, MORPH_KERNEL)\n contours, _ = cv2.findContours(opened, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n \n sample = lyft.get('sample', token); lidar_token = sample['data']['LIDAR_TOP']\n sd = lyft.get('sample_data', lidar_token); ego = lyft.get('ego_pose', sd['ego_pose_token'])\n \n # Honest Counter\n gt_boxes_raw = lyft.get_boxes(lidar_token); move_boxes_to_car_space(gt_boxes_raw, ego)\n boxes_in_view = sum(1 for b in gt_boxes_raw if abs(b.center[0]) < 67.2 and abs(b.center[1]) < 67.2)\n\n # 3D Reconstruction with Strict Filters\n pred_boxes_3d, pred_classes = [], []\n g_from_car = transform_matrix(ego['translation'], Quaternion(ego['rotation']), inverse=False)\n c_from_v = np.linalg.inv(create_transformation_matrix_to_voxel_space(cfg.BEV_SHAPE, cfg.VOXEL_SIZE, (0, 0, cfg.Z_OFFSET)))\n g_from_v = np.dot(g_from_car, c_from_v)\n\n for cnt in contours:\n area = cv2.contourArea(cnt)\n if area < 15: continue # Ignore tiny speckle noise\n \n rect = cv2.minAreaRect(cnt); box_pts = cv2.boxPoints(rect)\n w_px, l_px = rect[1]\n \n # 🚨 FILTER 1: Solidity (Reject hollow smudges)\n if area / (w_px * l_px + 1e-6) < 0.45: continue \n \n # 🚨 FILTER 2: Aspect Ratio (Reject long thin needles/ghosts)\n if max(w_px, l_px) / (min(w_px, l_px) + 1e-6) > 6.0: continue\n \n cx, cy = np.int0(np.mean(box_pts, axis=0)).clip(0, cfg.GRID_W-1), np.int0(np.mean(box_pts, axis=0)).clip(0, cfg.GRID_H-1)\n if pred_uint8[1:, cy[1], cx[0]].max() < int(0.40 * 255): continue\n \n cls_name = CLASSES[np.argmax(pred_uint8[1:, cy[1], cx[0]])]; h = CLASS_HEIGHTS[cls_name]\n pts_w = transform_points(np.vstack([box_pts.transpose(1, 0), np.zeros((1, 4))]), g_from_v)\n pts_w[2, :] = ego['translation'][2] \n center = pts_w.transpose(1, 0).mean(axis=0); center[2] += (h / 2)\n \n e1, e2 = np.linalg.norm(pts_w[:,0]-pts_w[:,1]), np.linalg.norm(pts_w[:,1]-pts_w[:,2])\n if e1 > e2: v = pts_w[:3,0]-pts_w[:3,1]\n else: v = pts_w[:3,1]-pts_w[:3,2]\n \n v /= (np.linalg.norm(v) + 1e-6)\n rot = np.array([[v[0], -v[1], 0], [v[1], v[0], 0], [0, 0, 1]])\n try: quat = R.from_matrix(rot).as_quat()[[3,0,1,2]]\n except: quat = R.from_dcm(rot).as_quat()[[3,0,1,2]]\n \n pred_boxes_3d.append(Box(center=list(center), size=[np.clip(min(e1,e2)/cfg.BOX_SCALE, 0.2, 4.0), \n np.clip(max(e1,e2)/cfg.BOX_SCALE, 0.5, 12.0), h],\n orientation=Quaternion(quat), name=cls_name))\n pred_classes.append(cls_name)\n\n # Visualization\n fig = plt.figure(figsize=(26, 20))\n fig.suptitle(f'SCORE: {len(pred_boxes_3d)} / {boxes_in_view} DETECTED', fontsize=30, fontweight='bold', y=0.98)\n \n # BEV Panels\n ax1=fig.add_subplot(3,4,1); ax1.imshow(bev_np[:,:,1], cmap='hot', origin='lower'); ax1.axis('off')\n ax2=fig.add_subplot(3,4,2); ax2.imshow(opened, cmap='hot', origin='lower'); ax2.axis('off')\n ax3=fig.add_subplot(3,4,3); ax3.imshow(target.numpy(), cmap='tab10', origin='lower', vmin=0, vmax=9); ax3.axis('off')\n ax4=fig.add_subplot(3,4,4); lyft.render_sample_data(lidar_token, nsweeps=1, ax=ax4); ax4.axis('off')\n\n # Cameras\n camera_channels = ['CAM_FRONT_LEFT', 'CAM_FRONT', 'CAM_FRONT_RIGHT', 'CAM_BACK_LEFT', 'CAM_BACK', 'CAM_BACK_RIGHT']\n for i, cam_name in enumerate(camera_channels):\n ax = fig.add_subplot(3,4,i+5); cam_tok=sample['data'][cam_name]; _, _, cam_intr=lyft.get_sample_data(cam_tok); img=PILImage.open(lyft.get_sample_data_path(cam_tok))\n ax.imshow(img); ax.set_xlim(0, 1920); ax.set_ylim(1080, 0)\n for box, cls in zip(pred_boxes_3d, pred_classes):\n b=box.copy(); cam_d=lyft.get('sample_data', cam_tok); c_ego=lyft.get('ego_pose', cam_d['ego_pose_token']); cal_s=lyft.get('calibrated_sensor', cam_d['calibrated_sensor_token'])\n b.translate(-np.array(c_ego['translation'])); b.rotate(Quaternion(c_ego['rotation']).inverse); b.translate(-np.array(cal_s['translation'])); b.rotate(Quaternion(cal_s['rotation']).inverse)\n if b.corners()[2,:].min() > 0.1: b.render(ax, view=np.array(cam_intr), normalize=True, colors=CLASS_COLORS.get(cls, ('white','white','white')))\n ax.set_title(cam_name.replace('_',' '), fontsize=14, fontweight='bold'); ax.axis('off')\n plt.subplots_adjust(hspace=0.02, wspace=0.02, top=0.94); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T20:19:40.417234Z","iopub.execute_input":"2026-04-06T20:19:40.417586Z","iopub.status.idle":"2026-04-06T20:20:01.251925Z","shell.execute_reply.started":"2026-04-06T20:19:40.417538Z","shell.execute_reply":"2026-04-06T20:20:01.251064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# EVALUATION β€” IoU BENCHMARKING (VALIDATION SET)\n# ============================================================\ndef evaluate_frame(token, model, cfg, device, iou_threshold=0.5):\n # GT Extraction\n sample = lyft.get('sample', token); lidar_token = sample['data']['LIDAR_TOP']; sd = lyft.get('sample_data', lidar_token); ego = lyft.get('ego_pose', sd['ego_pose_token'])\n gt_boxes_raw = lyft.get_boxes(lidar_token); move_boxes_to_car_space(gt_boxes_raw, ego)\n gt_pixel_boxes = []\n tm = create_transformation_matrix_to_voxel_space(cfg.BEV_SHAPE, cfg.VOXEL_SIZE, (0, 0, cfg.Z_OFFSET))\n for b in gt_boxes_raw:\n if abs(b.center[0]) < 67.2 and abs(b.center[1]) < 67.2:\n pts = transform_points(b.bottom_corners(), tm).transpose(1, 0)[:, :2].astype(np.float32); gt_pixel_boxes.append(pts)\n\n # AI Prediction Extraction\n temp_ds = LyftBEVDataset(lyft, [token], cfg); bev, _, _ = temp_ds[0]\n with torch.no_grad():\n preds = model(bev.unsqueeze(0).to(device)); probs = torch.softmax(preds, dim=1).squeeze().cpu().numpy()\n binary = ((255 - np.round(probs[0]*255)) > BACKGROUND_THRESHOLD).astype(np.uint8)\n opened = cv2.morphologyEx(binary, cv2.MORPH_OPEN, MORPH_KERNEL); contours, _ = cv2.findContours(opened, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n \n pred_pixel_boxes = []\n for cnt in contours:\n if cv2.contourArea(cnt) < 10: continue\n rect = cv2.minAreaRect(cnt); pred_pixel_boxes.append(cv2.boxPoints(rect).astype(np.float32))\n\n if not gt_pixel_boxes and not pred_pixel_boxes: return 0, 0, 0\n tp = 0; matched_gt = [False] * len(gt_pixel_boxes)\n for pb in pred_pixel_boxes:\n for i, gb in enumerate(gt_pixel_boxes):\n if not matched_gt[i]:\n ret, intersect = cv2.intersectConvexConvex(pb, gb)\n inter_area = cv2.contourArea(intersect) if ret > 0 else 0.0\n union_area = cv2.contourArea(pb) + cv2.contourArea(gb) - inter_area\n if (inter_area / union_area) >= iou_threshold: tp += 1; matched_gt[i] = True; break\n return tp, len(pred_pixel_boxes) - tp, len(gt_pixel_boxes) - tp\n\nprint(\"Benchmarking Accuracy...\")\nt_tp, t_fp, t_fn = 0, 0, 0\nfor t in tqdm(val_tokens[:100]):\n try: tp, fp, fn = evaluate_frame(t, model, cfg, device); t_tp += tp; t_fp += fp; t_fn += fn\n except: continue\nprec = t_tp / (t_tp + t_fp + 1e-6); rec = t_tp / (t_tp + t_fn + 1e-6); f1 = 2*prec*rec/(prec+rec+1e-6)\nprint(f\"\\nVALIDATION METRICS (IoU@0.5) β€” Precision: {prec:.3f} | Recall: {rec:.3f} | F1: {f1:.3f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-06T20:09:15.897386Z","iopub.execute_input":"2026-04-06T20:09:15.897657Z","iopub.status.idle":"2026-04-06T20:09:33.692737Z","shell.execute_reply.started":"2026-04-06T20:09:15.897614Z","shell.execute_reply":"2026-04-06T20:09:33.691946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ============================================================\n# # PHASE A: THE VALIDATION GIF GENERATOR\n# # ============================================================\n# import os\n# import glob\n# import cv2\n# import torch\n# import numpy as np\n# import matplotlib.pyplot as plt\n# from scipy.spatial.transform import Rotation as R\n# from PIL import Image as PILImage\n# from lyft_dataset_sdk.utils.geometry_utils import view_points, transform_matrix\n# from lyft_dataset_sdk.utils.data_classes import Box\n# from pyquaternion import Quaternion\n# from tqdm import tqdm\n\n# # 1. SETUP & HYPERPARAMETERS\n# BACKGROUND_THRESHOLD = int(0.85 * 255)\n# GIF_SAVE_PATH = '/kaggle/working/self_driving_validation.gif'\n# FRAME_DIR = '/kaggle/working/gif_frames/'\n# os.makedirs(FRAME_DIR, exist_ok=True)\n\n# # Clean out any old frames if you run this multiple times\n# for f in glob.glob(os.path.join(FRAME_DIR, \"*.png\")):\n# os.remove(f)\n\n# # 2. SELECT A CONTINUOUS SCENE\n# # We will grab the very first scene from the validation set\n# my_scene = val_scenes[0]\n# curr_token = my_scene['first_sample_token']\n\n# scene_tokens = []\n# while curr_token:\n# scene_tokens.append(curr_token)\n# curr_token = lyft.get('sample', curr_token)['next']\n\n# # To save time on the first test, let's just do the first 25 frames (about 8 seconds of driving)\n# scene_tokens = scene_tokens[:25] \n# print(f\"🎬 Generating video for {len(scene_tokens)} consecutive frames...\")\n\n# # 3. CHRONOLOGICAL INFERENCE LOOP\n# for frame_idx, token in enumerate(tqdm(scene_tokens, desc=\"Rendering Frames\")):\n\n# # ---- Load BEV ----\n# temp_ds = LyftBEVDataset(lyft, [token], cfg)\n# bev, target, _ = temp_ds[0]\n# bev_np = bev.numpy().transpose(1, 2, 0)\n# target_np = target.numpy()\n\n# # ---- Run model ----\n# with torch.no_grad():\n# pred_logits = model(bev.unsqueeze(0).to(device))\n# pred_softmax = F.softmax(pred_logits, dim=1)\n# pred_cpu = pred_softmax.squeeze().cpu().numpy()\n\n# pred_uint8 = np.round(pred_cpu * 255).astype(np.uint8)\n# pred_non_bg = 255 - pred_uint8[0]\n \n# thresholded = (pred_non_bg > BACKGROUND_THRESHOLD).astype(np.uint8)\n# opened = cv2.morphologyEx(thresholded, cv2.MORPH_OPEN, MORPH_KERNEL)\n \n# contours, _ = cv2.findContours(opened, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n \n# sample = lyft.get('sample', token)\n# lidar_token = sample['data']['LIDAR_TOP']\n# lidar_data = lyft.get('sample_data', lidar_token)\n# ego_pose = lyft.get('ego_pose', lidar_data['ego_pose_token'])\n \n# global_from_car = transform_matrix(\n# ego_pose['translation'], Quaternion(ego_pose['rotation']), inverse=False\n# )\n# car_from_voxel = np.linalg.inv(create_transformation_matrix_to_voxel_space(\n# cfg.BEV_SHAPE, cfg.VOXEL_SIZE, (0, 0, cfg.Z_OFFSET)\n# ))\n# global_from_voxel = np.dot(global_from_car, car_from_voxel)\n \n# pred_boxes_3d = []\n# pred_classes = []\n \n# for cnt in contours:\n# area = cv2.contourArea(cnt)\n# if area < 10 or area > 800: continue\n \n# rect = cv2.minAreaRect(cnt)\n# box_pts = cv2.boxPoints(rect)\n \n# box_center_index = np.int0(np.mean(box_pts, axis=0))\n# cx = int(np.clip(box_center_index[0], 0, cfg.GRID_W - 1))\n# cy = int(np.clip(box_center_index[1], 0, cfg.GRID_H - 1))\n \n# class_probs = pred_uint8[1:, cy, cx]\n# best_class_idx = np.argmax(class_probs)\n# best_class_prob = class_probs[best_class_idx]\n \n# if best_class_prob < int(0.50 * 255): continue\n \n# class_name = CLASSES[best_class_idx]\n# score = float(best_class_prob) / 255.0\n# height = CLASS_HEIGHTS[class_name]\n \n# pts_2d = box_pts.transpose(1, 0)\n# pts_3d = np.vstack([pts_2d, np.zeros((1, 4))])\n \n# pts_world = transform_points(pts_3d, global_from_voxel)\n# pts_world[2, :] = ego_pose['translation'][2] \n# pts_w4 = pts_world.transpose(1, 0)\n \n# center = pts_w4.mean(axis=0)\n# center[2] = ego_pose['translation'][2] + (height / 2)\n \n# edge1_len = np.linalg.norm(pts_w4[0] - pts_w4[1])\n# edge2_len = np.linalg.norm(pts_w4[1] - pts_w4[2])\n \n# if edge1_len > edge2_len:\n# length_raw, width_raw = edge1_len, edge2_len\n# v = pts_w4[0] - pts_w4[1] \n# else:\n# length_raw, width_raw = edge2_len, edge1_len\n# v = pts_w4[1] - pts_w4[2] \n \n# v /= np.linalg.norm(v) + 1e-6\n \n# length = np.clip((length_raw / cfg.BOX_SCALE), 0.5, 12.0)\n# width = np.clip((width_raw / cfg.BOX_SCALE), 0.2, 3.0)\n \n# rot_mat = np.array([\n# [ v[0], -v[1], 0],\n# [ v[1], v[0], 0],\n# [ 0, 0, 1]\n# ])\n \n# try: r_obj = R.from_matrix(rot_mat)\n# except AttributeError: r_obj = R.from_dcm(rot_mat)\n \n# quat = r_obj.as_quat()\n# quat = quat[[3, 0, 1, 2]]\n \n# box_3d = Box(\n# center=list(center), size=[width, length, height],\n# orientation=Quaternion(quat), name=class_name, score=score\n# )\n \n# pred_boxes_3d.append(box_3d)\n# pred_classes.append(class_name)\n\n# # ---- 4. SILENT VISUALISATION ----\n# FONTSIZE = 20\n# # Create figure but don't show it (prevents notebook from lagging)\n# fig = plt.figure(figsize=(28, 30))\n# fig.suptitle(f'Validation Scene | Frame {frame_idx+1}/{len(scene_tokens)} | Detections: {len(pred_boxes_3d)}', fontsize=FONTSIZE+4, fontweight='bold')\n\n# # ... [Standard Grid Plots] ...\n# ax = fig.add_subplot(3, 4, 1)\n# ax.imshow(bev_np[:, :, 1], cmap='hot', origin='upper')\n# ax.plot(cfg.GRID_W // 2, cfg.GRID_H // 2, 'r*', markersize=14)\n# ax.set_title('Input BEV Channel 1', fontsize=FONTSIZE, fontweight='bold')\n# ax.axis('off')\n\n# ax = fig.add_subplot(3, 4, 2)\n# ax.imshow(opened, cmap='hot', origin='upper')\n# for cnt in contours:\n# rect = cv2.minAreaRect(cnt)\n# box_pts = np.int0(cv2.boxPoints(rect))\n# ax.plot(np.append(box_pts[:, 0], box_pts[0, 0]), np.append(box_pts[:, 1], box_pts[0, 1]), 'r-', linewidth=1.5)\n# ax.set_title(f'AI Object Masks', fontsize=FONTSIZE, fontweight='bold')\n# ax.axis('off')\n\n# ax = fig.add_subplot(3, 4, 3)\n# ax.imshow(target_np, cmap='tab10', origin='upper', vmin=0, vmax=9)\n# ax.set_title('Ground Truth Labels', fontsize=FONTSIZE, fontweight='bold')\n# ax.axis('off')\n\n# ax = fig.add_subplot(3, 4, 4)\n# lyft.render_sample_data(sample['data']['LIDAR_TOP'], nsweeps=1, ax=ax)\n# ax.set_title('SDK LiDAR BEV (Reference)', fontsize=FONTSIZE, fontweight='bold')\n\n# camera_channels = ['CAM_FRONT_LEFT', 'CAM_FRONT', 'CAM_FRONT_RIGHT', 'CAM_BACK_LEFT', 'CAM_BACK', 'CAM_BACK_RIGHT']\n\n# for cam_idx, cam_name in enumerate(camera_channels):\n# ax = fig.add_subplot(3, 4, cam_idx + 5)\n# cam_tok = sample['data'][cam_name]\n# cam_d = lyft.get('sample_data', cam_tok)\n# cam_ego = lyft.get('ego_pose', cam_d['ego_pose_token'])\n# cal_s = lyft.get('calibrated_sensor', cam_d['calibrated_sensor_token'])\n# _, _, cam_intr = lyft.get_sample_data(cam_tok)\n# img = np.array(PILImage.open(lyft.get_sample_data_path(cam_tok)))\n \n# ax.imshow(img)\n# ax.set_xlim(0, img.shape[1])\n# ax.set_ylim(img.shape[0], 0)\n\n# n_rendered = 0\n# for box, cls_name in zip(pred_boxes_3d, pred_classes):\n# b = box.copy()\n# b.translate(-np.array(cam_ego['translation']))\n# b.rotate(Quaternion(cam_ego['rotation']).inverse)\n# b.translate(-np.array(cal_s['translation']))\n# b.rotate(Quaternion(cal_s['rotation']).inverse)\n\n# corners_3d = b.corners()\n# corners_img = view_points(corners_3d, np.array(cam_intr), normalize=True)[:2, :]\n# in_front = corners_3d[2, :] > 0.1\n \n# in_image = ((corners_img[0, :] > 0) & (corners_img[0, :] < img.shape[1]) & \n# (corners_img[1, :] > 0) & (corners_img[1, :] < img.shape[0]))\n\n# if any(in_image) and all(in_front):\n# colors = CLASS_COLORS.get(cls_name, ('white', 'white', 'white'))\n# b.render(ax, view=np.array(cam_intr), normalize=True, colors=colors)\n# n_rendered += 1\n\n# ax.set_title(f'{cam_name}', fontsize=FONTSIZE, fontweight='bold')\n# ax.axis('off')\n\n# plt.tight_layout()\n \n# # Save the frame silently and close the figure to prevent RAM explosion!\n# frame_path = os.path.join(FRAME_DIR, f'frame_{frame_idx:03d}.png')\n# plt.savefig(frame_path, bbox_inches='tight', facecolor='white')\n# plt.close(fig) \n\n# # ============================================================\n# # 5. STITCH THE FRAMES INTO A GIF\n# # ============================================================\n# print(\"\\n🎞️ Stitching frames into an animated GIF...\")\n# image_files = sorted(glob.glob(os.path.join(FRAME_DIR, \"*.png\")))\n# frames = [PILImage.open(image) for image in image_files]\n\n# # Save as GIF. duration=250 means 4 frames per second (matching real-life LiDAR sweep rate)\n# frames[0].save(\n# GIF_SAVE_PATH,\n# format='GIF',\n# append_images=frames[1:],\n# save_all=True,\n# duration=250, \n# loop=0\n# )\n\n# print(f\"βœ… Masterpiece complete! GIF saved to: {GIF_SAVE_PATH}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ============================================================\n# # PHASE A: CINEMATIC VALIDATION GIF GENERATOR\n# # ============================================================\n# import os\n# import glob\n# import cv2\n# import torch\n# import numpy as np\n# import matplotlib.pyplot as plt\n# import matplotlib.gridspec as gridspec\n# from scipy.spatial.transform import Rotation as R\n# from PIL import Image as PILImage\n# from lyft_dataset_sdk.utils.geometry_utils import view_points, transform_matrix\n# from lyft_dataset_sdk.utils.data_classes import Box\n# from pyquaternion import Quaternion\n# from tqdm import tqdm\n\n# # 1. SETUP & HYPERPARAMETERS\n# BACKGROUND_THRESHOLD = int(0.85 * 255)\n# GIF_SAVE_PATH = '/kaggle/working/cinematic_self_driving.gif'\n# FRAME_DIR = '/kaggle/working/cinematic_frames/'\n# os.makedirs(FRAME_DIR, exist_ok=True)\n\n# # Clean out old frames\n# for f in glob.glob(os.path.join(FRAME_DIR, \"*.png\")):\n# os.remove(f)\n\n# # 2. SELECT A CONTINUOUS SCENE\n# my_scene = val_scenes[0]\n# curr_token = my_scene['first_sample_token']\n\n# scene_tokens = []\n# while curr_token:\n# scene_tokens.append(curr_token)\n# curr_token = lyft.get('sample', curr_token)['next']\n\n# # Generate the first 30 frames\n# scene_tokens = scene_tokens[:30] \n# print(f\"🎬 Generating CINEMATIC video for {len(scene_tokens)} frames...\")\n\n# # 3. CHRONOLOGICAL INFERENCE LOOP\n# for frame_idx, token in enumerate(tqdm(scene_tokens, desc=\"Rendering Frames\")):\n\n# temp_ds = LyftBEVDataset(lyft, [token], cfg)\n# bev, target, _ = temp_ds[0]\n# bev_np = bev.numpy().transpose(1, 2, 0)\n\n# with torch.no_grad():\n# pred_logits = model(bev.unsqueeze(0).to(device))\n# pred_softmax = F.softmax(pred_logits, dim=1)\n# pred_cpu = pred_softmax.squeeze().cpu().numpy()\n\n# pred_uint8 = np.round(pred_cpu * 255).astype(np.uint8)\n# pred_non_bg = 255 - pred_uint8[0]\n \n# thresholded = (pred_non_bg > BACKGROUND_THRESHOLD).astype(np.uint8)\n# opened = cv2.morphologyEx(thresholded, cv2.MORPH_OPEN, MORPH_KERNEL)\n# contours, _ = cv2.findContours(opened, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n \n# sample = lyft.get('sample', token)\n# lidar_token = sample['data']['LIDAR_TOP']\n# lidar_data = lyft.get('sample_data', lidar_token)\n# ego_pose = lyft.get('ego_pose', lidar_data['ego_pose_token'])\n \n# global_from_car = transform_matrix(\n# ego_pose['translation'], Quaternion(ego_pose['rotation']), inverse=False\n# )\n# car_from_voxel = np.linalg.inv(create_transformation_matrix_to_voxel_space(\n# cfg.BEV_SHAPE, cfg.VOXEL_SIZE, (0, 0, cfg.Z_OFFSET)\n# ))\n# global_from_voxel = np.dot(global_from_car, car_from_voxel)\n \n# pred_boxes_3d = []\n# pred_classes = []\n# pred_scores = []\n# pred_rects_2d = [] # Store for the radar view\n \n# for cnt in contours:\n# area = cv2.contourArea(cnt)\n# if area < 10 or area > 800: continue\n \n# rect = cv2.minAreaRect(cnt)\n# box_pts = cv2.boxPoints(rect)\n \n# box_center_index = np.int0(np.mean(box_pts, axis=0))\n# cx = int(np.clip(box_center_index[0], 0, cfg.GRID_W - 1))\n# cy = int(np.clip(box_center_index[1], 0, cfg.GRID_H - 1))\n \n# class_probs = pred_uint8[1:, cy, cx]\n# best_class_idx = np.argmax(class_probs)\n# best_class_prob = class_probs[best_class_idx]\n \n# if best_class_prob < int(0.50 * 255): continue\n \n# class_name = CLASSES[best_class_idx]\n# score = float(best_class_prob) / 255.0\n# height = CLASS_HEIGHTS[class_name]\n \n# pts_2d = box_pts.transpose(1, 0)\n# pts_3d = np.vstack([pts_2d, np.zeros((1, 4))])\n \n# pts_world = transform_points(pts_3d, global_from_voxel)\n# pts_world[2, :] = ego_pose['translation'][2] \n# pts_w4 = pts_world.transpose(1, 0)\n \n# center = pts_w4.mean(axis=0)\n# center[2] = ego_pose['translation'][2] + (height / 2)\n \n# edge1_len = np.linalg.norm(pts_w4[0] - pts_w4[1])\n# edge2_len = np.linalg.norm(pts_w4[1] - pts_w4[2])\n \n# if edge1_len > edge2_len:\n# length_raw, width_raw = edge1_len, edge2_len\n# v = pts_w4[0] - pts_w4[1] \n# else:\n# length_raw, width_raw = edge2_len, edge1_len\n# v = pts_w4[1] - pts_w4[2] \n \n# v /= np.linalg.norm(v) + 1e-6\n \n# length = np.clip((length_raw / cfg.BOX_SCALE), 0.5, 12.0)\n# width = np.clip((width_raw / cfg.BOX_SCALE), 0.2, 3.0)\n \n# rot_mat = np.array([\n# [ v[0], -v[1], 0],\n# [ v[1], v[0], 0],\n# [ 0, 0, 1]\n# ])\n \n# try: r_obj = R.from_matrix(rot_mat)\n# except AttributeError: r_obj = R.from_dcm(rot_mat)\n \n# quat = r_obj.as_quat()\n# quat = quat[[3, 0, 1, 2]]\n \n# box_3d = Box(\n# center=list(center), size=[width, length, height],\n# orientation=Quaternion(quat), name=class_name, score=score\n# )\n \n# pred_boxes_3d.append(box_3d)\n# pred_classes.append(class_name)\n# pred_scores.append(score)\n# pred_rects_2d.append((np.int0(box_pts), class_name))\n\n# # ============================================================\n# # 4. SILENT CINEMATIC VISUALISATION\n# # ============================================================\n# fig = plt.figure(figsize=(24, 16))\n# fig.patch.set_facecolor('black') # Pro dark mode background\n \n# # GridSpec: 2 rows, 3 columns. Top row spans all 3 columns.\n# gs = gridspec.GridSpec(2, 3, height_ratios=[1.8, 1])\n \n# ax_front = fig.add_subplot(gs[0, :])\n# ax_fl = fig.add_subplot(gs[1, 0])\n# ax_radar = fig.add_subplot(gs[1, 1])\n# ax_fr = fig.add_subplot(gs[1, 2])\n \n# # --- 4A. The Center Radar (BEV Overlay) ---\n# ax_radar.imshow(bev_np[:, :, 1], cmap='gray', origin='upper')\n# ax_radar.plot(cfg.GRID_W // 2, cfg.GRID_H // 2, 'w*', markersize=16, label='Ego Vehicle')\n \n# # Draw strictly the 2D rectangles on the radar\n# for box_pts, cls_name in pred_rects_2d:\n# color_name = CLASS_COLORS.get(cls_name, ('cyan', 'cyan', 'cyan'))[0]\n# ax_radar.plot(np.append(box_pts[:, 0], box_pts[0, 0]), \n# np.append(box_pts[:, 1], box_pts[0, 1]), \n# color=color_name, linewidth=2.5)\n \n# ax_radar.set_title(f\"AI Radar β€” {len(pred_boxes_3d)} Targets\", color='white', fontsize=16, fontweight='bold', pad=10)\n# ax_radar.axis('off')\n\n# # --- 4B. The Camera Loop ---\n# cameras_to_plot = [\n# ('CAM_FRONT', ax_front), \n# ('CAM_FRONT_LEFT', ax_fl), \n# ('CAM_FRONT_RIGHT', ax_fr)\n# ]\n\n# for cam_name, ax in cameras_to_plot:\n# cam_tok = sample['data'][cam_name]\n# cam_d = lyft.get('sample_data', cam_tok)\n# cam_ego = lyft.get('ego_pose', cam_d['ego_pose_token'])\n# cal_s = lyft.get('calibrated_sensor', cam_d['calibrated_sensor_token'])\n# _, _, cam_intr = lyft.get_sample_data(cam_tok)\n# img = np.array(PILImage.open(lyft.get_sample_data_path(cam_tok)))\n \n# ax.imshow(img)\n# ax.set_xlim(0, img.shape[1])\n# ax.set_ylim(img.shape[0], 0)\n\n# for box, cls_name, score in zip(pred_boxes_3d, pred_classes, pred_scores):\n# b = box.copy()\n# b.translate(-np.array(cam_ego['translation']))\n# b.rotate(Quaternion(cam_ego['rotation']).inverse)\n# b.translate(-np.array(cal_s['translation']))\n# b.rotate(Quaternion(cal_s['rotation']).inverse)\n\n# corners_3d = b.corners()\n# corners_img = view_points(corners_3d, np.array(cam_intr), normalize=True)[:2, :]\n# in_front = corners_3d[2, :] > 0.1\n \n# in_image = ((corners_img[0, :] > 0) & (corners_img[0, :] < img.shape[1]) & \n# (corners_img[1, :] > 0) & (corners_img[1, :] < img.shape[0]))\n\n# if any(in_image) and all(in_front):\n# colors = CLASS_COLORS.get(cls_name, ('white', 'white', 'white'))\n# b.render(ax, view=np.array(cam_intr), normalize=True, colors=colors, linewidth=3)\n \n# # 🚨 CONFIDENCE HOVER TEXT\n# # Find the highest point of the 2D projected box to hover the text over\n# min_x = np.min(corners_img[0, :])\n# min_y = np.min(corners_img[1, :])\n# min_x = max(10, min_x) # Keep text on screen\n# min_y = max(30, min_y)\n \n# # Format: \"CAR 87%\"\n# label = f\"{cls_name[:3].upper()} {score*100:.0f}%\"\n# text_color = colors[0] if isinstance(colors[0], str) else 'white'\n \n# ax.text(min_x, min_y - 8, label, color=text_color, fontsize=12, fontweight='bold',\n# bbox=dict(facecolor='black', alpha=0.7, edgecolor='none', boxstyle='round,pad=0.3'))\n\n# ax.set_title(cam_name.replace('_', ' '), color='white', fontsize=18, fontweight='bold', pad=10)\n# ax.axis('off')\n\n# plt.tight_layout(pad=3.0)\n \n# # Add an overarching dashboard title\n# fig.text(0.5, 0.96, f\"JUGAADI PERCEPTION ENGINE v5.0 | Frame {frame_idx+1}/30\", \n# ha='center', color='white', fontsize=22, fontweight='bold')\n\n# frame_path = os.path.join(FRAME_DIR, f'frame_{frame_idx:03d}.png')\n# plt.savefig(frame_path, bbox_inches='tight', facecolor=fig.get_facecolor())\n# plt.close(fig) \n\n# # ============================================================\n# # 5. STITCH THE FRAMES INTO A HIGH-RES GIF\n# # ============================================================\n# print(\"\\n🎞️ Stitching Cinematic GIF...\")\n# image_files = sorted(glob.glob(os.path.join(FRAME_DIR, \"*.png\")))\n# frames = [PILImage.open(image) for image in image_files]\n\n# # Save as GIF at 4 Frames Per Second\n# frames[0].save(\n# GIF_SAVE_PATH,\n# format='GIF',\n# append_images=frames[1:],\n# save_all=True,\n# duration=250, \n# loop=0\n# )\n\n# print(f\"βœ… Cinematic Masterpiece complete! GIF saved to: {GIF_SAVE_PATH}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ============================================================\n# # PHASE B (LITE): RANDOM UNSEEN SCENE INSPECTOR\n# # ============================================================\n# import os\n# import glob\n# import cv2\n# import torch\n# import random\n# import numpy as np\n# import matplotlib.pyplot as plt\n# import matplotlib.gridspec as gridspec\n# from scipy.spatial.transform import Rotation as R\n# from PIL import Image as PILImage\n# from lyft_dataset_sdk.utils.geometry_utils import view_points, transform_matrix\n# from lyft_dataset_sdk.utils.data_classes import Box\n# from pyquaternion import Quaternion\n\n# # 1. SETUP & HYPERPARAMETERS\n# BACKGROUND_THRESHOLD = int(0.85 * 255)\n\n# # 2. FAILSAFE MODEL LOADER\n# print(\"πŸ” Loading AI Brain...\")\n# working_checkpoints = glob.glob('/kaggle/working/checkpoints/unet_epoch_*.pth')\n# if working_checkpoints:\n# def get_epoch_number(filepath): return int(os.path.basename(filepath).split('_')[-1].split('.')[0])\n# ckpt_path = max(working_checkpoints, key=get_epoch_number)\n# else:\n# raise FileNotFoundError(\"🚨 CRITICAL: No .pth files found!\")\n\n# print(f\"βœ… Auto-locked onto checkpoint: {ckpt_path}\")\n# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# if 'model' not in locals():\n# print(\"βš™οΈ Restoring blank SimpleUNet model...\")\n# model = SimpleUNet(in_channels=3, out_channels=10)\n# if torch.cuda.device_count() > 1: model = torch.nn.DataParallel(model)\n# model = model.to(device)\n\n# state_dict = torch.load(ckpt_path, map_location=device)\n# clean_state_dict = {k.replace('module.', ''): v for k, v in state_dict.items()}\n# if isinstance(model, torch.nn.DataParallel): model.module.load_state_dict(clean_state_dict)\n# else: model.load_state_dict(clean_state_dict)\n# model.eval()\n\n# # 3. SELECT 3 RANDOM, DISCONNECTED UNSEEN FRAMES\n# print(f\"\\nπŸ“Έ Inspecting 3 completely RANDOM UNSEEN frames...\")\n\n# inspect_tokens = []\n# # Pick 3 totally different street scenes from the unseen validation data\n# # (Ensures we test different intersections, lighting, and traffic)\n# random_scenes = random.sample(val_scenes, 3)\n\n# for scene in random_scenes:\n# curr_token = scene['first_sample_token']\n \n# # Skip 5 frames deep into the scene to get right into the action\n# for _ in range(5): \n# if curr_token:\n# curr_token = lyft.get('sample', curr_token)['next']\n \n# if curr_token:\n# inspect_tokens.append(curr_token)\n\n# # 4. INFERENCE & VISUALISATION\n# for frame_idx, token in enumerate(inspect_tokens):\n\n# temp_ds = LyftBEVDataset(lyft, [token], cfg)\n# bev, _, _ = temp_ds[0]\n# bev_np = bev.numpy().transpose(1, 2, 0)\n\n# with torch.no_grad():\n# pred_logits = model(bev.unsqueeze(0).to(device))\n# pred_softmax = F.softmax(pred_logits, dim=1)\n# pred_cpu = pred_softmax.squeeze().cpu().numpy()\n\n# pred_uint8 = np.round(pred_cpu * 255).astype(np.uint8)\n# pred_non_bg = 255 - pred_uint8[0]\n \n# thresholded = (pred_non_bg > BACKGROUND_THRESHOLD).astype(np.uint8)\n# opened = cv2.morphologyEx(thresholded, cv2.MORPH_OPEN, MORPH_KERNEL)\n# contours, _ = cv2.findContours(opened, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n \n# sample = lyft.get('sample', token)\n# lidar_token = sample['data']['LIDAR_TOP']\n# lidar_data = lyft.get('sample_data', lidar_token)\n# ego_pose = lyft.get('ego_pose', lidar_data['ego_pose_token'])\n \n# global_from_car = transform_matrix(\n# ego_pose['translation'], Quaternion(ego_pose['rotation']), inverse=False\n# )\n# car_from_voxel = np.linalg.inv(create_transformation_matrix_to_voxel_space(\n# cfg.BEV_SHAPE, cfg.VOXEL_SIZE, (0, 0, cfg.Z_OFFSET)\n# ))\n# global_from_voxel = np.dot(global_from_car, car_from_voxel)\n \n# pred_boxes_3d, pred_classes, pred_scores, pred_rects_2d = [], [], [], []\n \n# for cnt in contours:\n# area = cv2.contourArea(cnt)\n# if area < 10 or area > 800: continue\n \n# rect = cv2.minAreaRect(cnt)\n# box_pts = cv2.boxPoints(rect)\n \n# box_center_index = np.int0(np.mean(box_pts, axis=0))\n# cx = int(np.clip(box_center_index[0], 0, cfg.GRID_W - 1))\n# cy = int(np.clip(box_center_index[1], 0, cfg.GRID_H - 1))\n \n# class_probs = pred_uint8[1:, cy, cx]\n# best_class_idx = np.argmax(class_probs)\n# best_class_prob = class_probs[best_class_idx]\n \n# if best_class_prob < int(0.50 * 255): continue\n \n# class_name = CLASSES[best_class_idx]\n# score = float(best_class_prob) / 255.0\n# height = CLASS_HEIGHTS[class_name]\n \n# pts_2d = box_pts.transpose(1, 0)\n# pts_3d = np.vstack([pts_2d, np.zeros((1, 4))])\n \n# pts_world = transform_points(pts_3d, global_from_voxel)\n# pts_world[2, :] = ego_pose['translation'][2] \n# pts_w4 = pts_world.transpose(1, 0)\n \n# center = pts_w4.mean(axis=0)\n# center[2] = ego_pose['translation'][2] + (height / 2)\n \n# edge1_len = np.linalg.norm(pts_w4[0] - pts_w4[1])\n# edge2_len = np.linalg.norm(pts_w4[1] - pts_w4[2])\n \n# if edge1_len > edge2_len:\n# length_raw, width_raw = edge1_len, edge2_len\n# v = pts_w4[0] - pts_w4[1] \n# else:\n# length_raw, width_raw = edge2_len, edge1_len\n# v = pts_w4[1] - pts_w4[2] \n \n# v /= np.linalg.norm(v) + 1e-6\n \n# length = np.clip((length_raw / cfg.BOX_SCALE), 0.5, 12.0)\n# width = np.clip((width_raw / cfg.BOX_SCALE), 0.2, 3.0)\n \n# rot_mat = np.array([[ v[0], -v[1], 0], [ v[1], v[0], 0], [ 0, 0, 1]])\n \n# try: r_obj = R.from_matrix(rot_mat)\n# except AttributeError: r_obj = R.from_dcm(rot_mat)\n \n# quat = r_obj.as_quat()\n# quat = quat[[3, 0, 1, 2]]\n \n# box_3d = Box(\n# center=list(center), size=[width, length, height],\n# orientation=Quaternion(quat), name=class_name, score=score\n# )\n \n# pred_boxes_3d.append(box_3d)\n# pred_classes.append(class_name)\n# pred_scores.append(score)\n# pred_rects_2d.append((np.int0(box_pts), class_name))\n\n# # --- CINEMATIC PLOT ---\n# fig = plt.figure(figsize=(24, 14))\n# fig.patch.set_facecolor('black') \n \n# gs = gridspec.GridSpec(2, 3, height_ratios=[1.8, 1])\n# ax_front = fig.add_subplot(gs[0, :])\n# ax_fl = fig.add_subplot(gs[1, 0])\n# ax_radar = fig.add_subplot(gs[1, 1])\n# ax_fr = fig.add_subplot(gs[1, 2])\n \n# ax_radar.imshow(bev_np[:, :, 1], cmap='gray', origin='upper')\n# ax_radar.plot(cfg.GRID_W // 2, cfg.GRID_H // 2, 'w*', markersize=16, label='Ego Vehicle')\n \n# for box_pts, cls_name in pred_rects_2d:\n# color_name = CLASS_COLORS.get(cls_name, ('cyan', 'cyan', 'cyan'))[0]\n# ax_radar.plot(np.append(box_pts[:, 0], box_pts[0, 0]), \n# np.append(box_pts[:, 1], box_pts[0, 1]), \n# color=color_name, linewidth=2.5)\n \n# ax_radar.set_title(f\"AI Radar β€” {len(pred_boxes_3d)} Targets Detected\", color='white', fontsize=16, fontweight='bold', pad=10)\n# ax_radar.axis('off')\n\n# cameras_to_plot = [('CAM_FRONT', ax_front), ('CAM_FRONT_LEFT', ax_fl), ('CAM_FRONT_RIGHT', ax_fr)]\n\n# for cam_name, ax in cameras_to_plot:\n# cam_tok = sample['data'][cam_name]\n# cam_d = lyft.get('sample_data', cam_tok)\n# cam_ego = lyft.get('ego_pose', cam_d['ego_pose_token'])\n# cal_s = lyft.get('calibrated_sensor', cam_d['calibrated_sensor_token'])\n# _, _, cam_intr = lyft.get_sample_data(cam_tok)\n# img = np.array(PILImage.open(lyft.get_sample_data_path(cam_tok)))\n \n# ax.imshow(img)\n# ax.set_xlim(0, img.shape[1])\n# ax.set_ylim(img.shape[0], 0)\n\n# for box, cls_name, score in zip(pred_boxes_3d, pred_classes, pred_scores):\n# b = box.copy()\n# b.translate(-np.array(cam_ego['translation']))\n# b.rotate(Quaternion(cam_ego['rotation']).inverse)\n# b.translate(-np.array(cal_s['translation']))\n# b.rotate(Quaternion(cal_s['rotation']).inverse)\n\n# corners_3d = b.corners()\n# corners_img = view_points(corners_3d, np.array(cam_intr), normalize=True)[:2, :]\n# in_front = corners_3d[2, :] > 0.1\n \n# in_image = ((corners_img[0, :] > 0) & (corners_img[0, :] < img.shape[1]) & \n# (corners_img[1, :] > 0) & (corners_img[1, :] < img.shape[0]))\n\n# if any(in_image) and all(in_front):\n# colors = CLASS_COLORS.get(cls_name, ('white', 'white', 'white'))\n# b.render(ax, view=np.array(cam_intr), normalize=True, colors=colors, linewidth=3)\n \n# min_x = max(10, np.min(corners_img[0, :]))\n# min_y = max(30, np.min(corners_img[1, :]))\n \n# label = f\"{cls_name[:3].upper()} {score*100:.0f}%\"\n# text_color = colors[0] if isinstance(colors[0], str) else 'white'\n \n# ax.text(min_x, min_y - 8, label, color=text_color, fontsize=12, fontweight='bold',\n# bbox=dict(facecolor='black', alpha=0.7, edgecolor='none', boxstyle='round,pad=0.3'))\n\n# ax.set_title(cam_name.replace('_', ' '), color='white', fontsize=18, fontweight='bold', pad=10)\n# ax.axis('off')\n\n# plt.tight_layout(pad=3.0)\n# fig.text(0.5, 0.96, f\"JUGAADI PERCEPTION ENGINE v5.0 | RANDOM UNSEEN DATA | Frame {frame_idx+1}/3\", \n# ha='center', color='red', fontsize=22, fontweight='bold')\n \n# plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ============================================================\n# # DATA LEAKAGE VERIFICATION\n# # ============================================================\n# print(\"πŸ” Verifying strict separation of Training and Unseen Data...\\n\")\n\n# train_token_set = set(train_tokens)\n# val_token_set = set(val_tokens)\n\n# overlap = train_token_set.intersection(val_token_set)\n\n# print(f\"Total Training Frames Memorized : {len(train_token_set):,}\")\n# print(f\"Total Unseen Validation Frames : {len(val_token_set):,}\")\n# print(f\"Overlapping Frames (Data Leakage): {len(overlap)}\")\n\n# if len(overlap) == 0:\n# print(\"\\nβœ… MATHEMATICAL PROOF: Zero data leakage detected.\")\n# print(\"The AI has never seen a single frame of the validation data.\")\n# else:\n# print(\"\\n🚨 WARNING: Data leakage detected! The model has memorized test data.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}